From b738283f6fb31087141b0ae5c958e455d83c1116 Mon Sep 17 00:00:00 2001 From: Emre Hasegeli Date: Wed, 23 Oct 2019 10:22:23 +0100 Subject: [PATCH 001/104] charts: Add pods/exec permission (#694) --- charts/postgres-operator/templates/clusterrole.yaml | 6 ++++++ manifests/operator-service-account-rbac.yaml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index a00ea3ab7..f8550a539 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -87,6 +87,12 @@ rules: - list - watch - patch +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create - apiGroups: - "" resources: diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index bca1128f4..e95fe320b 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -88,6 +88,12 @@ rules: - list - watch - patch +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create - apiGroups: - "" resources: From aab4d511428e25e7364909cd96d612b72670fd18 Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Mon, 28 Oct 2019 23:33:16 +0800 Subject: [PATCH 002/104] Fix bad formatting in docs (#637) List requires a proceeding empty line --- docs/administrator.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/administrator.md b/docs/administrator.md index e7ac1bd7b..ab5368e7d 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -184,6 +184,7 @@ parameter of the PDB is set to `1` which prevents killing masters in single-node clusters and/or the last remaining running instance in a multi-node cluster. The PDB is only relaxed in two scenarios: + * If a cluster is scaled down to `0` instances (e.g. for draining nodes) * If the PDB is disabled in the configuration (`enable_pod_disruption_budget`) @@ -387,7 +388,7 @@ manifest. Notes: backup via `pg_dumpall` and upload of compressed and encrypted results to an S3 bucket; the default image ``registry.opensource.zalan.do/acid/logical-backup`` is the same image built with the Zalando-internal CI pipeline. `pg_dumpall` -requires a `superuser` access to a DB and runs on the replica when possible. +requires a `superuser` access to a DB and runs on the replica when possible. 2. Due to the [limitation of K8s cron jobs](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) it is highly advisable to set up additional monitoring for this feature; such From 0a7b09e7913a9916e767c584a57c37a4c0919063 Mon Sep 17 00:00:00 2001 From: haf-afa <56340843+haf-afa@users.noreply.github.com> Date: Wed, 30 Oct 2019 16:21:07 +0100 Subject: [PATCH 003/104] Update quickstart.md (#699) --- docs/quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index 968adfde2..500d4db30 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -63,7 +63,7 @@ kubectl apply -k github.com/zalando/postgres-operator/manifests ``` For convenience, we have automated starting the operator with minikube using the -`run_operator_locally` script. It applies the [`acid-minimal-cluster`](../manifests/minimal-postgres-manifest). +`run_operator_locally` script. It applies the [`acid-minimal-cluster`](../manifests/minimal-postgres-manifest.yaml). manifest. ```bash From 719505afba8ed8a1499cbd52fade7028b9df0ab9 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 30 Oct 2019 16:51:54 +0100 Subject: [PATCH 004/104] enable GO111MODULE also on go test (#701) * enable GO111MODULE also on go test --- Makefile | 2 +- delivery.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6ffa29326..8710f17ee 100644 --- a/Makefile +++ b/Makefile @@ -93,7 +93,7 @@ deps: test: hack/verify-codegen.sh - @go test ./pkg/... + GO111MODULE=on go test ./... e2e: docker # build operator image to be tested cd e2e; make tools test clean diff --git a/delivery.yaml b/delivery.yaml index 1a18c16f9..ef249292c 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -32,7 +32,7 @@ pipeline: - desc: 'Run unit tests' cmd: | export PATH=$PATH:$HOME/go/bin - go test ./pkg/... + go test ./... - desc: 'Run e2e tests' cmd: | make e2e From 340dc4aa3dc6c0ff9d575ca58bc6a7040c959171 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 31 Oct 2019 12:19:41 +0100 Subject: [PATCH 005/104] Fix operator_configuration_type.go (#703) fix typo --- pkg/apis/acid.zalan.do/v1/operator_configuration_type.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 6f1c8d59c..02648b83b 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -45,7 +45,7 @@ type KubernetesMetaConfiguration struct { PodServiceAccountDefinition string `json:"pod_service_account_definition,omitempty"` PodServiceAccountRoleBindingDefinition string `json:"pod_service_account_role_binding_definition,omitempty"` PodTerminateGracePeriod Duration `json:"pod_terminate_grace_period,omitempty"` - SpiloPrivileged bool `json:"spilo_privileged,omitemty"` + SpiloPrivileged bool `json:"spilo_privileged,omitempty"` SpiloFSGroup *int64 `json:"spilo_fsgroup,omitempty"` WatchedNamespace string `json:"watched_namespace,omitempty"` PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` From 33e1d607036db78aaea9d2d239b689527a235055 Mon Sep 17 00:00:00 2001 From: Emre Hasegeli Date: Fri, 8 Nov 2019 07:20:43 +0000 Subject: [PATCH 006/104] docs: Fix links on operator parameters reference (#712) --- docs/reference/operator_parameters.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 3e21340d5..41e62cc7f 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -10,12 +10,12 @@ configuration. maps. String values containing ':' should be enclosed in quotes. The configuration is flat, parameter group names below are not reflected in the configuration structure. There is an - [example](../manifests/configmap.yaml) + [example](../../manifests/configmap.yaml) * CRD-based configuration. The configuration is stored in a custom YAML manifest. The manifest is an instance of the custom resource definition (CRD) called `OperatorConfiguration`. The operator registers this CRD during the - start and uses it for configuration if the [operator deployment manifest](../manifests/postgres-operator.yaml#L36) + start and uses it for configuration if the [operator deployment manifest](../../manifests/postgres-operator.yaml#L36) sets the `POSTGRES_OPERATOR_CONFIGURATION_OBJECT` env variable to a non-empty value. The variable should point to the `postgresql-operator-configuration` object in the operator's namespace. @@ -24,7 +24,7 @@ configuration. simply represented in the usual YAML way. There are no default values built-in in the operator, each parameter that is not supplied in the configuration receives an empty value. In order to create your own configuration just copy - the [default one](../manifests/postgresql-operator-default-configuration.yaml) + the [default one](../../manifests/postgresql-operator-default-configuration.yaml) and change it. To test the CRD-based configuration locally, use the following @@ -58,11 +58,11 @@ parameters, those parameters have no effect and are replaced by the `CRD_READY_WAIT_INTERVAL` and `CRD_READY_WAIT_TIMEOUT` environment variables. They will be deprecated and removed in the future. -For the configmap configuration, the [default parameter values](../pkg/util/config/config.go#L14) +For the configmap configuration, the [default parameter values](../../pkg/util/config/config.go#L14) mentioned here are likely to be overwritten in your local operator installation via your local version of the operator configmap. In the case you use the operator CRD, all the CRD defaults are provided in the -[operator's default configuration manifest](../manifests/postgresql-operator-default-configuration.yaml) +[operator's default configuration manifest](../../manifests/postgresql-operator-default-configuration.yaml) Variable names are underscore-separated words. @@ -122,7 +122,7 @@ Those are top-level keys, containing both leaf keys and groups. containers with high memory limits due to the lack of memory on Kubernetes cluster nodes. This affects all containers created by the operator (Postgres, Scalyr sidecar, and other sidecars); to set resources for the operator's own - container, change the [operator deployment manually](../manifests/postgres-operator.yaml#L20). + container, change the [operator deployment manually](../../manifests/postgres-operator.yaml#L20). The default is `false`. ## Postgres users From 535517cd1b0cef2c307eb1592fc8e17fad918f34 Mon Sep 17 00:00:00 2001 From: Thomas Runyon Date: Mon, 11 Nov 2019 04:45:35 -0500 Subject: [PATCH 007/104] Custom annotations 329 (#657) * Add ability for custom annotations to database pods --- charts/postgres-operator/values-crd.yaml | 4 +- charts/postgres-operator/values.yaml | 5 +- docs/reference/cluster_manifest.md | 5 ++ docs/reference/operator_parameters.md | 5 ++ manifests/complete-postgres-manifest.yaml | 3 +- manifests/configmap.yaml | 10 ++-- ...gresql-operator-default-configuration.yaml | 3 + .../v1/operator_configuration_type.go | 1 + pkg/apis/acid.zalan.do/v1/postgresql_type.go | 1 + pkg/apis/acid.zalan.do/v1/util_test.go | 29 ++++++++++ .../acid.zalan.do/v1/zz_generated.deepcopy.go | 14 +++++ pkg/cluster/cluster_test.go | 56 ++++++++++++++++++- pkg/cluster/k8sres.go | 35 +++++++++++- pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 1 + 15 files changed, 160 insertions(+), 13 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 2728b245c..e1ed381cc 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -53,9 +53,11 @@ configKubernetes: cluster_domain: cluster.local # additional labels assigned to the cluster objects cluster_labels: - application: spilo + application: spilo # label assigned to Kubernetes objects created by the operator cluster_name_label: cluster-name + # additional annotations to add to every database pod + custom_pod_annotations: # toggles pod anti affinity on the Postgres pods enable_pod_antiaffinity: false # toggles PDB to set to MinAvailabe 0 or 1 diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index a14c8ab92..2af14984b 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -54,6 +54,8 @@ configKubernetes: cluster_labels: application:spilo # label assigned to Kubernetes objects created by the operator cluster_name_label: version + # annotations attached to each database pod + # custom_pod_annotations: keya:valuea # toggles pod anti affinity on the Postgres pods enable_pod_antiaffinity: "false" # toggles PDB to set to MinAvailabe 0 or 1 @@ -127,8 +129,7 @@ configLoadBalancer: # DNS zone for cluster DNS name when load balancer is configured for cluster db_hosted_zone: db.example.com # annotations to apply to service when load balancing is enabled - # custom_service_annotations: - # "keyx:valuez,keya:valuea" + # custom_service_annotations: "keyx:valuez,keya:valuea" # toggles service type load balancer pointing to the master pod of the cluster enable_master_load_balancer: "true" diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index edaada9b4..cf522d73d 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -118,6 +118,11 @@ These parameters are grouped directly under the `spec` key in the manifest. then the default priority class is taken. The priority class itself must be defined in advance. Optional. +* **podAnnotations** + A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) + to each pod created for the database. + + * **enableShmVolume** Start a database pod without limitations on shm memory. By default docker limit `/dev/shm` to `64M` (see e.g. the [docker diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 41e62cc7f..9e43919fd 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -168,6 +168,11 @@ configuration they are grouped under the `kubernetes` key. Postgres pods are [terminated forcefully](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods) after this timeout. The default is `5m`. +* **custom_pod_annotations** + This key/value map provides a list of annotations that get attached to each pod + of a database created by the operator. If the annotation key is also provided + by the database definition, the database definition value is used. + * **watched_namespace** The operator watches for Postgres objects in the given namespace. If not specified, the value is taken from the operator namespace. A special `*` diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index d1bd471db..402d007e6 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -25,7 +25,8 @@ spec: - 127.0.0.1/32 databases: foo: zalando - +# podAnnotations: +# annotation.key: value # Expert section enableShmVolume: true diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index f9223f3fe..879223a57 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -11,8 +11,8 @@ data: cluster_history_entries: "1000" cluster_labels: application:spilo cluster_name_label: version - # custom_service_annotations: - # "keyx:valuez,keya:valuea" + # custom_service_annotations: "keyx:valuez,keya:valuea" + # custom_pod_annotations: "keya:valuea" db_hosted_zone: db.example.com debug_logging: "true" # default_cpu_limit: "3" @@ -37,7 +37,7 @@ data: # logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" # logical_backup_s3_bucket: "my-bucket-url" # logical_backup_schedule: "30 00 * * *" - master_dns_name_format: '{cluster}.{team}.staging.{hostedzone}' + master_dns_name_format: "{cluster}.{team}.staging.{hostedzone}" # master_pod_move_timeout: 10m # max_instances: "-1" # min_instances: "-1" @@ -60,13 +60,13 @@ data: ready_wait_interval: 3s ready_wait_timeout: 30s repair_period: 5m - replica_dns_name_format: '{cluster}-repl.{team}.staging.{hostedzone}' + replica_dns_name_format: "{cluster}-repl.{team}.staging.{hostedzone}" replication_username: standby resource_check_interval: 3s resource_check_timeout: 10m resync_period: 5m ring_log_lines: "100" - secret_name_template: '{username}.{cluster}.credentials' + secret_name_template: "{username}.{cluster}.credentials" # sidecar_docker_images: "" # set_memory_request_to_limit: "false" spilo_privileged: "false" diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index ad4d028c3..e1ecd1038 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -22,6 +22,9 @@ configuration: cluster_labels: application: spilo cluster_name_label: cluster-name + # custom_pod_annotations: + # keya: valuea + # keyb: valueb enable_pod_antiaffinity: false enable_pod_disruption_budget: true # infrastructure_roles_secret_name: "" diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 02648b83b..6adfb778e 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -59,6 +59,7 @@ type KubernetesMetaConfiguration struct { InheritedLabels []string `json:"inherited_labels,omitempty"` ClusterNameLabel string `json:"cluster_name_label,omitempty"` NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` + CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` // TODO: use a proper toleration structure? PodToleration map[string]string `json:"toleration,omitempty"` // TODO: use namespacedname diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 72e40e122..515a73ff0 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -59,6 +59,7 @@ type PostgresSpec struct { EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"` LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` StandbyCluster *StandbyDescription `json:"standby"` + PodAnnotations map[string]string `json:"podAnnotations"` // deprecated json tags InitContainersOld []v1.Container `json:"init_containers,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index 6c1b63ece..cf3b080a5 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -437,6 +437,16 @@ var postgresqlList = []struct { PostgresqlList{}, errors.New("unexpected end of JSON input")}} +var annotations = []struct { + in []byte + annotations map[string]string + err error +}{{ + in: []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "acid-testcluster1"}, "spec": {"podAnnotations": {"foo": "bar"},"teamId": "acid", "clone": {"cluster": "team-batman"}}}`), + annotations: map[string]string{"foo": "bar"}, + err: nil}, +} + func mustParseTime(s string) metav1.Time { v, err := time.Parse("15:04", s) if err != nil { @@ -482,6 +492,25 @@ func TestWeekdayTime(t *testing.T) { } } +func TestClusterAnnotations(t *testing.T) { + for _, tt := range annotations { + var cluster Postgresql + err := cluster.UnmarshalJSON(tt.in) + if err != nil { + if tt.err == nil || err.Error() != tt.err.Error() { + t.Errorf("Unable to marshal cluster with annotations: expected %v got %v", tt.err, err) + } + continue + } + for k, v := range cluster.Spec.PodAnnotations { + found, expected := v, tt.annotations[k] + if found != expected { + t.Errorf("Didn't find correct value for key %v in for podAnnotations: Expected %v found %v", k, expected, found) + } + } + } +} + func TestClusterName(t *testing.T) { for _, tt := range clusterNames { name, err := extractClusterName(tt.in, tt.inTeam) diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 8e3411219..793f236a5 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -102,6 +102,13 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura (*out)[key] = val } } + if in.CustomPodAnnotations != nil { + in, out := &in.CustomPodAnnotations, &out.CustomPodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.PodToleration != nil { in, out := &in.PodToleration, &out.PodToleration *out = make(map[string]string, len(*in)) @@ -513,6 +520,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = new(StandbyDescription) **out = **in } + if in.PodAnnotations != nil { + in, out := &in.PodAnnotations, &out.PodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.InitContainersOld != nil { in, out := &in.InitContainersOld, &out.InitContainersOld *out = make([]corev1.Container, len(*in)) diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 6f10aae22..85d014d8a 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -11,7 +11,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/teams" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) const ( @@ -328,3 +328,57 @@ func TestShouldDeleteSecret(t *testing.T) { } } } + +func TestPodAnnotations(t *testing.T) { + testName := "TestPodAnnotations" + tests := []struct { + subTest string + operator map[string]string + database map[string]string + merged map[string]string + }{ + { + subTest: "No Annotations", + operator: make(map[string]string), + database: make(map[string]string), + merged: make(map[string]string), + }, + { + subTest: "Operator Config Annotations", + operator: map[string]string{"foo": "bar"}, + database: make(map[string]string), + merged: map[string]string{"foo": "bar"}, + }, + { + subTest: "Database Config Annotations", + operator: make(map[string]string), + database: map[string]string{"foo": "bar"}, + merged: map[string]string{"foo": "bar"}, + }, + { + subTest: "Database Config overrides Operator Config Annotations", + operator: map[string]string{"foo": "bar", "global": "foo"}, + database: map[string]string{"foo": "baz", "local": "foo"}, + merged: map[string]string{"foo": "baz", "global": "foo", "local": "foo"}, + }, + } + + for _, tt := range tests { + cl.OpConfig.CustomPodAnnotations = tt.operator + cl.Postgresql.Spec.PodAnnotations = tt.database + + annotations := cl.generatePodAnnotations(&cl.Postgresql.Spec) + for k, v := range annotations { + if observed, expected := v, tt.merged[k]; observed != expected { + t.Errorf("%v expects annotation value %v for key %v, but found %v", + testName+"/"+tt.subTest, expected, observed, k) + } + } + for k, v := range tt.merged { + if observed, expected := annotations[k], v; observed != expected { + t.Errorf("%v expects annotation value %v for key %v, but found %v", + testName+"/"+tt.subTest, expected, observed, k) + } + } + } +} diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 78d128387..ddadf8d52 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -430,6 +430,7 @@ func mountShmVolumeNeeded(opConfig config.Config, pgSpec *acidv1.PostgresSpec) * func generatePodTemplate( namespace string, labels labels.Set, + annotations map[string]string, spiloContainer *v1.Container, initContainers []v1.Container, sidecarContainers []v1.Container, @@ -485,13 +486,17 @@ func generatePodTemplate( template := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - Namespace: namespace, + Labels: labels, + Namespace: namespace, + Annotations: annotations, }, Spec: podSpec, } if kubeIAMRole != "" { - template.Annotations = map[string]string{constants.KubeIAmAnnotation: kubeIAMRole} + if template.Annotations == nil{ + template.Annotations = make(map[string]string) + } + template.Annotations[constants.KubeIAmAnnotation] = kubeIAMRole } return &template, nil @@ -881,10 +886,13 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef effectiveFSGroup = spec.SpiloFSGroup } + annotations := c.generatePodAnnotations(spec) + // generate pod template for the statefulset, based on the spilo container and sidecars if podTemplate, err = generatePodTemplate( c.Namespace, c.labelsSet(true), + annotations, spiloContainer, spec.InitContainers, sidecarContainers, @@ -949,6 +957,24 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef return statefulSet, nil } +func (c *Cluster) generatePodAnnotations(spec *acidv1.PostgresSpec) map[string]string { + annotations := make(map[string]string) + for k, v := range c.OpConfig.CustomPodAnnotations { + annotations[k] = v + } + if spec != nil || spec.PodAnnotations != nil { + for k, v := range spec.PodAnnotations { + annotations[k] = v + } + } + + if len(annotations) == 0 { + return nil + } + + return annotations +} + func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage string, containerResources *acidv1.Resources, logger *logrus.Entry) *acidv1.Sidecar { if APIKey == "" || dockerImage == "" { @@ -1462,10 +1488,13 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { }, }} + annotations := c.generatePodAnnotations(&c.Spec) + // re-use the method that generates DB pod templates if podTemplate, err = generatePodTemplate( c.Namespace, labels, + annotations, logicalBackupContainer, []v1.Container{}, []v1.Container{}, diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 9da4bbaf8..b91fd511f 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -41,6 +41,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ReplicationUsername = fromCRD.PostgresUsersConfiguration.ReplicationUsername // kubernetes config + result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations result.PodServiceAccountName = fromCRD.Kubernetes.PodServiceAccountName result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 5dd25d401..bdbbf0fdc 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -109,6 +109,7 @@ type Config struct { EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"` EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"` CustomServiceAnnotations map[string]string `name:"custom_service_annotations"` + CustomPodAnnotations map[string]string `name:"custom_pod_annotations"` EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"` // deprecated and kept for backward compatibility From de14323f8ed11f087722f7424b49faf8cfb79ca8 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 12 Nov 2019 10:57:48 +0100 Subject: [PATCH 008/104] disable load balancers by default in examples (#715) --- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- manifests/complete-postgres-manifest.yaml | 4 ++-- manifests/configmap.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index e1ed381cc..06a5b483f 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -138,7 +138,7 @@ configLoadBalancer: # keya: valuea # toggles service type load balancer pointing to the master pod of the cluster - enable_master_load_balancer: true + enable_master_load_balancer: false # toggles service type load balancer pointing to the replica pod of the cluster enable_replica_load_balancer: false # defines the DNS name string template for the master load balancer cluster diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 2af14984b..2c499a036 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -132,7 +132,7 @@ configLoadBalancer: # custom_service_annotations: "keyx:valuez,keya:valuea" # toggles service type load balancer pointing to the master pod of the cluster - enable_master_load_balancer: "true" + enable_master_load_balancer: "false" # toggles service type load balancer pointing to the replica pod of the cluster enable_replica_load_balancer: "false" # defines the DNS name string template for the master load balancer cluster diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 402d007e6..34dd6cf83 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -19,8 +19,8 @@ spec: zalando: - superuser - createdb - enableMasterLoadBalancer: true - enableReplicaLoadBalancer: true + enableMasterLoadBalancer: false + enableReplicaLoadBalancer: false allowedSourceRanges: # load balancers' source ranges for both master and replica services - 127.0.0.1/32 databases: diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 879223a57..c64e45cbd 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -22,7 +22,7 @@ data: docker_image: registry.opensource.zalan.do/acid/spilo-11:1.6-p1 # enable_admin_role_for_users: "true" # enable_database_access: "true" - enable_master_load_balancer: "true" + enable_master_load_balancer: "false" # enable_pod_antiaffinity: "false" # enable_pod_disruption_budget: "true" enable_replica_load_balancer: "false" From 63d1f3bbe4ec456d8025f18c5e47cb3f872ea756 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 18 Nov 2019 12:13:05 +0100 Subject: [PATCH 009/104] add get for deployments to operator RBAC (#724) --- manifests/operator-service-account-rbac.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index e95fe320b..9be8c7ab9 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -103,6 +103,12 @@ rules: - delete - get - patch +- apiGroups: + - apps + resources: + - deployments + verbs: + - get - apiGroups: - apps resources: From 0b544ae43f2081ccd9bb1d1dc067b43473becec6 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 19 Nov 2019 18:06:55 +0100 Subject: [PATCH 010/104] pass additionalSecretMount to logical backup pod (#714) --- pkg/cluster/k8sres.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index ddadf8d52..a0d9604fd 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -493,7 +493,7 @@ func generatePodTemplate( Spec: podSpec, } if kubeIAMRole != "" { - if template.Annotations == nil{ + if template.Annotations == nil { template.Annotations = make(map[string]string) } template.Annotations[constants.KubeIAmAnnotation] = kubeIAMRole @@ -967,7 +967,7 @@ func (c *Cluster) generatePodAnnotations(spec *acidv1.PostgresSpec) map[string]s annotations[k] = v } } - + if len(annotations) == 0 { return nil } @@ -1508,8 +1508,8 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { util.False(), false, "", - "", - ""); err != nil { + c.OpConfig.AdditionalSecretMount, + c.OpConfig.AdditionalSecretMountPath); err != nil { return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) } From f9487e41c1cdfc27ef461fb35fe35e0df2f0e121 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 20 Nov 2019 13:58:41 +0100 Subject: [PATCH 011/104] inject cluster name label into logical backup pod (#725) * inject cluster name label into logical backup pod --- docker/logical-backup/dump.sh | 41 +++----------------- manifests/operator-service-account-rbac.yaml | 6 --- pkg/cluster/k8sres.go | 4 ++ 3 files changed, 9 insertions(+), 42 deletions(-) diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh index c3cf5ba67..ee64bd811 100755 --- a/docker/logical-backup/dump.sh +++ b/docker/logical-backup/dump.sh @@ -14,9 +14,8 @@ PG_BIN=$PG_DIR/$PG_VERSION/bin DUMP_SIZE_COEFF=5 TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) -K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT +K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1 CERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt -CLUSTER_NAME_LABEL=cluster-name function estimate_size { "$PG_BIN"/psql -tqAc "${ALL_DB_SIZE_QUERY}" @@ -49,53 +48,23 @@ function aws_upload { function get_pods { declare -r SELECTOR="$1" - curl "${K8S_API_URL}/api/v1/namespaces/${POD_NAMESPACE}/pods?$SELECTOR" \ - --cacert $CERT \ + curl "${K8S_API_URL}/namespaces/${POD_NAMESPACE}/pods?$SELECTOR" \ + --cacert $CERT \ -H "Authorization: Bearer ${TOKEN}" | jq .items[].status.podIP -r } function get_current_pod { - curl "${K8S_API_URL}/api/v1/namespaces/${POD_NAMESPACE}/pods?fieldSelector=metadata.name%3D${HOSTNAME}" \ - --cacert $CERT \ + curl "${K8S_API_URL}/namespaces/${POD_NAMESPACE}/pods?fieldSelector=metadata.name%3D${HOSTNAME}" \ + --cacert $CERT \ -H "Authorization: Bearer ${TOKEN}" } declare -a search_strategy=( - get_cluster_name_label list_all_replica_pods_current_node list_all_replica_pods_any_node get_master_pod ) -function get_config_resource() { - curl "${K8S_API_URL}/apis/apps/v1/namespaces/default/deployments/postgres-operator" \ - --cacert $CERT \ - -H "Authorization: Bearer ${TOKEN}" | jq '.spec.template.spec.containers[0].env[] | select(.name == "$1") | .value' -} - -function get_cluster_name_label { - local config - local clustername - - config=$(get_config_resource "CONFIG_MAP_NAME") - if [ -n "$config" ]; then - clustername=$(curl "${K8S_API_URL}/api/v1/namespaces/default/configmaps/${config}" \ - --cacert $CERT \ - -H "Authorization: Bearer ${TOKEN}" | jq '.data.cluster_name_label') - else - config=$(get_config_resource "POSTGRES_OPERATOR_CONFIGURATION_OBJECT") - if [ -n "$config" ]; then - clustername=$(curl "${K8S_API_URL}/apis/acid.zalan.do/v1/namespaces/default/operatorconfigurations/${config}" \ - --cacert $CERT \ - -H "Authorization: Bearer ${TOKEN}" | jq '.configuration.kubernetes.cluster_name_label') - fi - fi - - if [ -n "$clustername" ]; then - CLUSTER_NAME_LABEL=${clustername} - fi; -} - function list_all_replica_pods_current_node { get_pods "labelSelector=${CLUSTER_NAME_LABEL}%3D${SCOPE},spilo-role%3Dreplica&fieldSelector=spec.nodeName%3D${CURRENT_NODENAME}" | head -n 1 } diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 9be8c7ab9..e95fe320b 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -103,12 +103,6 @@ rules: - delete - get - patch -- apiGroups: - - apps - resources: - - deployments - verbs: - - get - apiGroups: - apps resources: diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index a0d9604fd..d5cc4ce39 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1557,6 +1557,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { Name: "SCOPE", Value: c.Name, }, + { + Name: "CLUSTER_NAME_LABEL", + Value: c.OpConfig.ClusterNameLabel, + }, { Name: "POD_NAMESPACE", ValueFrom: &v1.EnvVarSource{ From 2ce602fcd7a472d3c586f1619efdbacdb8418d4f Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 26 Nov 2019 10:28:32 +0100 Subject: [PATCH 012/104] fix errors when changing service type (#716) * fix errors when changing service type * nullify service and endpoint before recreation * improve wait for delete logic and reuse config parameters --- pkg/cluster/resources.go | 46 +++++++++++++++++++++++++------- pkg/util/constants/kubernetes.go | 8 +++--- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 23ac3f348..3e8f73916 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -13,7 +13,6 @@ import ( "k8s.io/apimachinery/pkg/types" "github.com/zalando/postgres-operator/pkg/util" - "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/retryutil" ) @@ -278,7 +277,8 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error { oldStatefulset := c.Statefulset options := metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy} - if err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Delete(oldStatefulset.Name, &options); err != nil { + err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Delete(oldStatefulset.Name, &options) + if err != nil { return fmt.Errorf("could not delete statefulset %q: %v", statefulSetName, err) } // make sure we clear the stored statefulset status if the subsequent create fails. @@ -286,11 +286,16 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error { // wait until the statefulset is truly deleted c.logger.Debugf("waiting for the statefulset to be deleted") - err := retryutil.Retry(constants.StatefulsetDeletionInterval, constants.StatefulsetDeletionTimeout, + err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, func() (bool, error) { - _, err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Get(oldStatefulset.Name, metav1.GetOptions{}) - - return err != nil, nil + _, err2 := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Get(oldStatefulset.Name, metav1.GetOptions{}) + if err2 == nil { + return false, nil + } + if k8sutil.ResourceNotFound(err2) { + return true, nil + } + return false, err2 }) if err != nil { return fmt.Errorf("could not delete statefulset: %v", err) @@ -380,13 +385,27 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error return fmt.Errorf("could not delete service %q: %v", serviceName, err) } - c.Endpoints[role] = nil - svc, err := c.KubeClient.Services(serviceName.Namespace).Create(newService) + // wait until the service is truly deleted + c.logger.Debugf("waiting for service to be deleted") + + err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, + func() (bool, error) { + _, err2 := c.KubeClient.Services(serviceName.Namespace).Get(serviceName.Name, metav1.GetOptions{}) + if err2 == nil { + return false, nil + } + if k8sutil.ResourceNotFound(err2) { + return true, nil + } + return false, err2 + }) if err != nil { - return fmt.Errorf("could not create service %q: %v", serviceName, err) + return fmt.Errorf("could not delete service %q: %v", serviceName, err) } - c.Services[role] = svc + // make sure we clear the stored service and endpoint status if the subsequent create fails. + c.Services[role] = nil + c.Endpoints[role] = nil if role == Master { // create the new endpoint using the addresses obtained from the previous one endpointSpec := c.generateEndpoint(role, currentEndpoint.Subsets) @@ -398,6 +417,13 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error c.Endpoints[role] = ep } + svc, err := c.KubeClient.Services(serviceName.Namespace).Create(newService) + if err != nil { + return fmt.Errorf("could not create service %q: %v", serviceName, err) + } + + c.Services[role] = svc + return nil } diff --git a/pkg/util/constants/kubernetes.go b/pkg/util/constants/kubernetes.go index a4ea73e80..be79687eb 100644 --- a/pkg/util/constants/kubernetes.go +++ b/pkg/util/constants/kubernetes.go @@ -4,11 +4,9 @@ import "time" // General kubernetes-related constants const ( - PostgresContainerName = "postgres" - PostgresContainerIdx = 0 - K8sAPIPath = "/apis" - StatefulsetDeletionInterval = 1 * time.Second - StatefulsetDeletionTimeout = 30 * time.Second + PostgresContainerName = "postgres" + PostgresContainerIdx = 0 + K8sAPIPath = "/apis" QueueResyncPeriodPod = 5 * time.Minute QueueResyncPeriodTPR = 5 * time.Minute From 5f87384d7f1b80805e909f50ca773d46d3609321 Mon Sep 17 00:00:00 2001 From: Armin Nesiren Date: Tue, 26 Nov 2019 10:40:49 +0100 Subject: [PATCH 013/104] Passing endpoint, access and secret key to logical-backup container (#628) * Added possibility to add custom annotations to LoadBalancer service. * Added parameters for custom endpoint, access and secret key for logical backup. * Modified dump.sh so it knows how to handle new features. Configurable S3 SSE --- charts/postgres-operator/values-crd.yaml | 12 ++++++++++-- charts/postgres-operator/values.yaml | 12 ++++++++++-- docker/logical-backup/dump.sh | 12 +++++++----- docs/reference/operator_parameters.md | 13 +++++++++++++ manifests/configmap.yaml | 4 ++++ ...stgresql-operator-default-configuration.yaml | 7 ++++++- .../v1/operator_configuration_type.go | 11 ++++++++--- pkg/cluster/k8sres.go | 17 ++++++++++++++++- pkg/controller/operator_config.go | 4 ++++ pkg/util/config/config.go | 12 ++++++++---- 10 files changed, 86 insertions(+), 18 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 06a5b483f..af05ae56c 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -184,12 +184,20 @@ configAwsOrGcp: # configure K8s cron job managed by the operator configLogicalBackup: - # backup schedule in the cron format - logical_backup_schedule: "30 00 * * *" # image for pods of the logical backup job (example runs pg_dumpall) logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + # S3 Access Key ID + logical_backup_s3_access_key_id: "" # S3 bucket to store backup results logical_backup_s3_bucket: "my-bucket-url" + # S3 endpoint url when not using AWS + logical_backup_s3_endpoint: "" + # S3 Secret Access Key + logical_backup_s3_secret_access_key: "" + # S3 server side encription + logical_backup_s3_sse: "AES256" + # backup schedule in the cron format + logical_backup_schedule: "30 00 * * *" # automate creation of human users with teams API service configTeamsApi: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 2c499a036..b572b5844 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -178,12 +178,20 @@ configAwsOrGcp: # configure K8s cron job managed by the operator configLogicalBackup: - # backup schedule in the cron format - logical_backup_schedule: "30 00 * * *" # image for pods of the logical backup job (example runs pg_dumpall) logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + # S3 Access Key ID + logical_backup_s3_access_key_id: "" # S3 bucket to store backup results logical_backup_s3_bucket: "my-bucket-url" + # S3 endpoint url when not using AWS + logical_backup_s3_endpoint: "" + # S3 Secret Access Key + logical_backup_s3_secret_access_key: "" + # S3 server side encription + logical_backup_s3_sse: "AES256" + # backup schedule in the cron format + logical_backup_schedule: "30 00 * * *" # automate creation of human users with teams API service configTeamsApi: diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh index ee64bd811..5c2e478a9 100755 --- a/docker/logical-backup/dump.sh +++ b/docker/logical-backup/dump.sh @@ -38,11 +38,13 @@ function aws_upload { # NB: $LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX already contains the leading "/" when set by the Postgres Operator PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz - if [ -z "$EXPECTED_SIZE" ]; then - aws s3 cp - "$PATH_TO_BACKUP" --debug --sse="AES256" - else - aws s3 cp - "$PATH_TO_BACKUP" --debug --expected-size "$EXPECTED_SIZE" --sse="AES256" - fi; + args=() + + [[ ! -z "$EXPECTED_SIZE" ]] && args+=("--expected-size=$EXPECTED_SIZE") + [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=\"$LOGICAL_BACKUP_S3_ENDPOINT\"") + [[ ! "$LOGICAL_BACKUP_S3_SSE" == "" ]] && args+=("--sse=\"$LOGICAL_BACKUP_S3_SSE\"") + + aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}" --debug } function get_pods { diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 9e43919fd..a024aad1f 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -441,6 +441,19 @@ grouped under the `logical_backup` key. S3 bucket to store backup results. The bucket has to be present and accessible by Postgres pods. Default: empty. +* **logical_backup_s3_endpoint** + When using non-AWS S3 storage, endpoint can be set as a ENV variable. + +* **logical_backup_s3_sse** + Specify server side encription that S3 storage is using. If empty string + is specified, no argument will be passed to `aws s3` command. Default: "AES256". + +* **logical_backup_s3_access_key_id** + When set, value will be in AWS_ACCESS_KEY_ID env variable. The Default is empty. + +* **logical_backup_s3_secret_access_key** + When set, value will be in AWS_SECRET_ACCESS_KEY env variable. The Default is empty. + ## Debugging the operator Options to aid debugging of the operator itself. Grouped under the `debug` key. diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index c64e45cbd..40aca9716 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -35,7 +35,11 @@ data: # kube_iam_role: "" # log_s3_bucket: "" # logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + # logical_backup_s3_access_key_id: "" # logical_backup_s3_bucket: "my-bucket-url" + # logical_backup_s3_endpoint: "" + # logical_backup_s3_secret_access_key: "" + # logical_backup_s3_sse: "AES256" # logical_backup_schedule: "30 00 * * *" master_dns_name_format: "{cluster}.{team}.staging.{hostedzone}" # master_pod_move_timeout: 10m diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index e1ecd1038..94f91f1f0 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -74,9 +74,14 @@ configuration: # log_s3_bucket: "" # wal_s3_bucket: "" logical_backup: - logical_backup_schedule: "30 00 * * *" + log_s3_bucket: "" logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + logical_backup_s3_access_key_id: "" logical_backup_s3_bucket: "my-bucket-url" + logical_backup_s3_endpoint: "" + logical_backup_s3_secret_access_key: "" + logical_backup_s3_sse: "AES256" + logical_backup_schedule: "30 00 * * *" debug: debug_logging: true enable_database_access: true diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 6adfb778e..d00d40532 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -184,8 +184,13 @@ type OperatorConfigurationUsers struct { //Duration shortens this frequently used name type Duration time.Duration +// OperatorLogicalBackupConfiguration defines configration for logical backup type OperatorLogicalBackupConfiguration struct { - Schedule string `json:"logical_backup_schedule,omitempty"` - DockerImage string `json:"logical_backup_docker_image,omitempty"` - S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` + Schedule string `json:"logical_backup_schedule,omitempty"` + DockerImage string `json:"logical_backup_docker_image,omitempty"` + S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` + S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"` + S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` + S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"` + S3SSE string `json:"logical_backup_s3_sse,omitempty"` } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index d5cc4ce39..95e83454f 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1575,6 +1575,14 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { Name: "LOGICAL_BACKUP_S3_BUCKET", Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket, }, + { + Name: "LOGICAL_BACKUP_S3_ENDPOINT", + Value: c.OpConfig.LogicalBackup.LogicalBackupS3Endpoint, + }, + { + Name: "LOGICAL_BACKUP_S3_SSE", + Value: c.OpConfig.LogicalBackup.LogicalBackupS3SSE, + }, { Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())), @@ -1613,8 +1621,15 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { }, } - c.logger.Debugf("Generated logical backup env vars %v", envVars) + if c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID != "" { + envVars = append(envVars, v1.EnvVar{Name: "AWS_ACCESS_KEY_ID", Value: c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID}) + } + if c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey != "" { + envVars = append(envVars, v1.EnvVar{Name: "AWS_SECRET_ACCESS_KEY", Value: c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey}) + } + + c.logger.Debugf("Generated logical backup env vars %v", envVars) return envVars } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index b91fd511f..3ea513879 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -101,6 +101,10 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.LogicalBackupSchedule = fromCRD.LogicalBackup.Schedule result.LogicalBackupDockerImage = fromCRD.LogicalBackup.DockerImage result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket + result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint + result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID + result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey + result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE // debug config result.DebugLogging = fromCRD.OperatorDebug.DebugLogging diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index bdbbf0fdc..e8de85e20 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -68,11 +68,15 @@ type Scalyr struct { ScalyrMemoryLimit string `name:"scalyr_memory_limit" default:"1Gi"` } -// LogicalBackup +// LogicalBackup defines configration for logical backup type LogicalBackup struct { - LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` - LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"` - LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` + LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` + LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"` + LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` + LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` + LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` + LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` + LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:"AES256"` } // Config describes operator config From 052940862ae96756a3643aab3ade1a3fba004908 Mon Sep 17 00:00:00 2001 From: Jonas Brunsgaard Date: Wed, 27 Nov 2019 16:43:46 +0100 Subject: [PATCH 014/104] Introduce crds directory for compatibility with Helm v3 (#738) * Introduce crds directory for compatibility with Helm v3 This commit introduce a crd directory for the helm chart which has all custom resource definitions. The files in the crd directory is plain YAML. The crds got the label `app.kubernetes.io/name: postgres-operator` and removes all the templating. Helm v3 ignores the objects from the `templates` directory which have a `crd-install` hook. This commit addes templates/crds.yaml which generates YAMLs for CRDs. The hooks from these CRDs are detected by Helm v2 as well as v3. Helm v2 executes the hook and Helm v3 ignores the hook (YAML files are not applied) The approach is inspired by the prometheus-operator chart helm/charts@89b233eef6dbc1b6fac418bde3a5a6f4e14406d4 --- .../crds/operatorconfigurations.yaml | 41 +++++++ .../postgres-operator/crds/postgresqls.yaml | 53 +++++++++ charts/postgres-operator/templates/crds.yaml | 6 + .../templates/customrresourcedefinition.yaml | 103 ------------------ docs/quickstart.md | 24 ++-- 5 files changed, 112 insertions(+), 115 deletions(-) create mode 100644 charts/postgres-operator/crds/operatorconfigurations.yaml create mode 100644 charts/postgres-operator/crds/postgresqls.yaml create mode 100644 charts/postgres-operator/templates/crds.yaml delete mode 100644 charts/postgres-operator/templates/customrresourcedefinition.yaml diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml new file mode 100644 index 000000000..acf132edd --- /dev/null +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -0,0 +1,41 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: operatorconfigurations.acid.zalan.do + labels: + app.kubernetes.io/name: postgres-operator + annotations: + "helm.sh/hook": crd-install +spec: + group: acid.zalan.do + names: + kind: OperatorConfiguration + listKind: OperatorConfigurationList + plural: operatorconfigurations + singular: operatorconfiguration + shortNames: + - opconfig + additionalPrinterColumns: + - name: Image + type: string + description: Spilo image to be used for Pods + JSONPath: .configuration.docker_image + - name: Cluster-Label + type: string + description: Label for K8s resources created by operator + JSONPath: .configuration.kubernetes.cluster_name_label + - name: Service-Account + type: string + description: Name of service account to be used + JSONPath: .configuration.kubernetes.pod_service_account_name + - name: Min-Instances + type: integer + description: Minimum number of instances per Postgres cluster + JSONPath: .configuration.min_instances + - name: Age + type: date + JSONPath: .metadata.creationTimestamp + scope: Namespaced + subresources: + status: {} + version: v1 diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml new file mode 100644 index 000000000..c7505ed67 --- /dev/null +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -0,0 +1,53 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: postgresqls.acid.zalan.do + labels: + app.kubernetes.io/name: postgres-operator + annotations: + "helm.sh/hook": crd-install +spec: + group: acid.zalan.do + names: + kind: postgresql + listKind: postgresqlList + plural: postgresqls + singular: postgresql + shortNames: + - pg + additionalPrinterColumns: + - name: Team + type: string + description: Team responsible for Postgres CLuster + JSONPath: .spec.teamId + - name: Version + type: string + description: PostgreSQL version + JSONPath: .spec.postgresql.version + - name: Pods + type: integer + description: Number of Pods per Postgres cluster + JSONPath: .spec.numberOfInstances + - name: Volume + type: string + description: Size of the bound volume + JSONPath: .spec.volume.size + - name: CPU-Request + type: string + description: Requested CPU for Postgres containers + JSONPath: .spec.resources.requests.cpu + - name: Memory-Request + type: string + description: Requested memory for Postgres containers + JSONPath: .spec.resources.requests.memory + - name: Age + type: date + JSONPath: .metadata.creationTimestamp + - name: Status + type: string + description: Current sync status of postgresql resource + JSONPath: .status.PostgresClusterStatus + scope: Namespaced + subresources: + status: {} + version: v1 diff --git a/charts/postgres-operator/templates/crds.yaml b/charts/postgres-operator/templates/crds.yaml new file mode 100644 index 000000000..733830014 --- /dev/null +++ b/charts/postgres-operator/templates/crds.yaml @@ -0,0 +1,6 @@ +{{ if .Values.crd.create }} +{{- range $path, $bytes := .Files.Glob "crds/*.yaml" }} +{{ $.Files.Get $path }} +--- +{{- end }} +{{- end }} diff --git a/charts/postgres-operator/templates/customrresourcedefinition.yaml b/charts/postgres-operator/templates/customrresourcedefinition.yaml deleted file mode 100644 index 88ee1b614..000000000 --- a/charts/postgres-operator/templates/customrresourcedefinition.yaml +++ /dev/null @@ -1,103 +0,0 @@ -{{ if .Values.crd.create }} -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: postgresqls.acid.zalan.do - labels: - app.kubernetes.io/name: {{ template "postgres-operator.name" . }} - helm.sh/chart: {{ template "postgres-operator.chart" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} - annotations: - "helm.sh/hook": crd-install -spec: - group: acid.zalan.do - names: - kind: postgresql - listKind: postgresqlList - plural: postgresqls - singular: postgresql - shortNames: - - pg - additionalPrinterColumns: - - name: Team - type: string - description: Team responsible for Postgres CLuster - JSONPath: .spec.teamId - - name: Version - type: string - description: PostgreSQL version - JSONPath: .spec.postgresql.version - - name: Pods - type: integer - description: Number of Pods per Postgres cluster - JSONPath: .spec.numberOfInstances - - name: Volume - type: string - description: Size of the bound volume - JSONPath: .spec.volume.size - - name: CPU-Request - type: string - description: Requested CPU for Postgres containers - JSONPath: .spec.resources.requests.cpu - - name: Memory-Request - type: string - description: Requested memory for Postgres containers - JSONPath: .spec.resources.requests.memory - - name: Age - type: date - JSONPath: .metadata.creationTimestamp - - name: Status - type: string - description: Current sync status of postgresql resource - JSONPath: .status.PostgresClusterStatus - scope: Namespaced - subresources: - status: {} - version: v1 ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: operatorconfigurations.acid.zalan.do - labels: - app.kubernetes.io/name: {{ template "postgres-operator.name" . }} - helm.sh/chart: {{ template "postgres-operator.chart" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} - annotations: - "helm.sh/hook": crd-install -spec: - group: acid.zalan.do - names: - kind: OperatorConfiguration - listKind: OperatorConfigurationList - plural: operatorconfigurations - singular: operatorconfiguration - shortNames: - - opconfig - additionalPrinterColumns: - - name: Image - type: string - description: Spilo image to be used for Pods - JSONPath: .configuration.docker_image - - name: Cluster-Label - type: string - description: Label for K8s resources created by operator - JSONPath: .configuration.kubernetes.cluster_name_label - - name: Service-Account - type: string - description: Name of service account to be used - JSONPath: .configuration.kubernetes.pod_service_account_name - - name: Min-Instances - type: integer - description: Minimum number of instances per Postgres cluster - JSONPath: .configuration.min_instances - - name: Age - type: date - JSONPath: .metadata.creationTimestamp - scope: Namespaced - subresources: - status: {} - version: v1 -{{ end }} diff --git a/docs/quickstart.md b/docs/quickstart.md index 500d4db30..8fca8b62c 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -73,22 +73,22 @@ manifest. ### Helm chart Alternatively, the operator can be installed by using the provided [Helm](https://helm.sh/) -chart which saves you the manual steps. Therefore, install the helm CLI on your -machine. After initializing helm (and its server component Tiller) in your local -cluster you can install the operator chart. You can define a release name that -is prepended to the operator resource's names. - -Use `--name zalando` to match with the default service account name as older -operator versions do not support custom names for service accounts. To use -CRD-based configuration you need to specify the [values-crd yaml file](../charts/postgres-operator/values-crd.yaml). +chart which saves you the manual steps. Clone this repo and change directory to +the repo root. With Helm v3 installed you should be able to run: ```bash -# 1) initialize helm -helm init -# 2) install postgres-operator chart -helm install --name zalando ./charts/postgres-operator +helm install postgres-operator ./charts/postgres-operator ``` +To use CRD-based configuration you need to specify the [values-crd yaml file](../charts/postgres-operator/values-crd.yaml). + +```bash +helm install postgres-operator ./charts/postgres-operator -f ./charts/postgres-operator/values-crd.yaml +``` + +The chart works with both Helm 2 and Helm 3. Documentation for installing +applications with helm2 can be found in the [helm2 docs](https://v2.helm.sh/docs/). + ### Operator Lifecycle Manager (OLM) The [Operator Lifecycle Manager (OLM)](https://github.com/operator-framework/operator-lifecycle-manager) From a3b34f146f1be5cd0e6e71b8882a5cdd1be62639 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 28 Nov 2019 12:02:05 +0100 Subject: [PATCH 015/104] Add CRD validation (#599) * add CRD manifests with validation * update documentation * patroni slots is not an array but a nested hash map * make deps call tools * cover validation in docs and export it in crds.go * add toggle to disable creation of CRD validation and document it * use templated service account also for CRD-configured helm deployment --- Makefile | 2 +- .../crds/operatorconfigurations.yaml | 259 +++++ .../postgres-operator/crds/postgresqls.yaml | 310 ++++++ .../templates/operatorconfiguration.yaml | 2 +- charts/postgres-operator/values-crd.yaml | 23 +- charts/postgres-operator/values.yaml | 13 +- delivery.yaml | 2 +- docs/administrator.md | 82 +- docs/developer.md | 4 +- docs/quickstart.md | 27 +- docs/reference/operator_parameters.md | 18 +- manifests/complete-postgres-manifest.yaml | 52 +- manifests/configmap.yaml | 9 +- manifests/operatorconfiguration.crd.yaml | 276 ++++++ ...gresql-operator-default-configuration.yaml | 16 +- manifests/postgresql.crd.yaml | 327 +++++++ pkg/apis/acid.zalan.do/v1/crds.go | 922 +++++++++++++++++- .../v1/operator_configuration_type.go | 1 + .../acid.zalan.do/v1/zz_generated.deepcopy.go | 5 + pkg/controller/controller.go | 6 +- pkg/controller/operator_config.go | 1 + pkg/controller/util.go | 10 +- pkg/util/config/config.go | 9 +- run_operator_locally.sh | 2 +- 24 files changed, 2270 insertions(+), 108 deletions(-) create mode 100644 manifests/operatorconfiguration.crd.yaml create mode 100644 manifests/postgresql.crd.yaml diff --git a/Makefile b/Makefile index 8710f17ee..f42cfe09a 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ vet: @go vet $(PKG) @staticcheck $(PKG) -deps: +deps: tools GO111MODULE=on go mod vendor test: diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index acf132edd..ff92bc064 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -39,3 +39,262 @@ spec: subresources: status: {} version: v1 + validation: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - configuration + properties: + kind: + type: string + enum: + - OperatorConfiguration + apiVersion: + type: string + enum: + - acid.zalan.do/v1 + configuration: + type: object + properties: + docker_image: + type: string + enable_crd_validation: + type: boolean + enable_shm_volume: + type: boolean + etcd_host: + type: string + max_instances: + type: integer + minimum: -1 # -1 = disabled + min_instances: + type: integer + minimum: -1 # -1 = disabled + resync_period: + type: string + repair_period: + type: string + set_memory_request_to_limit: + type: boolean + sidecar_docker_images: + type: object + additionalProperties: + type: string + workers: + type: integer + minimum: 1 + users: + type: object + properties: + replication_username: + type: string + super_username: + type: string + kubernetes: + type: object + properties: + cluster_domain: + type: string + cluster_labels: + type: object + additionalProperties: + type: string + cluster_name_label: + type: string + custom_pod_annotations: + type: object + additionalProperties: + type: string + enable_pod_antiaffinity: + type: boolean + enable_pod_disruption_budget: + type: boolean + infrastructure_roles_secret_name: + type: string + inherited_labels: + type: array + items: + type: string + node_readiness_label: + type: object + additionalProperties: + type: string + oauth_token_secret_name: + type: string + pdb_name_format: + type: string + pod_antiaffinity_topology_key: + type: string + pod_environment_configmap: + type: string + pod_management_policy: + type: string + enum: + - "ordered_ready" + - "parallel" + pod_role_label: + type: string + pod_service_account_name: + type: string + pod_terminate_grace_period: + type: string + secret_name_template: + type: string + spilo_fsgroup: + type: integer + spilo_privileged: + type: boolean + toleration: + type: object + additionalProperties: + type: string + watched_namespace: + type: string + postgres_pod_resources: + type: object + properties: + default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + timeouts: + type: object + properties: + pod_label_wait_timeout: + type: string + pod_deletion_wait_timeout: + type: string + ready_wait_interval: + type: string + ready_wait_timeout: + type: string + resource_check_interval: + type: string + resource_check_timeout: + type: string + load_balancer: + type: object + properties: + db_hosted_zone: + type: string + enable_master_load_balancer: + type: boolean + enable_replica_load_balancer: + type: boolean + custom_service_annotations: + type: object + additionalProperties: + type: string + master_dns_name_format: + type: string + replica_dns_name_format: + type: string + aws_or_gcp: + type: object + properties: + additional_secret_mount: + type: string + additional_secret_mount_path: + type: string + aws_region: + type: string + kube_iam_role: + type: string + log_s3_bucket: + type: string + wal_s3_bucket: + type: string + logical_backup: + type: object + properties: + logical_backup_schedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + logical_backup_docker_image: + type: string + logical_backup_s3_bucket: + type: string + logical_backup_s3_endpoint: + type: string + logical_backup_s3_sse: + type: string + logical_backup_s3_access_key_id: + type: string + logical_backup_s3_secret_access_key: + type: string + debug: + type: object + properties: + debug_logging: + type: boolean + enable_database_access: + type: boolean + teams_api: + type: object + properties: + enable_admin_role_for_users: + type: boolean + enable_team_superuser: + type: boolean + enable_teams_api: + type: boolean + pam_configuration: + type: string + pam_role_name: + type: string + postgres_superuser_teams: + type: array + items: + type: string + protected_role_names: + type: array + items: + type: string + team_admin_role: + type: string + team_api_role_configuration: + type: object + additionalProperties: + type: string + teams_api_url: + type: string + logging_rest_api: + type: object + properties: + api_port: + type: integer + cluster_history_entries: + type: integer + ring_log_lines: + type: integer + scalyr: + type: object + properties: + scalyr_api_key: + type: string + scalyr_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + scalyr_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + scalyr_image: + type: string + scalyr_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + scalyr_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + scalyr_server_url: + type: string diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index c7505ed67..a8c5f2954 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -51,3 +51,313 @@ spec: subresources: status: {} version: v1 + validation: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - spec + properties: + kind: + type: string + enum: + - postgresql + apiVersion: + type: string + enum: + - acid.zalan.do/v1 + spec: + type: object + required: + - numberOfInstances + - teamId + - postgresql + properties: + allowedSourceRanges: + type: array + nullable: true + items: + type: string + pattern: '^(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])$' + clone: + type: object + required: + - cluster + properties: + cluster: + type: string + s3_endpoint: + type: string + s3_access_key_id: + type: string + s3_secret_access_key: + type: string + s3_force_path_style: + type: string + s3_wal_path: + type: string + timestamp: + type: string + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' + # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC + # Example: 1996-12-19T16:39:57-08:00 + # Note: this field requires a timezone + uid: + format: uuid + type: string + databases: + type: object + additionalProperties: + type: string + # Note: usernames specified here as database owners must be declared in the users key of the spec key. + dockerImage: + type: string + enableLogicalBackup: + type: boolean + enableMasterLoadBalancer: + type: boolean + enableReplicaLoadBalancer: + type: boolean + enableShmVolume: + type: boolean + init_containers: # deprecated + type: array + nullable: true + items: + type: object + additionalProperties: true + initContainers: + type: array + nullable: true + items: + type: object + additionalProperties: true + logicalBackupSchedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + maintenanceWindows: + type: array + items: + type: string + pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' + numberOfInstances: + type: integer + minimum: 0 + patroni: + type: object + properties: + initdb: + type: object + additionalProperties: + type: string + pg_hba: + type: array + items: + type: string + slots: + type: object + additionalProperties: + type: object + additionalProperties: + type: string + ttl: + type: integer + loop_wait: + type: integer + retry_timeout: + type: integer + maximum_lag_on_failover: + type: integer + podAnnotations: + type: object + additionalProperties: + type: string + pod_priority_class_name: # deprecated + type: string + podPriorityClassName: + type: string + postgresql: + type: object + required: + - version + properties: + version: + type: string + enum: + - "9.3" + - "9.4" + - "9.5" + - "9.6" + - "10" + - "11" + - "12" + parameters: + type: object + additionalProperties: + type: string + replicaLoadBalancer: # deprecated + type: boolean + resources: + type: object + required: + - requests + - limits + properties: + limits: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + # Decimal natural followed by m, or decimal natural followed by + # dot followed by up to three decimal digits. + # + # This is because the Kubernetes CPU resource has millis as the + # maximum precision. The actual values are checked in code + # because the regular expression would be huge and horrible and + # not very helpful in validation error messages; this one checks + # only the format of the given number. + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + pattern: '^(\d+m|\d+\.\d{1,3})$' + # Note: the value specified here must not be zero or be lower + # than the corresponding request. + memory: + type: string + # You can express memory as a plain integer or as a fixed-point + # integer using one of these suffixes: E, P, T, G, M, k. You can + # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero or be lower + # than the corresponding request. + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + # Decimal natural followed by m, or decimal natural followed by + # dot followed by up to three decimal digits. + # + # This is because the Kubernetes CPU resource has millis as the + # maximum precision. The actual values are checked in code + # because the regular expression would be huge and horrible and + # not very helpful in validation error messages; this one checks + # only the format of the given number. + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + pattern: '^(\d+m|\d+\.\d{1,3})$' + # Note: the value specified here must not be zero or be higher + # than the corresponding limit. + memory: + type: string + # You can express memory as a plain integer or as a fixed-point + # integer using one of these suffixes: E, P, T, G, M, k. You can + # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero or be higher + # than the corresponding limit. + sidecars: + type: array + nullable: true + items: + type: object + additionalProperties: true + spiloFSGroup: + type: integer + standby: + type: object + required: + - s3_wal_path + properties: + s3_wal_path: + type: string + teamId: + type: string + tolerations: + type: array + items: + type: object + required: + - key + - operator + - effect + properties: + key: + type: string + operator: + type: string + enum: + - Equal + - Exists + value: + type: string + effect: + type: string + enum: + - NoExecute + - NoSchedule + - PreferNoSchedule + tolerationSeconds: + type: integer + useLoadBalancer: # deprecated + type: boolean + users: + type: object + additionalProperties: + type: array + nullable: true + description: "Role flags specified here must not contradict each other" + items: + type: string + enum: + - bypassrls + - BYPASSRLS + - nobypassrls + - NOBYPASSRLS + - createdb + - CREATEDB + - nocreatedb + - NOCREATEDB + - createrole + - CREATEROLE + - nocreaterole + - NOCREATEROLE + - inherit + - INHERIT + - noinherit + - NOINHERIT + - login + - LOGIN + - nologin + - NOLOGIN + - replication + - REPLICATION + - noreplication + - NOREPLICATION + - superuser + - SUPERUSER + - nosuperuser + - NOSUPERUSER + volume: + type: object + required: + - size + properties: + size: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero. + storageClass: + type: string + subPath: + type: string diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index c6e9f78b7..6a301c1fb 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -14,7 +14,7 @@ configuration: {{ toYaml .Values.configUsers | indent 4 }} kubernetes: oauth_token_secret_name: {{ template "postgres-operator.fullname" . }} - pod_service_account_name: operator + pod_service_account_name: {{ include "postgres-operator.serviceAccountName" . }} {{ toYaml .Values.configKubernetes | indent 4 }} postgres_pod_resources: {{ toYaml .Values.configPostgresPodResources | indent 4 }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index af05ae56c..c6f11e493 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -17,6 +17,8 @@ configTarget: "OperatorConfigurationCRD" # general top-level configuration parameters configGeneral: + # choose if deployment creates/updates CRDs with OpenAPIV3Validation + enable_crd_validation: true # start any new database pod without limitations on shm memory enable_shm_volume: true # etcd connection string for Patroni. Empty uses K8s-native DCS. @@ -57,7 +59,10 @@ configKubernetes: # label assigned to Kubernetes objects created by the operator cluster_name_label: cluster-name # additional annotations to add to every database pod - custom_pod_annotations: + # custom_pod_annotations: + # keya: valuea + # keyb: valueb + # toggles pod anti affinity on the Postgres pods enable_pod_antiaffinity: false # toggles PDB to set to MinAvailabe 0 or 1 @@ -74,7 +79,8 @@ configKubernetes: # master_pod_move_timeout: 20m # set of labels that a running and active node should possess to be considered ready - # node_readiness_label: "" + # node_readiness_label: + # status: ready # name of the secret containing the OAuth2 token to pass to the teams API # oauth_token_secret_name: postgresql-operator @@ -186,7 +192,7 @@ configAwsOrGcp: configLogicalBackup: # image for pods of the logical backup job (example runs pg_dumpall) logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" - # S3 Access Key ID + # S3 Access Key ID logical_backup_s3_access_key_id: "" # S3 bucket to store backup results logical_backup_s3_bucket: "my-bucket-url" @@ -194,7 +200,7 @@ configLogicalBackup: logical_backup_s3_endpoint: "" # S3 Secret Access Key logical_backup_s3_secret_access_key: "" - # S3 server side encription + # S3 server side encription logical_backup_s3_sse: "AES256" # backup schedule in the cron format logical_backup_schedule: "30 00 * * *" @@ -214,7 +220,8 @@ configTeamsApi: # operator will add all team member roles to this group and add a pg_hba line pam_role_name: zalandos # List of teams which members need the superuser role in each Postgres cluster - # postgres_superuser_teams: "postgres_superusers" + # postgres_superuser_teams: + # - postgres_superusers # List of roles that cannot be overwritten by an application, team or infrastructure role protected_role_names: @@ -228,7 +235,7 @@ configTeamsApi: # teams_api_url: http://fake-teams-api.default.svc.cluster.local # Scalyr is a log management tool that Zalando uses as a sidecar -scalyr: +configScalyr: # API key for the Scalyr sidecar # scalyr_api_key: "" @@ -257,9 +264,7 @@ serviceAccount: create: true # The name of the ServiceAccount to use. # If not set and create is true, a name is generated using the fullname template - # When relying solely on the OperatorConfiguration CRD, this value has to be "operator" - # Otherwise, the operator tries to use the "default" service account which is forbidden - name: operator + name: priorityClassName: "" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index b572b5844..ae9e06fe9 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -17,6 +17,8 @@ configTarget: "ConfigMap" # general configuration parameters configGeneral: + # choose if deployment creates/updates CRDs with OpenAPIV3Validation + enable_crd_validation: "true" # start any new database pod without limitations on shm memory enable_shm_volume: "true" # etcd connection string for Patroni. Empty uses K8s-native DCS. @@ -55,7 +57,8 @@ configKubernetes: # label assigned to Kubernetes objects created by the operator cluster_name_label: version # annotations attached to each database pod - # custom_pod_annotations: keya:valuea + # custom_pod_annotations: keya:valuea,keyb:valueb + # toggles pod anti affinity on the Postgres pods enable_pod_antiaffinity: "false" # toggles PDB to set to MinAvailabe 0 or 1 @@ -136,9 +139,9 @@ configLoadBalancer: # toggles service type load balancer pointing to the replica pod of the cluster enable_replica_load_balancer: "false" # defines the DNS name string template for the master load balancer cluster - master_dns_name_format: '{cluster}.{team}.staging.{hostedzone}' + master_dns_name_format: '{cluster}.{team}.{hostedzone}' # defines the DNS name string template for the replica load balancer cluster - replica_dns_name_format: '{cluster}-repl.{team}.staging.{hostedzone}' + replica_dns_name_format: '{cluster}-repl.{team}.{hostedzone}' # options to aid debugging of the operator itself configDebug: @@ -180,7 +183,7 @@ configAwsOrGcp: configLogicalBackup: # image for pods of the logical backup job (example runs pg_dumpall) logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" - # S3 Access Key ID + # S3 Access Key ID logical_backup_s3_access_key_id: "" # S3 bucket to store backup results logical_backup_s3_bucket: "my-bucket-url" @@ -188,7 +191,7 @@ configLogicalBackup: logical_backup_s3_endpoint: "" # S3 Secret Access Key logical_backup_s3_secret_access_key: "" - # S3 server side encription + # S3 server side encription logical_backup_s3_sse: "AES256" # backup schedule in the cron format logical_backup_schedule: "30 00 * * *" diff --git a/delivery.yaml b/delivery.yaml index ef249292c..be35d3e27 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -28,7 +28,7 @@ pipeline: IMAGE=registry-write.opensource.zalan.do/acid/postgres-operator-test fi export IMAGE - make tools deps docker + make deps docker - desc: 'Run unit tests' cmd: | export PATH=$PATH:$HOME/go/bin diff --git a/docs/administrator.md b/docs/administrator.md index ab5368e7d..5cde06ade 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -3,6 +3,30 @@ Learn how to configure and manage the Postgres Operator in your Kubernetes (K8s) environment. +## CRD Validation + +[CustomResourceDefinitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions) +will be registered with schema validation by default when the operator is +deployed. The `OperatorConfiguration` CRD will only get created if the +`POSTGRES_OPERATOR_CONFIGURATION_OBJECT` [environment variable](../manifests/postgres-operator.yaml#L36) +in the deployment yaml is set and not empty. + +When submitting manifests of [`postgresql`](../manifests/postgresql.crd.yaml) or +[`OperatorConfiguration`](../manifests/operatorconfiguration.crd.yaml) custom +resources with kubectl, validation can be bypassed with `--validate=false`. The +operator can also be configured to not register CRDs with validation on `ADD` or +`UPDATE` events. Running instances are not affected when enabling the validation +afterwards unless the manifests is not changed then. Note, that the provided CRD +manifests contain the validation for users to understand what schema is +enforced. + +Once the validation is enabled it can only be disabled manually by editing or +patching the CRD manifest: + +```bash +zk8 patch crd postgresqls.acid.zalan.do -p '{"spec":{"validation": null}}' +``` + ## Namespaces ### Select the namespace to deploy to @@ -32,7 +56,7 @@ By default, the operator watches the namespace it is deployed to. You can change this by setting the `WATCHED_NAMESPACE` var in the `env` section of the [operator deployment](../manifests/postgres-operator.yaml) manifest or by altering the `watched_namespace` field in the operator -[ConfigMap](../manifests/configmap.yaml#L79). +[configuration](../manifests/postgresql-operator-default-configuration.yaml#L49). In the case both are set, the env var takes the precedence. To make the operator listen to all namespaces, explicitly set the field/env var to "`*`". @@ -115,7 +139,7 @@ that are aggregated into the K8s [default roles](https://kubernetes.io/docs/refe To ensure Postgres pods are running on nodes without any other application pods, you can use [taints and tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) -and configure the required toleration in the operator ConfigMap. +and configure the required toleration in the operator configuration. As an example you can set following node taint: @@ -136,6 +160,21 @@ data: ... ``` +For an OperatorConfiguration resource the toleration should be defined like +this: + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration +metadata: + name: postgresql-configuration +configuration: + kubernetes: + toleration: + postgres: "key:postgres,operator:Exists,effect:NoSchedule" + ... +``` + Note that the K8s version 1.13 brings [taint-based eviction](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#taint-based-evictions) to the beta stage and enables it by default. Postgres pods by default receive tolerations for `unreachable` and `noExecute` taints with the timeout of `5m`. @@ -148,7 +187,7 @@ completely, specify the toleration by leaving out the `tolerationSeconds` value To ensure Postgres pods are running on different topologies, you can use [pod anti affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) -and configure the required topology in the operator ConfigMap. +and configure the required topology in the operator configuration. Enable pod anti affinity by adding following line to the operator ConfigMap: @@ -161,21 +200,22 @@ data: enable_pod_antiaffinity: "true" ``` -By default the topology key for the pod anti affinity is set to -`kubernetes.io/hostname`, you can set another topology key e.g. -`failure-domain.beta.kubernetes.io/zone` by adding following line to the -operator ConfigMap, see [built-in node labels](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#interlude-built-in-node-labels) for available topology keys: +Likewise, when using an OperatorConfiguration resource add: ```yaml -apiVersion: v1 -kind: ConfigMap +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration metadata: - name: postgres-operator -data: - enable_pod_antiaffinity: "true" - pod_antiaffinity_topology_key: "failure-domain.beta.kubernetes.io/zone" + name: postgresql-configuration +configuration: + kubernetes: + enable_pod_antiaffinity: true ``` +By default the topology key for the pod anti affinity is set to +`kubernetes.io/hostname`, you can set another topology key e.g. +`failure-domain.beta.kubernetes.io/zone`. See [built-in node labels](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#interlude-built-in-node-labels) for available topology keys. + ## Pod Disruption Budget By default the operator uses a PodDisruptionBudget (PDB) to protect the cluster @@ -280,6 +320,20 @@ data: ... ``` +**OperatorConfiguration** + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration +metadata: + name: postgresql-operator-configuration +configuration: + kubernetes: + # referencing config map with custom settings + pod_environment_configmap: postgres-pod-config + ... +``` + **referenced ConfigMap `postgres-pod-config`** ```yaml @@ -312,7 +366,7 @@ services: one for the master pod and one for replica pods. To expose these services to an outer network, one can attach load balancers to them by setting `enableMasterLoadBalancer` and/or `enableReplicaLoadBalancer` to `true` in the cluster manifest. In the case any of these variables are omitted from the -manifest, the operator configmap's settings `enable_master_load_balancer` and +manifest, the operator configuration settings `enable_master_load_balancer` and `enable_replica_load_balancer` apply. Note that the operator settings affect all Postgresql services running in all namespaces watched by the operator. diff --git a/docs/developer.md b/docs/developer.md index 490217362..f8351e28a 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -33,7 +33,7 @@ by setting the `GO111MODULE` environment variable to `on`. The make targets do this for you, so simply run ```bash -make tools deps +make deps ``` This would take a while to complete. You have to redo `make deps` every time @@ -284,6 +284,7 @@ manifest files: Postgres manifest parameters are defined in the [api package](../pkg/apis/acid.zalan.do/v1/postgresql_type.go). The operator behavior has to be implemented at least in [k8sres.go](../pkg/cluster/k8sres.go). +Validation of CRD parameters is controlled in [crd.go](../pkg/apis/acid.zalan.do/v1/crds.go). Please, reflect your changes in tests, for example in: * [config_test.go](../pkg/util/config/config_test.go) * [k8sres_test.go](../pkg/cluster/k8sres_test.go) @@ -294,6 +295,7 @@ Please, reflect your changes in tests, for example in: For the CRD-based configuration, please update the following files: * the default [OperatorConfiguration](../manifests/postgresql-operator-default-configuration.yaml) * the Helm chart's [values-crd file](../charts/postgres-operator/values.yaml) +* the CRD's [validation](../manifests/operatorconfiguration.crd.yaml) Reflect the changes in the ConfigMap configuration as well (note that numeric and boolean parameters have to use double quotes here): diff --git a/docs/quickstart.md b/docs/quickstart.md index 8fca8b62c..2da2cab7c 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -55,8 +55,8 @@ kubectl create -f manifests/postgres-operator.yaml # deployment ``` There is a [Kustomization](https://github.com/kubernetes-sigs/kustomize) -manifest that [combines the mentioned resources](../manifests/kustomization.yaml) - -it can be used with kubectl 1.14 or newer as easy as: +manifest that [combines the mentioned resources](../manifests/kustomization.yaml) +(except for the CRD) - it can be used with kubectl 1.14 or newer as easy as: ```bash kubectl apply -k github.com/zalando/postgres-operator/manifests @@ -86,8 +86,9 @@ To use CRD-based configuration you need to specify the [values-crd yaml file](.. helm install postgres-operator ./charts/postgres-operator -f ./charts/postgres-operator/values-crd.yaml ``` -The chart works with both Helm 2 and Helm 3. Documentation for installing -applications with helm2 can be found in the [helm2 docs](https://v2.helm.sh/docs/). +The chart works with both Helm 2 and Helm 3. The `crd-install` hook from v2 will +be skipped with warning when using v3. Documentation for installing applications +with Helm 2 can be found in the [v2 docs](https://v2.helm.sh/docs/). ### Operator Lifecycle Manager (OLM) @@ -119,15 +120,15 @@ kubectl get pod -l app.kubernetes.io/name=postgres-operator kubectl create -f manifests/minimal-postgres-manifest.yaml ``` -After the cluster manifest is submitted the operator will create Service and -Endpoint resources and a StatefulSet which spins up new Pod(s) given the number -of instances specified in the manifest. All resources are named like the -cluster. The database pods can be identified by their number suffix, starting -from `-0`. They run the [Spilo](https://github.com/zalando/spilo) container -image by Zalando. As for the services and endpoints, there will be one for the -master pod and another one for all the replicas (`-repl` suffix). Check if all -components are coming up. Use the label `application=spilo` to filter and list -the label `spilo-role` to see who is currently the master. +After the cluster manifest is submitted and passed the validation the operator +will create Service and Endpoint resources and a StatefulSet which spins up new +Pod(s) given the number of instances specified in the manifest. All resources +are named like the cluster. The database pods can be identified by their number +suffix, starting from `-0`. They run the [Spilo](https://github.com/zalando/spilo) +container image by Zalando. As for the services and endpoints, there will be one +for the master pod and another one for all the replicas (`-repl` suffix). Check +if all components are coming up. Use the label `application=spilo` to filter and +list the label `spilo-role` to see who is currently the master. ```bash # check the deployed cluster diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index a024aad1f..e16282fd2 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -29,21 +29,20 @@ configuration. To test the CRD-based configuration locally, use the following ```bash + kubectl create -f manifests/operatorconfiguration.crd.yaml # registers the CRD + kubectl create -f manifests/postgresql-operator-default-configuration.yaml + kubectl create -f manifests/operator-service-account-rbac.yaml kubectl create -f manifests/postgres-operator.yaml # set the env var as mentioned above - kubectl create -f manifests/postgresql-operator-default-configuration.yaml + kubectl get operatorconfigurations postgresql-operator-default-configuration -o yaml ``` - Note that the operator first attempts to register the CRD of the - `OperatorConfiguration` and then waits for an instance to be created. In - between these two event the operator pod may be failing since it cannot fetch - the not-yet-existing `OperatorConfiguration` instance. The CRD-based configuration is more powerful than the one based on ConfigMaps and should be used unless there is a compatibility requirement to use an already existing configuration. Even in that case, it should be rather straightforward -to convert the configmap based configuration into the CRD-based one and restart -the operator. The ConfigMaps-based configuration will be deprecated and +to convert the ConfigMap-based configuration into the CRD-based one and restart +the operator. The ConfigMap-based configuration will be deprecated and subsequently removed in future releases. Note that for the CRD-based configuration groups of configuration options below @@ -71,6 +70,11 @@ Variable names are underscore-separated words. Those are top-level keys, containing both leaf keys and groups. +* **enable_crd_validation** + toggles if the operator will create or update CRDs with + [OpenAPI v3 schema validation](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#validation) + The default is `true`. + * **etcd_host** Etcd connection string for Patroni defined as `host:port`. Not required when Patroni native Kubernetes support is used. The default is empty (use diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 34dd6cf83..f8495caad 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -13,7 +13,7 @@ spec: teamId: "ACID" volume: size: 1Gi -# storageClass: my-sc +# storageClass: my-sc numberOfInstances: 2 users: # Application/Robot users zalando: @@ -30,7 +30,7 @@ spec: # Expert section enableShmVolume: true -# spiloFSGroup: 103 +# spiloFSGroup: 103 postgresql: version: "11" parameters: @@ -52,13 +52,13 @@ spec: pg_hba: - hostssl all all 0.0.0.0/0 md5 - host all all 0.0.0.0/0 md5 -# slots: -# - permanent_physical_1: -# type: physical -# - permanent_logical_1: -# type: logical -# database: foo -# plugin: pgoutput +# slots: +# permanent_physical_1: +# type: physical +# permanent_logical_1: +# type: logical +# database: foo +# plugin: pgoutput ttl: 30 loop_wait: &loop_wait 10 retry_timeout: 10 @@ -66,28 +66,28 @@ spec: # restore a Postgres DB with point-in-time-recovery # with a non-empty timestamp, clone from an S3 bucket using the latest backup before the timestamp # with an empty/absent timestamp, clone from an existing alive cluster using pg_basebackup -# clone: -# uid: "efd12e58-5786-11e8-b5a7-06148230260c" -# cluster: "acid-batman" -# timestamp: "2017-12-19T12:40:33+01:00" # timezone required (offset relative to UTC, see RFC 3339 section 5.6) -# s3_wal_path: "s3://custom/path/to/bucket" +# clone: +# uid: "efd12e58-5786-11e8-b5a7-06148230260c" +# cluster: "acid-batman" +# timestamp: "2017-12-19T12:40:33+01:00" # timezone required (offset relative to UTC, see RFC 3339 section 5.6) +# s3_wal_path: "s3://custom/path/to/bucket" # run periodic backups with k8s cron jobs -# enableLogicalBackup: true -# logicalBackupSchedule: "30 00 * * *" +# enableLogicalBackup: true +# logicalBackupSchedule: "30 00 * * *" maintenanceWindows: - 01:00-06:00 #UTC - Sat:00:00-04:00 -# sidecars: -# - name: "telegraf-sidecar" -# image: "telegraf:latest" -# resources: -# limits: -# cpu: 500m -# memory: 500Mi -# requests: -# cpu: 100m -# memory: 100Mi +# sidecars: +# - name: "telegraf-sidecar" +# image: "telegraf:latest" +# resources: +# limits: +# cpu: 500m +# memory: 500Mi +# requests: +# cpu: 100m +# memory: 100Mi # env: # - name: "USEFUL_VAR" # value: "perhaps-true" diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 40aca9716..59d3abfde 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -12,7 +12,7 @@ data: cluster_labels: application:spilo cluster_name_label: version # custom_service_annotations: "keyx:valuez,keya:valuea" - # custom_pod_annotations: "keya:valuea" + # custom_pod_annotations: "keya:valuea,keyb:valueb" db_hosted_zone: db.example.com debug_logging: "true" # default_cpu_limit: "3" @@ -21,6 +21,7 @@ data: # default_memory_request: 100Mi docker_image: registry.opensource.zalan.do/acid/spilo-11:1.6-p1 # enable_admin_role_for_users: "true" + # enable_crd_validation: "true" # enable_database_access: "true" enable_master_load_balancer: "false" # enable_pod_antiaffinity: "false" @@ -41,7 +42,7 @@ data: # logical_backup_s3_secret_access_key: "" # logical_backup_s3_sse: "AES256" # logical_backup_schedule: "30 00 * * *" - master_dns_name_format: "{cluster}.{team}.staging.{hostedzone}" + master_dns_name_format: "{cluster}.{team}.{hostedzone}" # master_pod_move_timeout: 10m # max_instances: "-1" # min_instances: "-1" @@ -64,11 +65,11 @@ data: ready_wait_interval: 3s ready_wait_timeout: 30s repair_period: 5m - replica_dns_name_format: "{cluster}-repl.{team}.staging.{hostedzone}" + replica_dns_name_format: "{cluster}-repl.{team}.{hostedzone}" replication_username: standby resource_check_interval: 3s resource_check_timeout: 10m - resync_period: 5m + resync_period: 30m ring_log_lines: "100" secret_name_template: "{username}.{cluster}.credentials" # sidecar_docker_images: "" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml new file mode 100644 index 000000000..753415a15 --- /dev/null +++ b/manifests/operatorconfiguration.crd.yaml @@ -0,0 +1,276 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: operatorconfigurations.acid.zalan.do +spec: + group: acid.zalan.do + names: + kind: OperatorConfiguration + listKind: OperatorConfigurationList + plural: operatorconfigurations + singular: operatorconfiguration + shortNames: + - opconfig + scope: Namespaced + subresources: + status: {} + version: v1 + validation: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - configuration + properties: + kind: + type: string + enum: + - OperatorConfiguration + apiVersion: + type: string + enum: + - acid.zalan.do/v1 + configuration: + type: object + properties: + docker_image: + type: string + enable_crd_validation: + type: boolean + enable_shm_volume: + type: boolean + etcd_host: + type: string + max_instances: + type: integer + minimum: -1 # -1 = disabled + min_instances: + type: integer + minimum: -1 # -1 = disabled + resync_period: + type: string + repair_period: + type: string + set_memory_request_to_limit: + type: boolean + sidecar_docker_images: + type: object + additionalProperties: + type: string + workers: + type: integer + minimum: 1 + users: + type: object + properties: + replication_username: + type: string + super_username: + type: string + kubernetes: + type: object + properties: + cluster_domain: + type: string + cluster_labels: + type: object + additionalProperties: + type: string + cluster_name_label: + type: string + custom_pod_annotations: + type: object + additionalProperties: + type: string + enable_pod_antiaffinity: + type: boolean + enable_pod_disruption_budget: + type: boolean + infrastructure_roles_secret_name: + type: string + inherited_labels: + type: array + items: + type: string + node_readiness_label: + type: object + additionalProperties: + type: string + oauth_token_secret_name: + type: string + pdb_name_format: + type: string + pod_antiaffinity_topology_key: + type: string + pod_environment_configmap: + type: string + pod_management_policy: + type: string + enum: + - "ordered_ready" + - "parallel" + pod_role_label: + type: string + pod_service_account_name: + type: string + pod_terminate_grace_period: + type: string + secret_name_template: + type: string + spilo_fsgroup: + type: integer + spilo_privileged: + type: boolean + toleration: + type: object + additionalProperties: + type: string + watched_namespace: + type: string + postgres_pod_resources: + type: object + properties: + default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + timeouts: + type: object + properties: + pod_label_wait_timeout: + type: string + pod_deletion_wait_timeout: + type: string + ready_wait_interval: + type: string + ready_wait_timeout: + type: string + resource_check_interval: + type: string + resource_check_timeout: + type: string + load_balancer: + type: object + properties: + db_hosted_zone: + type: string + enable_master_load_balancer: + type: boolean + enable_replica_load_balancer: + type: boolean + custom_service_annotations: + type: object + additionalProperties: + type: string + master_dns_name_format: + type: string + replica_dns_name_format: + type: string + aws_or_gcp: + type: object + properties: + additional_secret_mount: + type: string + additional_secret_mount_path: + type: string + aws_region: + type: string + kube_iam_role: + type: string + log_s3_bucket: + type: string + wal_s3_bucket: + type: string + logical_backup: + type: object + properties: + logical_backup_schedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + logical_backup_docker_image: + type: string + logical_backup_s3_bucket: + type: string + logical_backup_s3_endpoint: + type: string + logical_backup_s3_sse: + type: string + logical_backup_s3_access_key_id: + type: string + logical_backup_s3_secret_access_key: + type: string + debug: + type: object + properties: + debug_logging: + type: boolean + enable_database_access: + type: boolean + teams_api: + type: object + properties: + enable_admin_role_for_users: + type: boolean + enable_team_superuser: + type: boolean + enable_teams_api: + type: boolean + pam_configuration: + type: string + pam_role_name: + type: string + postgres_superuser_teams: + type: array + items: + type: string + protected_role_names: + type: array + items: + type: string + team_admin_role: + type: string + team_api_role_configuration: + type: object + additionalProperties: + type: string + teams_api_url: + type: string + logging_rest_api: + type: object + properties: + api_port: + type: integer + cluster_history_entries: + type: integer + ring_log_lines: + type: integer + scalyr: + type: object + properties: + scalyr_api_key: + type: string + scalyr_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + scalyr_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + scalyr_image: + type: string + scalyr_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + scalyr_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + scalyr_server_url: + type: string diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 94f91f1f0..7a867d5df 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -3,6 +3,7 @@ kind: OperatorConfiguration metadata: name: postgresql-operator-default-configuration configuration: + # enable_crd_validation: true etcd_host: "" docker_image: registry.opensource.zalan.do/acid/spilo-11:1.6-p1 # enable_shm_volume: true @@ -27,24 +28,25 @@ configuration: # keyb: valueb enable_pod_antiaffinity: false enable_pod_disruption_budget: true - # infrastructure_roles_secret_name: "" + # infrastructure_roles_secret_name: postgresql-infrastructure-roles # inherited_labels: # - application # - environment - # node_readiness_label: "" + # node_readiness_label: + # status: ready oauth_token_secret_name: postgresql-operator pdb_name_format: "postgres-{cluster}-pdb" pod_antiaffinity_topology_key: "kubernetes.io/hostname" # pod_environment_configmap: "" pod_management_policy: "ordered_ready" pod_role_label: spilo-role - pod_service_account_name: operator + pod_service_account_name: zalando-postgres-operator pod_terminate_grace_period: 5m secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" # spilo_fsgroup: 103 spilo_privileged: false # toleration: {} - # watched_namespace:"" + # watched_namespace: "" postgres_pod_resources: default_cpu_limit: "3" default_cpu_request: 100m @@ -74,7 +76,6 @@ configuration: # log_s3_bucket: "" # wal_s3_bucket: "" logical_backup: - log_s3_bucket: "" logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" logical_backup_s3_access_key_id: "" logical_backup_s3_bucket: "my-bucket-url" @@ -91,9 +92,10 @@ configuration: enable_teams_api: false # pam_configuration: "" pam_role_name: zalandos - # postgres_superuser_teams: "postgres_superusers" + # postgres_superuser_teams: + # - postgres_superusers protected_role_names: - - admin + - admin team_admin_role: admin team_api_role_configuration: log_statement: all diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml new file mode 100644 index 000000000..4a578b324 --- /dev/null +++ b/manifests/postgresql.crd.yaml @@ -0,0 +1,327 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: postgresqls.acid.zalan.do +spec: + group: acid.zalan.do + names: + kind: postgresql + listKind: postgresqlList + plural: postgresqls + singular: postgresql + shortNames: + - pg + scope: Namespaced + subresources: + status: {} + version: v1 + validation: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - spec + properties: + kind: + type: string + enum: + - postgresql + apiVersion: + type: string + enum: + - acid.zalan.do/v1 + spec: + type: object + required: + - numberOfInstances + - teamId + - postgresql + properties: + allowedSourceRanges: + type: array + nullable: true + items: + type: string + pattern: '^(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])$' + clone: + type: object + required: + - cluster + properties: + cluster: + type: string + s3_endpoint: + type: string + s3_access_key_id: + type: string + s3_secret_access_key: + type: string + s3_force_path_style: + type: string + s3_wal_path: + type: string + timestamp: + type: string + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' + # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC + # Example: 1996-12-19T16:39:57-08:00 + # Note: this field requires a timezone + uid: + format: uuid + type: string + databases: + type: object + additionalProperties: + type: string + # Note: usernames specified here as database owners must be declared in the users key of the spec key. + dockerImage: + type: string + enableLogicalBackup: + type: boolean + enableMasterLoadBalancer: + type: boolean + enableReplicaLoadBalancer: + type: boolean + enableShmVolume: + type: boolean + init_containers: # deprecated + type: array + nullable: true + items: + type: object + additionalProperties: true + initContainers: + type: array + nullable: true + items: + type: object + additionalProperties: true + logicalBackupSchedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + maintenanceWindows: + type: array + items: + type: string + pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' + numberOfInstances: + type: integer + minimum: 0 + patroni: + type: object + properties: + initdb: + type: object + additionalProperties: + type: string + pg_hba: + type: array + items: + type: string + slots: + type: object + additionalProperties: + type: object + additionalProperties: + type: string + ttl: + type: integer + loop_wait: + type: integer + retry_timeout: + type: integer + maximum_lag_on_failover: + type: integer + podAnnotations: + type: object + additionalProperties: + type: string + pod_priority_class_name: # deprecated + type: string + podPriorityClassName: + type: string + postgresql: + type: object + required: + - version + properties: + version: + type: string + enum: + - "9.3" + - "9.4" + - "9.5" + - "9.6" + - "10" + - "11" + - "12" + parameters: + type: object + additionalProperties: + type: string + replicaLoadBalancer: # deprecated + type: boolean + resources: + type: object + required: + - requests + - limits + properties: + limits: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + # Decimal natural followed by m, or decimal natural followed by + # dot followed by up to three decimal digits. + # + # This is because the Kubernetes CPU resource has millis as the + # maximum precision. The actual values are checked in code + # because the regular expression would be huge and horrible and + # not very helpful in validation error messages; this one checks + # only the format of the given number. + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + pattern: '^(\d+m|\d+\.\d{1,3})$' + # Note: the value specified here must not be zero or be lower + # than the corresponding request. + memory: + type: string + # You can express memory as a plain integer or as a fixed-point + # integer using one of these suffixes: E, P, T, G, M, k. You can + # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero or be lower + # than the corresponding request. + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + # Decimal natural followed by m, or decimal natural followed by + # dot followed by up to three decimal digits. + # + # This is because the Kubernetes CPU resource has millis as the + # maximum precision. The actual values are checked in code + # because the regular expression would be huge and horrible and + # not very helpful in validation error messages; this one checks + # only the format of the given number. + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + pattern: '^(\d+m|\d+\.\d{1,3})$' + # Note: the value specified here must not be zero or be higher + # than the corresponding limit. + memory: + type: string + # You can express memory as a plain integer or as a fixed-point + # integer using one of these suffixes: E, P, T, G, M, k. You can + # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + # + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero or be higher + # than the corresponding limit. + sidecars: + type: array + nullable: true + items: + type: object + additionalProperties: true + spiloFSGroup: + type: integer + standby: + type: object + required: + - s3_wal_path + properties: + s3_wal_path: + type: string + teamId: + type: string + tolerations: + type: array + items: + type: object + required: + - key + - operator + - effect + properties: + key: + type: string + operator: + type: string + enum: + - Equal + - Exists + value: + type: string + effect: + type: string + enum: + - NoExecute + - NoSchedule + - PreferNoSchedule + tolerationSeconds: + type: integer + useLoadBalancer: # deprecated + type: boolean + users: + type: object + additionalProperties: + type: array + nullable: true + description: "Role flags specified here must not contradict each other" + items: + type: string + enum: + - bypassrls + - BYPASSRLS + - nobypassrls + - NOBYPASSRLS + - createdb + - CREATEDB + - nocreatedb + - NOCREATEDB + - createrole + - CREATEROLE + - nocreaterole + - NOCREATEROLE + - inherit + - INHERIT + - noinherit + - NOINHERIT + - login + - LOGIN + - nologin + - NOLOGIN + - replication + - REPLICATION + - noreplication + - NOREPLICATION + - superuser + - SUPERUSER + - nosuperuser + - NOSUPERUSER + volume: + type: object + required: + - size + properties: + size: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + # Note: the value specified here must not be zero. + storageClass: + type: string + subPath: + type: string diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 50833db26..9ee76103f 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1,7 +1,7 @@ package v1 import ( - "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do" + acidzalando "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -103,7 +103,902 @@ var OperatorConfigCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefin }, } -func buildCRD(name, kind, plural, short string, columns []apiextv1beta1.CustomResourceColumnDefinition) *apiextv1beta1.CustomResourceDefinition { +var min0 = 0.0 +var min1 = 1.0 +var minDisable = -1.0 + +// PostgresCRDResourceValidation to check applied manifest parameters +var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + Required: []string{"kind", "apiVersion", "spec"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "kind": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"postgresql"`), + }, + }, + }, + "apiVersion": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"acid.zalan.do/v1"`), + }, + }, + }, + "spec": { + Type: "object", + Required: []string{"numberOfInstances", "teamId", "postgresql"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "allowedSourceRanges": { + Type: "array", + Nullable: true, + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + Pattern: "^(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\/(\\d|[1-2]\\d|3[0-2])$", + }, + }, + }, + "clone": { + Type: "object", + Required: []string{"cluster"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "cluster": { + Type: "string", + }, + "s3_endpoint": { + Type: "string", + }, + "s3_access_key_id": { + Type: "string", + }, + "s3_secret_access_key": { + Type: "string", + }, + "s3_force_path_style": { + Type: "string", + }, + "s3_wal_path": { + Type: "string", + }, + "timestamp": { + Type: "string", + Description: "Date-time format that specifies a timezone as an offset relative to UTC e.g. 1996-12-19T16:39:57-08:00", + Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$", + }, + "uid": { + Type: "string", + Format: "uuid", + }, + }, + }, + "databases": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + Description: "User names specified here as database owners must be declared in the users key of the spec key", + }, + }, + }, + "dockerImage": { + Type: "string", + }, + "enableLogicalBackup": { + Type: "boolean", + }, + "enableMasterLoadBalancer": { + Type: "boolean", + }, + "enableReplicaLoadBalancer": { + Type: "boolean", + }, + "enableShmVolume": { + Type: "boolean", + }, + "init_containers": { + Type: "array", + Description: "Deprecated", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Allows: true, + }, + }, + }, + }, + "initContainers": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Allows: true, + }, + }, + }, + }, + "logicalBackupSchedule": { + Type: "string", + Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$", + }, + "maintenanceWindows": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + Pattern: "^\\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\\d):([0-5]?\\d)|(2[0-3]|[01]?\\d):([0-5]?\\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\\d):([0-5]?\\d)|(2[0-3]|[01]?\\d):([0-5]?\\d))\\ *$", + }, + }, + }, + "numberOfInstances": { + Type: "integer", + Minimum: &min0, + }, + "patroni": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "initdb": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "pg_hba": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "slots": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + }, + }, + "ttl": { + Type: "integer", + }, + "loop_wait": { + Type: "integer", + }, + "retry_timeout": { + Type: "integer", + }, + "maximum_lag_on_failover": { + Type: "integer", + }, + }, + }, + "podAnnotations": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "pod_priority_class_name": { + Type: "string", + Description: "Deprecated", + }, + "podPriorityClassName": { + Type: "string", + }, + "postgresql": { + Type: "object", + Required: []string{"version"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "version": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"9.3"`), + }, + { + Raw: []byte(`"9.4"`), + }, + { + Raw: []byte(`"9.5"`), + }, + { + Raw: []byte(`"9.6"`), + }, + { + Raw: []byte(`"10"`), + }, + { + Raw: []byte(`"11"`), + }, + { + Raw: []byte(`"12"`), + }, + }, + }, + "parameters": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + }, + }, + "replicaLoadBalancer": { + Type: "boolean", + Description: "Deprecated", + }, + "resources": { + Type: "object", + Required: []string{"requests", "limits"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "limits": { + Type: "object", + Required: []string{"cpu", "memory"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "cpu": { + Type: "string", + Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", + Pattern: "^(\\d+m|\\d+\\.\\d{1,3})$", + }, + "memory": { + Type: "string", + Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + }, + }, + "requests": { + Type: "object", + Required: []string{"cpu", "memory"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "cpu": { + Type: "string", + Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", + Pattern: "^(\\d+m|\\d+\\.\\d{1,3})$", + }, + "memory": { + Type: "string", + Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + }, + }, + }, + }, + "sidecars": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Allows: true, + }, + }, + }, + }, + "spiloFSGroup": { + Type: "integer", + }, + "standby": { + Type: "object", + Required: []string{"s3_wal_path"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "s3_wal_path": { + Type: "string", + }, + }, + }, + "teamId": { + Type: "string", + }, + "tolerations": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + Required: []string{"key", "operator", "effect"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "key": { + Type: "string", + }, + "operator": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"Equal"`), + }, + { + Raw: []byte(`"Exists"`), + }, + }, + }, + "value": { + Type: "string", + }, + "effect": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"NoExecute"`), + }, + { + Raw: []byte(`"NoSchedule"`), + }, + { + Raw: []byte(`"PreferNoSchedule"`), + }, + }, + }, + "tolerationSeconds": { + Type: "integer", + }, + }, + }, + }, + }, + "useLoadBalancer": { + Type: "boolean", + Description: "Deprecated", + }, + "users": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "array", + Description: "Role flags specified here must not contradict each other", + Nullable: true, + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"bypassrls"`), + }, + { + Raw: []byte(`"BYPASSRLS"`), + }, + { + Raw: []byte(`"nobypassrls"`), + }, + { + Raw: []byte(`"NOBYPASSRLS"`), + }, + { + Raw: []byte(`"createdb"`), + }, + { + Raw: []byte(`"CREATEDB"`), + }, + { + Raw: []byte(`"nocreatedb"`), + }, + { + Raw: []byte(`"NOCREATEDB"`), + }, + { + Raw: []byte(`"createrole"`), + }, + { + Raw: []byte(`"CREATEROLE"`), + }, + { + Raw: []byte(`"nocreaterole"`), + }, + { + Raw: []byte(`"NOCREATEROLE"`), + }, + { + Raw: []byte(`"inherit"`), + }, + { + Raw: []byte(`"INHERIT"`), + }, + { + Raw: []byte(`"noinherit"`), + }, + { + Raw: []byte(`"NOINHERIT"`), + }, + { + Raw: []byte(`"login"`), + }, + { + Raw: []byte(`"LOGIN"`), + }, + { + Raw: []byte(`"nologin"`), + }, + { + Raw: []byte(`"NOLOGIN"`), + }, + { + Raw: []byte(`"replication"`), + }, + { + Raw: []byte(`"REPLICATION"`), + }, + { + Raw: []byte(`"noreplication"`), + }, + { + Raw: []byte(`"NOREPLICATION"`), + }, + { + Raw: []byte(`"superuser"`), + }, + { + Raw: []byte(`"SUPERUSER"`), + }, + { + Raw: []byte(`"nosuperuser"`), + }, + { + Raw: []byte(`"NOSUPERUSER"`), + }, + }, + }, + }, + }, + }, + }, + "volume": { + Type: "object", + Required: []string{"size"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "size": { + Type: "string", + Description: "Value must not be zero", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "storageClass": { + Type: "string", + }, + "subPath": { + Type: "string", + }, + }, + }, + }, + }, + }, + }, +} + +// OperatorConfigCRDResourceValidation to check applied manifest parameters +var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + Required: []string{"kind", "apiVersion", "configuration"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "kind": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"OperatorConfiguration"`), + }, + }, + }, + "apiVersion": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"acid.zalan.do/v1"`), + }, + }, + }, + "configuration": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "docker_image": { + Type: "string", + }, + "enable_crd_validation": { + Type: "boolean", + }, + "enable_shm_volume": { + Type: "boolean", + }, + "etcd_host": { + Type: "string", + }, + "max_instances": { + Type: "integer", + Description: "-1 = disabled", + Minimum: &minDisable, + }, + "min_instances": { + Type: "integer", + Description: "-1 = disabled", + Minimum: &minDisable, + }, + "resync_period": { + Type: "string", + }, + "repair_period": { + Type: "string", + }, + "set_memory_request_to_limit": { + Type: "boolean", + }, + "sidecar_docker_images": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "workers": { + Type: "integer", + Minimum: &min1, + }, + "users": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "replication_username": { + Type: "string", + }, + "super_username": { + Type: "string", + }, + }, + }, + "kubernetes": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "cluster_domain": { + Type: "string", + }, + "cluster_labels": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "cluster_name_label": { + Type: "string", + }, + "custom_pod_annotations": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "enable_pod_antiaffinity": { + Type: "boolean", + }, + "enable_pod_disruption_budget": { + Type: "boolean", + }, + "infrastructure_roles_secret_name": { + Type: "string", + }, + "inherited_labels": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "node_readiness_label": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "oauth_token_secret_name": { + Type: "string", + }, + "pdb_name_format": { + Type: "string", + }, + "pod_antiaffinity_topology_key": { + Type: "string", + }, + "pod_environment_configmap": { + Type: "string", + }, + "pod_management_policy": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"ordered_ready"`), + }, + { + Raw: []byte(`"parallel"`), + }, + }, + }, + "pod_role_label": { + Type: "string", + }, + "pod_service_account_name": { + Type: "string", + }, + "pod_terminate_grace_period": { + Type: "string", + }, + "secret_name_template": { + Type: "string", + }, + "spilo_fsgroup": { + Type: "integer", + }, + "spilo_privileged": { + Type: "boolean", + }, + "toleration": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "watched_namespace": { + Type: "string", + }, + }, + }, + "postgres_pod_resources": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "default_cpu_limit": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "default_cpu_request": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "default_memory_limit": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "default_memory_request": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + }, + }, + "timeouts": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "pod_label_wait_timeout": { + Type: "string", + }, + "pod_deletion_wait_timeout": { + Type: "string", + }, + "ready_wait_interval": { + Type: "string", + }, + "ready_wait_timeout": { + Type: "string", + }, + "resource_check_interval": { + Type: "string", + }, + "resource_check_timeout": { + Type: "string", + }, + }, + }, + "load_balancer": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "db_hosted_zone": { + Type: "string", + }, + "enable_master_load_balancer": { + Type: "boolean", + }, + "enable_replica_load_balancer": { + Type: "boolean", + }, + "custom_service_annotations": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "master_dns_name_format": { + Type: "string", + }, + "replica_dns_name_format": { + Type: "string", + }, + }, + }, + "aws_or_gcp": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "additional_secret_mount": { + Type: "string", + }, + "additional_secret_mount_path": { + Type: "string", + }, + "aws_region": { + Type: "string", + }, + "kube_iam_role": { + Type: "string", + }, + "log_s3_bucket": { + Type: "string", + }, + "wal_s3_bucket": { + Type: "string", + }, + }, + }, + "logical_backup": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "logical_backup_schedule": { + Type: "string", + Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$", + }, + "logical_backup_docker_image": { + Type: "string", + }, + "logical_backup_s3_bucket": { + Type: "string", + }, + "logical_backup_s3_endpoint": { + Type: "string", + }, + "logical_backup_s3_sse": { + Type: "string", + }, + "logical_backup_s3_access_key_id": { + Type: "string", + }, + "logical_backup_s3_secret_access_key": { + Type: "string", + }, + }, + }, + "debug": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "debug_logging": { + Type: "boolean", + }, + "enable_database_access": { + Type: "boolean", + }, + }, + }, + "teams_api": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "enable_admin_role_for_users": { + Type: "boolean", + }, + "enable_team_superuser": { + Type: "boolean", + }, + "enable_teams_api": { + Type: "boolean", + }, + "pam_configuration": { + Type: "string", + }, + "pam_role_name": { + Type: "string", + }, + "postgres_superuser_teams": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "protected_role_names": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "team_admin_role": { + Type: "string", + }, + "team_api_role_configuration": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "teams_api_url": { + Type: "string", + }, + }, + }, + "logging_rest_api": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "api_port": { + Type: "integer", + }, + "cluster_history_entries": { + Type: "integer", + }, + "ring_log_lines": { + Type: "integer", + }, + }, + }, + "scalyr": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "scalyr_api_key": { + Type: "string", + }, + "scalyr_cpu_limit": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "scalyr_cpu_request": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "scalyr_image": { + Type: "string", + }, + "scalyr_memory_limit": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "scalyr_memory_request": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "scalyr_server_url": { + Type: "string", + }, + }, + }, + }, + }, + }, + }, +} + +func buildCRD(name, kind, plural, short string, columns []apiextv1beta1.CustomResourceColumnDefinition, validation apiextv1beta1.CustomResourceValidation) *apiextv1beta1.CustomResourceDefinition { return &apiextv1beta1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -121,24 +1016,39 @@ func buildCRD(name, kind, plural, short string, columns []apiextv1beta1.CustomRe Status: &apiextv1beta1.CustomResourceSubresourceStatus{}, }, AdditionalPrinterColumns: columns, + Validation: &validation, }, } } // PostgresCRD returns CustomResourceDefinition built from PostgresCRDResource -func PostgresCRD() *apiextv1beta1.CustomResourceDefinition { +func PostgresCRD(enableValidation *bool) *apiextv1beta1.CustomResourceDefinition { + postgresCRDvalidation := apiextv1beta1.CustomResourceValidation{} + + if enableValidation != nil && *enableValidation { + postgresCRDvalidation = PostgresCRDResourceValidation + } + return buildCRD(PostgresCRDResouceName, PostgresCRDResourceKind, PostgresCRDResourcePlural, PostgresCRDResourceShort, - PostgresCRDResourceColumns) + PostgresCRDResourceColumns, + postgresCRDvalidation) } // ConfigurationCRD returns CustomResourceDefinition built from OperatorConfigCRDResource -func ConfigurationCRD() *apiextv1beta1.CustomResourceDefinition { +func ConfigurationCRD(enableValidation *bool) *apiextv1beta1.CustomResourceDefinition { + opconfigCRDvalidation := apiextv1beta1.CustomResourceValidation{} + + if enableValidation != nil && *enableValidation { + opconfigCRDvalidation = OperatorConfigCRDResourceValidation + } + return buildCRD(OperatorConfigCRDResourceName, OperatorConfigCRDResouceKind, OperatorConfigCRDResourcePlural, OperatorConfigCRDResourceShort, - OperatorConfigCRDResourceColumns) + OperatorConfigCRDResourceColumns, + opconfigCRDvalidation) } diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index d00d40532..d97852b2f 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -150,6 +150,7 @@ type ScalyrConfiguration struct { // OperatorConfigurationData defines the operation config type OperatorConfigurationData struct { + EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` EtcdHost string `json:"etcd_host,omitempty"` DockerImage string `json:"docker_image,omitempty"` Workers uint32 `json:"workers,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 793f236a5..433d37f87 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -216,6 +216,11 @@ func (in *OperatorConfiguration) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData) { *out = *in + if in.EnableCRDValidation != nil { + in, out := &in.EnableCRDValidation, &out.EnableCRDValidation + *out = new(bool) + **out = **in + } if in.ShmVolume != nil { in, out := &in.ShmVolume, &out.ShmVolume *out = new(bool) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 9162ce27d..9db03ceb1 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -6,7 +6,7 @@ import ( "sync" "github.com/sirupsen/logrus" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" rbacv1beta1 "k8s.io/api/rbac/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -240,7 +240,7 @@ func (c *Controller) initController() { c.initClients() if configObjectName := os.Getenv("POSTGRES_OPERATOR_CONFIGURATION_OBJECT"); configObjectName != "" { - if err := c.createConfigurationCRD(); err != nil { + if err := c.createConfigurationCRD(c.opConfig.EnableCRDValidation); err != nil { c.logger.Fatalf("could not register Operator Configuration CustomResourceDefinition: %v", err) } if cfg, err := c.readOperatorConfigurationFromCRD(spec.GetOperatorNamespace(), configObjectName); err != nil { @@ -256,7 +256,7 @@ func (c *Controller) initController() { c.modifyConfigFromEnvironment() - if err := c.createPostgresCRD(); err != nil { + if err := c.createPostgresCRD(c.opConfig.EnableCRDValidation); err != nil { c.logger.Fatalf("could not register Postgres CustomResourceDefinition: %v", err) } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 3ea513879..9bed7ed13 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -25,6 +25,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result := &config.Config{} // general config + result.EnableCRDValidation = fromCRD.EnableCRDValidation result.EtcdHost = fromCRD.EtcdHost result.DockerImage = fromCRD.DockerImage result.Workers = fromCRD.Workers diff --git a/pkg/controller/util.go b/pkg/controller/util.go index 0adb85dbd..9b7dca063 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -4,7 +4,7 @@ import ( "encoding/json" "fmt" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -91,12 +91,12 @@ func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefiniti }) } -func (c *Controller) createPostgresCRD() error { - return c.createOperatorCRD(acidv1.PostgresCRD()) +func (c *Controller) createPostgresCRD(enableValidation *bool) error { + return c.createOperatorCRD(acidv1.PostgresCRD(enableValidation)) } -func (c *Controller) createConfigurationCRD() error { - return c.createOperatorCRD(acidv1.ConfigurationCRD()) +func (c *Controller) createConfigurationCRD(enableValidation *bool) error { + return c.createOperatorCRD(acidv1.ConfigurationCRD(enableValidation)) } func readDecodedRole(s string) (*spec.PgUser, error) { diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index e8de85e20..52a0c4020 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -12,10 +12,11 @@ import ( // CRD describes CustomResourceDefinition specific configuration parameters type CRD struct { - ReadyWaitInterval time.Duration `name:"ready_wait_interval" default:"4s"` - ReadyWaitTimeout time.Duration `name:"ready_wait_timeout" default:"30s"` - ResyncPeriod time.Duration `name:"resync_period" default:"30m"` - RepairPeriod time.Duration `name:"repair_period" default:"5m"` + ReadyWaitInterval time.Duration `name:"ready_wait_interval" default:"4s"` + ReadyWaitTimeout time.Duration `name:"ready_wait_timeout" default:"30s"` + ResyncPeriod time.Duration `name:"resync_period" default:"30m"` + RepairPeriod time.Duration `name:"repair_period" default:"5m"` + EnableCRDValidation *bool `name:"enable_crd_validation" default:"true"` } // Resources describes kubernetes resource specific configuration parameters diff --git a/run_operator_locally.sh b/run_operator_locally.sh index f5044dc14..9e3e082da 100755 --- a/run_operator_locally.sh +++ b/run_operator_locally.sh @@ -98,7 +98,7 @@ function build_operator_binary(){ # redirecting stderr greatly reduces non-informative output during normal builds echo "Build operator binary (stderr redirected to /dev/null)..." - make clean tools deps local test > /dev/null 2>&1 + make clean deps local test > /dev/null 2>&1 } From 551564011139d5a6cb9f635d091b2b54f56f125d Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 28 Nov 2019 17:39:25 +0100 Subject: [PATCH 016/104] install kind as GO module (#742) * install kind as GO modules * no need to set KIND_PATH --- .gitignore | 3 + e2e/Makefile | 6 +- e2e/run.sh | 14 ++-- go.mod | 20 ++++-- go.sum | 194 ++++++++++++++++++++++++++++++++++++++------------- 5 files changed, 171 insertions(+), 66 deletions(-) diff --git a/.gitignore b/.gitignore index 34d526578..991fe754f 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,9 @@ coverage.xml .hypothesis/ .pytest_cache/ +# e2e tests +e2e/manifests + # Translations *.mo *.pot diff --git a/e2e/Makefile b/e2e/Makefile index ab19539f8..77059f3eb 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -23,7 +23,6 @@ ifndef GOPATH GOPATH := $(HOME)/go endif -KIND_PATH := $(GOPATH)/bin PATH := $(GOPATH)/bin:$(PATH) default: tools @@ -43,10 +42,7 @@ push: docker tools: docker # install pinned version of 'kind' - # leave the name as is to avoid overwriting official binary named `kind` - wget https://github.com/kubernetes-sigs/kind/releases/download/v0.5.1/kind-linux-amd64 - chmod +x kind-linux-amd64 - mv kind-linux-amd64 $(KIND_PATH) + GO111MODULE=on go get sigs.k8s.io/kind@v0.5.1 test: ./run.sh diff --git a/e2e/run.sh b/e2e/run.sh index 237960b89..c7825bfd3 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -30,15 +30,15 @@ function pull_images(){ function start_kind(){ # avoid interference with previous test runs - if [[ $(kind-linux-amd64 get clusters | grep "^${cluster_name}*") != "" ]] + if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]] then - kind-linux-amd64 delete cluster --name ${cluster_name} + kind delete cluster --name ${cluster_name} fi - kind-linux-amd64 create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml - kind-linux-amd64 load docker-image "${operator_image}" --name ${cluster_name} - kind-linux-amd64 load docker-image "${e2e_test_image}" --name ${cluster_name} - KUBECONFIG="$(kind-linux-amd64 get kubeconfig-path --name=${cluster_name})" + kind create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml + kind load docker-image "${operator_image}" --name ${cluster_name} + kind load docker-image "${e2e_test_image}" --name ${cluster_name} + KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" export KUBECONFIG } @@ -58,7 +58,7 @@ function run_tests(){ function clean_up(){ unset KUBECONFIG - kind-linux-amd64 delete cluster --name ${cluster_name} + kind delete cluster --name ${cluster_name} rm -rf ${kubeconfig_path} } diff --git a/go.mod b/go.mod index 8efd4939d..9012721d1 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,22 @@ go 1.12 require ( github.com/aws/aws-sdk-go v1.25.1 + github.com/emicklei/go-restful v2.9.6+incompatible // indirect + github.com/evanphx/json-patch v4.5.0+incompatible // indirect + github.com/googleapis/gnostic v0.3.0 // indirect github.com/imdario/mergo v0.3.7 // indirect github.com/lib/pq v1.2.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/sirupsen/logrus v1.4.2 - gopkg.in/yaml.v2 v2.2.2 - k8s.io/api v0.0.0-20190927115716-5d581ce610b0 - k8s.io/apiextensions-apiserver v0.0.0-20190927042040-728319705b32 - k8s.io/apimachinery v0.0.0-20190927035529-0104e33c351d - k8s.io/client-go v0.0.0-20190926235751-95884bf844a9 - k8s.io/code-generator v0.0.0-20190927075303-016f2b3d74d0 + golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c // indirect + golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 // indirect + golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 // indirect + golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/yaml.v2 v2.2.5 + k8s.io/api v0.0.0-20191121015604-11707872ac1c + k8s.io/apiextensions-apiserver v0.0.0-20191121021419-88daf26ec3b8 + k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a + k8s.io/client-go v11.0.0+incompatible + k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e ) diff --git a/go.sum b/go.sum index 8995710ba..e7a0a15e3 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,7 @@ github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxB github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -19,8 +20,10 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -28,18 +31,21 @@ github.com/aws/aws-sdk-go v1.25.1 h1:d7zDXFT2Tgq/yw7Wku49+lKisE8Xc85erb+8PlE/Shk github.com/aws/aws-sdk-go v1.25.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -47,16 +53,27 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w= +github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -66,6 +83,7 @@ github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70t github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= @@ -74,37 +92,51 @@ github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwds github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.2 h1:SStNd1jRcYtfKCN7R0laGNs80WYYvn5CbBjM2sOmCrE= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -112,7 +144,10 @@ github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -122,20 +157,25 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= +github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= @@ -148,13 +188,17 @@ github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBv github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= @@ -168,6 +212,12 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -181,41 +231,56 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d h1:LznySqW8MqVeFh+pW6rOkFdld9QQ7jRydBKKM6jyPVI= github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d/go.mod h1:u3hJ0kqCQu/cPpsu3RbCOPZ0d7V3IjPjv1adNRleM9I= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.4/go.mod h1:oCXIBxdI62A4cR6aTRJCgetEjecSIYzOEaeAn4iYEpM= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -223,14 +288,24 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -238,8 +313,13 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c h1:/nJuwDLoL/zrqY6gf57vxC+Pi+pZ8bfhpPkicO5H7W4= +golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 h1:I6A9Ag9FpEKOjcKrRNjQkPHawoXIhKyTGfvvjFAiiAk= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -253,14 +333,19 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc h1:gkKoSkUmnU6bpS/VhkuO27bzQeSA51uaEfbOW5dNb68= -golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 h1:e6HwijUxhDe+hPNjZQQn9bA5PW3vNmnN64U2ZW759Lk= +golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -274,6 +359,7 @@ golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -284,17 +370,25 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 h1:ZBzSG/7F4eNKz2L3GE9o300RX0Az1Bw5HF7PDraD+qU= +golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -302,11 +396,15 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f h1:3MlESg/jvTr87F4ttA/q4B+uhe/q6qleC9/DP+IwQmY= +golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 h1:OB/uP/Puiu5vS5QMRPrXCDWUPb+kt8f1KW8oQzFejQw= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e h1:jRyg0XfpwWlhEV8mDfdNGBeSJM2fuyh9Yjrnd8kF2Ts= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -318,65 +416,65 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o= -gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48= -k8s.io/api v0.0.0-20190925180651-d58b53da08f5/go.mod h1:blPYY5r6fKug8SVOnjDtFAlzZzInCRL9NNls66SFhFI= -k8s.io/api v0.0.0-20190927115716-5d581ce610b0 h1:fwx2jAKNlXBQ8uiB3RNN5hVU/nEJTEBg/CfxoXEYri4= -k8s.io/api v0.0.0-20190927115716-5d581ce610b0/go.mod h1:l2ZHS8QbgqodGx7yrYsOSwIxOR76BpGiW1OywXo9PFI= -k8s.io/apiextensions-apiserver v0.0.0-20190927042040-728319705b32 h1:ss1T2mi6o+ji42DAoFxrophF9dysSKUXyq6fTmCMvFI= -k8s.io/apiextensions-apiserver v0.0.0-20190927042040-728319705b32/go.mod h1:SMmFyjBO4fGs1v/nrTzNbhyg4PEGUH+h3ilWsi90RPk= -k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4= -k8s.io/apimachinery v0.0.0-20190923155427-ec87dd743e08/go.mod h1:grJJH0hgilA2pYoUiJcPu2EDUal95NTq1vpxxvMLSu8= -k8s.io/apimachinery v0.0.0-20190925235427-62598f38f24e/go.mod h1:grJJH0hgilA2pYoUiJcPu2EDUal95NTq1vpxxvMLSu8= -k8s.io/apimachinery v0.0.0-20190927035529-0104e33c351d h1:oYLB5Nk2IOm17BHdatnaWAgzNGzq/5dlWy7Bzo5Htdc= -k8s.io/apimachinery v0.0.0-20190927035529-0104e33c351d/go.mod h1:grJJH0hgilA2pYoUiJcPu2EDUal95NTq1vpxxvMLSu8= -k8s.io/apiserver v0.0.0-20190927000204-dd401ce564d5/go.mod h1:oXS8tPljvXCBPjzDnAM7xnx26pzwFT+oIqIUCmG8Pj8= -k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90 h1:mLmhKUm1X+pXu0zXMEzNsOF5E2kKFGe5o6BZBIIqA6A= -k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90/go.mod h1:J69/JveO6XESwVgG53q3Uz5OSfgsv4uxpScmmyYOOlk= -k8s.io/client-go v0.0.0-20190925235746-07054768d98d/go.mod h1:KumMj5rt+3qCPy5LJipGocsmMx6RW8vdDAs8QNK6jvU= -k8s.io/client-go v0.0.0-20190926235751-95884bf844a9 h1:YpRGAa4i68p7SGIuJAaZgE7oumAMxzsTux2gTur2eMM= -k8s.io/client-go v0.0.0-20190926235751-95884bf844a9/go.mod h1:KumMj5rt+3qCPy5LJipGocsmMx6RW8vdDAs8QNK6jvU= +k8s.io/api v0.0.0-20191121015604-11707872ac1c h1:Z87my3sF4WhG0OMxzARkWY/IKBtOr+MhXZAb4ts6qFc= +k8s.io/api v0.0.0-20191121015604-11707872ac1c/go.mod h1:R/s4gKT0V/cWEnbQa9taNRJNbWUK57/Dx6cPj6MD3A0= +k8s.io/apiextensions-apiserver v0.0.0-20191121021419-88daf26ec3b8 h1:SrFLwOURsuwzuCi0zJdaBbPF31AcV9JUwpwIVosnnE4= +k8s.io/apiextensions-apiserver v0.0.0-20191121021419-88daf26ec3b8/go.mod h1:NMIy5Wa/or8CsLhYRleOp9CWAHVdcWpzT6Ufx1SNVjA= +k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a h1:9V03T5lHv/iF4fSgvMCd+iB86AgEgmzLpheMqIJy7hs= +k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apiserver v0.0.0-20191121020624-6eed2f5a3289/go.mod h1:7P+0qMKoaggchirHLUSCVD22ohdkjN19+qQOKcAdfbI= +k8s.io/client-go v0.0.0-20191121015835-571c0ef67034 h1:+/ppGIi1rJThJAz/xJSSOuD82gb6E5jRv2305MSznxQ= +k8s.io/client-go v0.0.0-20191121015835-571c0ef67034/go.mod h1:Adhj+OyDRsEXTnL9BfL7xbLWGWMCqGLWpMqGHkZI4J8= k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= -k8s.io/code-generator v0.0.0-20190925195306-32dfb485ddce/go.mod h1:4MfOrxyyZxxCuenwsdaJRtoSnOP5T13jE2LRYPZ6KeY= -k8s.io/code-generator v0.0.0-20190927075303-016f2b3d74d0 h1:rhwEVFHoBm42V0b7yN9SUdbWzfCVndLzRV8YGIi0uWY= -k8s.io/code-generator v0.0.0-20190927075303-016f2b3d74d0/go.mod h1:4MfOrxyyZxxCuenwsdaJRtoSnOP5T13jE2LRYPZ6KeY= -k8s.io/component-base v0.0.0-20190926082537-804254d56004/go.mod h1:+sedDd0Yj/9lFSZjan8FdX4Jednr2we+Q0ZDeicbKSc= +k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e h1:HB9Zu5ZUvJfNpLiTPhz+CebVKV8C39qTBMQkAgAZLNw= +k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= +k8s.io/component-base v0.0.0-20191121020327-771114ba3383/go.mod h1:tv9ITs6VEFWkF+kHwY4GiFvDr9vUGKJ4X/8+Z+oqVLk= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM= k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf h1:EYm5AW/UUDbnmnI+gK0TJDVK9qPLhM+sRHYanNKw0EQ= -k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20190920012459-5008bf6f8cd6 h1:rfepARh/ECp66dk9TTmT//1PBkHffjnxhdOrgH4m+eA= -k8s.io/utils v0.0.0-20190920012459-5008bf6f8cd6/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= +k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= +k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= +sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 11c2e815f7740d2905d6088d23f1318b4b9aebef Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 2 Dec 2019 15:27:47 +0100 Subject: [PATCH 017/104] include status subresource in validation (#744) * include status subresource in validation --- .../crds/operatorconfigurations.yaml | 4 ++++ manifests/operatorconfiguration.crd.yaml | 4 ++++ manifests/postgresql.crd.yaml | 4 ++++ pkg/apis/acid.zalan.do/v1/crds.go | 16 ++++++++++++++++ 4 files changed, 28 insertions(+) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index ff92bc064..f50180c2a 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -298,3 +298,7 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string + status: + type: object + additionalProperties: + type: string diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 753415a15..7e9c03509 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -274,3 +274,7 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string + status: + type: object + additionalProperties: + type: string diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 4a578b324..d563962bb 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -325,3 +325,7 @@ spec: type: string subPath: type: string + status: + type: object + additionalProperties: + type: string diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 9ee76103f..35a8227b0 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -578,6 +578,14 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "status": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, }, }, } @@ -994,6 +1002,14 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "status": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, }, }, } From 7037974d4812d9d80dba134bef1082e6f3723c20 Mon Sep 17 00:00:00 2001 From: Frederik Wagner <55986520+fwagner-smf@users.noreply.github.com> Date: Wed, 4 Dec 2019 17:47:05 +0100 Subject: [PATCH 018/104] fix s3 bucket endpoint/sse configuration in logical backup dump.sh (#750) --- docker/logical-backup/dump.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh index 5c2e478a9..78217322b 100755 --- a/docker/logical-backup/dump.sh +++ b/docker/logical-backup/dump.sh @@ -41,8 +41,8 @@ function aws_upload { args=() [[ ! -z "$EXPECTED_SIZE" ]] && args+=("--expected-size=$EXPECTED_SIZE") - [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=\"$LOGICAL_BACKUP_S3_ENDPOINT\"") - [[ ! "$LOGICAL_BACKUP_S3_SSE" == "" ]] && args+=("--sse=\"$LOGICAL_BACKUP_S3_SSE\"") + [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT") + [[ ! "$LOGICAL_BACKUP_S3_SSE" == "" ]] && args+=("--sse=$LOGICAL_BACKUP_S3_SSE") aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}" --debug } From 076116589d88b6bf2d2788da1800c31b726bdea6 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Tue, 10 Dec 2019 13:54:12 +0000 Subject: [PATCH 019/104] hack: update shebang (#755) Make the shebang work on NixOS and other non-Ubuntu systems. --- hack/update-codegen.sh | 2 +- hack/verify-codegen.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 4625f7c9a..280da9385 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -o errexit set -o nounset diff --git a/hack/verify-codegen.sh b/hack/verify-codegen.sh index 904586d05..68710015e 100755 --- a/hack/verify-codegen.sh +++ b/hack/verify-codegen.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -o errexit set -o nounset From 107334fe71af5fa4cc7df1b2af7043f3cf28bc24 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 10 Dec 2019 15:45:54 +0100 Subject: [PATCH 020/104] Add global option to enable/disable init containers and sidecars (#478) * Add global option to enable/disable init containers and sidecars * update dependencies --- .../crds/operatorconfigurations.yaml | 4 ++ charts/postgres-operator/values-crd.yaml | 4 ++ charts/postgres-operator/values.yaml | 6 ++- docs/reference/operator_parameters.md | 14 ++++-- docs/user.md | 8 ++++ e2e/tests/test_e2e.py | 17 ++++--- go.mod | 14 +++--- go.sum | 44 +++++++++++-------- manifests/configmap.yaml | 2 + manifests/operatorconfiguration.crd.yaml | 4 ++ ...gresql-operator-default-configuration.yaml | 6 ++- pkg/apis/acid.zalan.do/v1/crds.go | 6 +++ .../v1/operator_configuration_type.go | 2 + .../acid.zalan.do/v1/zz_generated.deepcopy.go | 10 +++++ pkg/cluster/k8sres.go | 23 +++++++--- pkg/cluster/k8sres_test.go | 2 +- pkg/cluster/resources.go | 11 +++++ pkg/cluster/types.go | 2 +- pkg/controller/operator_config.go | 2 + pkg/controller/util_test.go | 2 +- pkg/util/config/config.go | 2 + 21 files changed, 138 insertions(+), 47 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index f50180c2a..d50a2b431 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -107,10 +107,14 @@ spec: type: object additionalProperties: type: string + enable_init_containers: + type: boolean enable_pod_antiaffinity: type: boolean enable_pod_disruption_budget: type: boolean + enable_sidecars: + type: boolean infrastructure_roles_secret_name: type: string inherited_labels: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index c6f11e493..40cccfb54 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -63,10 +63,14 @@ configKubernetes: # keya: valuea # keyb: valueb + # enables initContainers to run actions before Spilo is started + enable_init_containers: true # toggles pod anti affinity on the Postgres pods enable_pod_antiaffinity: false # toggles PDB to set to MinAvailabe 0 or 1 enable_pod_disruption_budget: true + # enables sidecar containers to run alongside Spilo in the same pod + enable_sidecars: true # name of the secret containing infrastructure roles names and passwords # infrastructure_roles_secret_name: postgresql-infrastructure-roles diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index ae9e06fe9..826862c4a 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -57,12 +57,16 @@ configKubernetes: # label assigned to Kubernetes objects created by the operator cluster_name_label: version # annotations attached to each database pod - # custom_pod_annotations: keya:valuea,keyb:valueb + # custom_pod_annotations: "keya:valuea,keyb:valueb" + # enables initContainers to run actions before Spilo is started + enable_init_containers: "true" # toggles pod anti affinity on the Postgres pods enable_pod_antiaffinity: "false" # toggles PDB to set to MinAvailabe 0 or 1 enable_pod_disruption_budget: "true" + # enables sidecar containers to run alongside Spilo in the same pod + enable_sidecars: "true" # name of the secret containing infrastructure roles names and passwords # infrastructure_roles_secret_name: postgresql-infrastructure-roles diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index e16282fd2..3de4a5be4 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -87,9 +87,9 @@ Those are top-level keys, containing both leaf keys and groups. repository](https://github.com/zalando/spilo). * **sidecar_docker_images** - a map of sidecar names to docker images for the containers to run alongside - Spilo. In case of the name conflict with the definition in the cluster - manifest the cluster-specific one is preferred. + a map of sidecar names to docker images to run with Spilo. In case of the name + conflict with the definition in the cluster manifest the cluster-specific one + is preferred. * **enable_shm_volume** Instruct operator to start any new database pod without limitations on shm @@ -196,6 +196,14 @@ configuration they are grouped under the `kubernetes` key. [admin docs](../administrator.md#pod-disruption-budget) for more information. Default is true. +* **enable_init_containers** + global option to allow for creating init containers to run actions before + Spilo is started. Default is true. + +* **enable_sidecars** + global option to allow for creating sidecar containers to run alongside Spilo + on the same pod. Default is true. + * **secret_name_template** a template for the name of the database user secrets generated by the operator. `{username}` is replaced with name of the secret, `{cluster}` with diff --git a/docs/user.md b/docs/user.md index ee8e7183c..272defe08 100644 --- a/docs/user.md +++ b/docs/user.md @@ -350,6 +350,10 @@ variables are always passed to sidecars: The PostgreSQL volume is shared with sidecars and is mounted at `/home/postgres/pgdata`. +**Note**: The operator will not create a cluster if sidecar containers are +specified but globally disabled in the configuration. The `enable_sidecars` +option must be set to `true`. + ## InitContainers Support Each cluster can specify arbitrary init containers to run. These containers can @@ -374,6 +378,10 @@ spec: `initContainers` accepts full `v1.Container` definition. +**Note**: The operator will not create a cluster if `initContainers` are +specified but globally disabled in the configuration. The +`enable_init_containers` option must be set to `true`. + ## Increase volume size PostgreSQL operator supports statefulset volume resize if you're using the diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 52aa0549a..88a7f1f34 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -182,17 +182,12 @@ class EndToEndTestCase(unittest.TestCase): # update the cluster-wide image of the logical backup pod image = "test-image-name" - config_map_patch = { + patch_logical_backup_image = { "data": { "logical_backup_docker_image": image, } } - k8s.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) - - operator_pod = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector="name=postgres-operator").items[0].metadata.name - k8s.api.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf - k8s.wait_for_operator_pod_start() + k8s.update_config(patch_logical_backup_image) jobs = k8s.get_logical_backup_job().items actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image @@ -319,6 +314,14 @@ class K8s: def wait_for_logical_backup_job_creation(self): self.wait_for_logical_backup_job(expected_num_of_jobs=1) + def update_config(self, config_map_patch): + self.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) + + operator_pod = self.api.core_v1.list_namespaced_pod( + 'default', label_selector="name=postgres-operator").items[0].metadata.name + self.api.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf + self.wait_for_operator_pod_start() + def create_with_kubectl(self, path): subprocess.run(["kubectl", "create", "-f", path]) diff --git a/go.mod b/go.mod index 9012721d1..b8c7b1615 100644 --- a/go.mod +++ b/go.mod @@ -3,23 +3,21 @@ module github.com/zalando/postgres-operator go 1.12 require ( - github.com/aws/aws-sdk-go v1.25.1 - github.com/emicklei/go-restful v2.9.6+incompatible // indirect - github.com/evanphx/json-patch v4.5.0+incompatible // indirect - github.com/googleapis/gnostic v0.3.0 // indirect - github.com/imdario/mergo v0.3.7 // indirect + github.com/aws/aws-sdk-go v1.25.44 + github.com/imdario/mergo v0.3.8 // indirect github.com/lib/pq v1.2.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/sirupsen/logrus v1.4.2 - golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c // indirect + golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e // indirect golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 // indirect golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 // indirect - golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f // indirect + golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v2 v2.2.5 + gopkg.in/yaml.v2 v2.2.4 k8s.io/api v0.0.0-20191121015604-11707872ac1c k8s.io/apiextensions-apiserver v0.0.0-20191121021419-88daf26ec3b8 k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a k8s.io/client-go v11.0.0+incompatible k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e + sigs.k8s.io/kind v0.5.1 // indirect ) diff --git a/go.sum b/go.sum index e7a0a15e3..c7f7be037 100644 --- a/go.sum +++ b/go.sum @@ -27,8 +27,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.25.1 h1:d7zDXFT2Tgq/yw7Wku49+lKisE8Xc85erb+8PlE/Shk= -github.com/aws/aws-sdk-go v1.25.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.25.44 h1:n9ahFoiyn66smjF34hYr3tb6/ZdBcLuFz7BCDhHyJ7I= +github.com/aws/aws-sdk-go v1.25.44/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -90,14 +90,12 @@ github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= @@ -112,7 +110,6 @@ github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29g github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.2 h1:SStNd1jRcYtfKCN7R0laGNs80WYYvn5CbBjM2sOmCrE= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= @@ -123,7 +120,6 @@ github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= @@ -160,6 +156,7 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= @@ -177,9 +174,10 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= -github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -210,8 +208,8 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= @@ -237,13 +235,14 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -265,12 +264,14 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= @@ -311,12 +312,11 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c h1:/nJuwDLoL/zrqY6gf57vxC+Pi+pZ8bfhpPkicO5H7W4= -golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e h1:egKlR8l7Nu9vHGWbcUV8lqR4987UfUbBd7GbhqGzNYU= +golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 h1:I6A9Ag9FpEKOjcKrRNjQkPHawoXIhKyTGfvvjFAiiAk= @@ -368,8 +368,8 @@ golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190621203818-d432491b9138/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 h1:ZBzSG/7F4eNKz2L3GE9o300RX0Az1Bw5HF7PDraD+qU= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -398,8 +398,8 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f h1:3MlESg/jvTr87F4ttA/q4B+uhe/q6qleC9/DP+IwQmY= -golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d h1:/iIZNFGxc/a7C3yWjGcnboV+Tkc7mxr+p6fDztwoxuM= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 h1:OB/uP/Puiu5vS5QMRPrXCDWUPb+kt8f1KW8oQzFejQw= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= @@ -438,16 +438,18 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20191121015604-11707872ac1c h1:Z87my3sF4WhG0OMxzARkWY/IKBtOr+MhXZAb4ts6qFc= k8s.io/api v0.0.0-20191121015604-11707872ac1c/go.mod h1:R/s4gKT0V/cWEnbQa9taNRJNbWUK57/Dx6cPj6MD3A0= k8s.io/apiextensions-apiserver v0.0.0-20191121021419-88daf26ec3b8 h1:SrFLwOURsuwzuCi0zJdaBbPF31AcV9JUwpwIVosnnE4= k8s.io/apiextensions-apiserver v0.0.0-20191121021419-88daf26ec3b8/go.mod h1:NMIy5Wa/or8CsLhYRleOp9CWAHVdcWpzT6Ufx1SNVjA= +k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a h1:9V03T5lHv/iF4fSgvMCd+iB86AgEgmzLpheMqIJy7hs= k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apiserver v0.0.0-20191121020624-6eed2f5a3289/go.mod h1:7P+0qMKoaggchirHLUSCVD22ohdkjN19+qQOKcAdfbI= @@ -463,8 +465,10 @@ k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKO k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= @@ -474,6 +478,10 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +sigs.k8s.io/kind v0.5.1 h1:BYnHEJ9DC+0Yjlyyehqd3xnKtEmFdLKU8QxqOqvQzdw= +sigs.k8s.io/kind v0.5.1/go.mod h1:L+Kcoo83/D1+ryU5P2VFbvYm0oqbkJn9zTZq0KNxW68= +sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296 h1:iQaIG5Dq+3qSiaFrJ/l/0MjjxKmdwyVNpKRYJwUe/+0= +sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296/go.mod h1:ztX4zYc/QIww3gSripwF7TBOarBTm5BvyAMem0kCzOE= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 59d3abfde..5a975c68f 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -23,11 +23,13 @@ data: # enable_admin_role_for_users: "true" # enable_crd_validation: "true" # enable_database_access: "true" + # enable_init_containers: "true" enable_master_load_balancer: "false" # enable_pod_antiaffinity: "false" # enable_pod_disruption_budget: "true" enable_replica_load_balancer: "false" # enable_shm_volume: "true" + # enable_sidecars: "true" # enable_team_superuser: "false" enable_teams_api: "false" # etcd_host: "" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 7e9c03509..bed892dc8 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -83,10 +83,14 @@ spec: type: object additionalProperties: type: string + enable_init_containers: + type: boolean enable_pod_antiaffinity: type: boolean enable_pod_disruption_budget: type: boolean + enable_sidecars: + type: boolean infrastructure_roles_secret_name: type: string inherited_labels: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 7a867d5df..84e12b4ee 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -26,14 +26,16 @@ configuration: # custom_pod_annotations: # keya: valuea # keyb: valueb + enable_init_containers: true enable_pod_antiaffinity: false enable_pod_disruption_budget: true - # infrastructure_roles_secret_name: postgresql-infrastructure-roles + enable_sidecars: true + # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" # inherited_labels: # - application # - environment # node_readiness_label: - # status: ready + # status: ready oauth_token_secret_name: postgresql-operator pdb_name_format: "postgres-{cluster}-pdb" pod_antiaffinity_topology_key: "kubernetes.io/hostname" diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 35a8227b0..2a25c374f 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -694,12 +694,18 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "enable_init_containers": { + Type: "boolean", + }, "enable_pod_antiaffinity": { Type: "boolean", }, "enable_pod_disruption_budget": { Type: "boolean", }, + "enable_sidecars": { + Type: "boolean", + }, "infrastructure_roles_secret_name": { Type: "string", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index d97852b2f..f76790ad5 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -50,6 +50,8 @@ type KubernetesMetaConfiguration struct { WatchedNamespace string `json:"watched_namespace,omitempty"` PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` EnablePodDisruptionBudget *bool `json:"enable_pod_disruption_budget,omitempty"` + EnableInitContainers *bool `json:"enable_init_containers,omitempty"` + EnableSidecars *bool `json:"enable_sidecars,omitempty"` SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"` ClusterDomain string `json:"cluster_domain"` OAuthTokenSecretName spec.NamespacedName `json:"oauth_token_secret_name,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 433d37f87..16f5a9d67 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -81,6 +81,16 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura *out = new(bool) **out = **in } + if in.EnableInitContainers != nil { + in, out := &in.EnableInitContainers, &out.EnableInitContainers + *out = new(bool) + **out = **in + } + if in.EnableSidecars != nil { + in, out := &in.EnableSidecars, &out.EnableSidecars + *out = new(bool) + **out = **in + } out.OAuthTokenSecretName = in.OAuthTokenSecretName out.InfrastructureRolesSecretName = in.InfrastructureRolesSecretName if in.ClusterLabels != nil { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 95e83454f..73be712ca 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -720,6 +720,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef var ( err error + initContainers []v1.Container sidecarContainers []v1.Container podTemplate *v1.PodTemplateSpec volumeClaimTemplate *v1.PersistentVolumeClaim @@ -786,6 +787,13 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef return nil, fmt.Errorf("could not generate resource requirements: %v", err) } + if spec.InitContainers != nil && len(spec.InitContainers) > 0 { + if c.OpConfig.EnableInitContainers != nil && !(*c.OpConfig.EnableInitContainers) { + c.logger.Warningf("initContainers specified but disabled in configuration - next statefulset creation would fail") + } + initContainers = spec.InitContainers + } + customPodEnvVarsList := make([]v1.EnvVar, 0) if c.OpConfig.PodEnvironmentConfigMap != "" { @@ -872,9 +880,14 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef } // generate sidecar containers - if sidecarContainers, err = generateSidecarContainers(sideCars, volumeMounts, defaultResources, - c.OpConfig.SuperUsername, c.credentialSecretName(c.OpConfig.SuperUsername), c.logger); err != nil { - return nil, fmt.Errorf("could not generate sidecar containers: %v", err) + if sideCars != nil && len(sideCars) > 0 { + if c.OpConfig.EnableSidecars != nil && !(*c.OpConfig.EnableSidecars) { + c.logger.Warningf("sidecars specified but disabled in configuration - next statefulset creation would fail") + } + if sidecarContainers, err = generateSidecarContainers(sideCars, volumeMounts, defaultResources, + c.OpConfig.SuperUsername, c.credentialSecretName(c.OpConfig.SuperUsername), c.logger); err != nil { + return nil, fmt.Errorf("could not generate sidecar containers: %v", err) + } } tolerationSpec := tolerations(&spec.Tolerations, c.OpConfig.PodToleration) @@ -894,7 +907,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.labelsSet(true), annotations, spiloContainer, - spec.InitContainers, + initContainers, sidecarContainers, &tolerationSpec, effectiveFSGroup, @@ -1412,7 +1425,7 @@ func (c *Cluster) generatePodDisruptionBudget() *policybeta1.PodDisruptionBudget pdbEnabled := c.OpConfig.EnablePodDisruptionBudget // if PodDisruptionBudget is disabled or if there are no DB pods, set the budget to 0. - if (pdbEnabled != nil && !*pdbEnabled) || c.Spec.NumberOfInstances <= 0 { + if (pdbEnabled != nil && !(*pdbEnabled)) || c.Spec.NumberOfInstances <= 0 { minAvailable = intstr.FromInt(0) } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 5b206f760..e8fe05456 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -3,7 +3,7 @@ package cluster import ( "reflect" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "testing" diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 3e8f73916..c94a7bb46 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -65,6 +65,17 @@ func (c *Cluster) listResources() error { func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { c.setProcessName("creating statefulset") + // check if it's allowed that spec contains initContainers + if c.Spec.InitContainers != nil && len(c.Spec.InitContainers) > 0 && + c.OpConfig.EnableInitContainers != nil && !(*c.OpConfig.EnableInitContainers) { + return nil, fmt.Errorf("initContainers specified but disabled in configuration") + } + // check if it's allowed that spec contains sidecars + if c.Spec.Sidecars != nil && len(c.Spec.Sidecars) > 0 && + c.OpConfig.EnableSidecars != nil && !(*c.OpConfig.EnableSidecars) { + return nil, fmt.Errorf("sidecar containers specified but disabled in configuration") + } + statefulSetSpec, err := c.generateStatefulSet(&c.Spec) if err != nil { return nil, fmt.Errorf("could not generate statefulset: %v", err) diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index afdc7376e..138b7015c 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -5,7 +5,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" appsv1 "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" policybeta1 "k8s.io/api/policy/v1beta1" "k8s.io/apimachinery/pkg/types" ) diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 9bed7ed13..56ba91d02 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -54,6 +54,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat result.EnablePodDisruptionBudget = fromCRD.Kubernetes.EnablePodDisruptionBudget + result.EnableInitContainers = fromCRD.Kubernetes.EnableInitContainers + result.EnableSidecars = fromCRD.Kubernetes.EnableSidecars result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate result.OAuthTokenSecretName = fromCRD.Kubernetes.OAuthTokenSecretName result.InfrastructureRolesSecretName = fromCRD.Kubernetes.InfrastructureRolesSecretName diff --git a/pkg/controller/util_test.go b/pkg/controller/util_test.go index c9e16cbd9..a5d3c7ac5 100644 --- a/pkg/controller/util_test.go +++ b/pkg/controller/util_test.go @@ -9,7 +9,7 @@ import ( "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/k8sutil" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 52a0c4020..d46cba2b2 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -123,6 +123,8 @@ type Config struct { ReplicaDNSNameFormat StringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"` PDBNameFormat StringTemplate `name:"pdb_name_format" default:"postgres-{cluster}-pdb"` EnablePodDisruptionBudget *bool `name:"enable_pod_disruption_budget" default:"true"` + EnableInitContainers *bool `name:"enable_init_containers" default:"true"` + EnableSidecars *bool `name:"enable_sidecars" default:"true"` Workers uint32 `name:"workers" default:"4"` APIPort int `name:"api_port" default:"8080"` RingLogLines int `name:"ring_log_lines" default:"100"` From 062843925686cd155c254ce86eb33522f3a4ba43 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 10 Dec 2019 16:30:57 +0100 Subject: [PATCH 021/104] fix cpu resource validation (#757) --- charts/postgres-operator/crds/postgresqls.yaml | 4 ++-- manifests/postgresql.crd.yaml | 4 ++-- pkg/apis/acid.zalan.do/v1/crds.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index a8c5f2954..198afe119 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -222,7 +222,7 @@ spec: # only the format of the given number. # # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+\.\d{1,3})$' + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' # Note: the value specified here must not be zero or be lower # than the corresponding request. memory: @@ -253,7 +253,7 @@ spec: # only the format of the given number. # # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+\.\d{1,3})$' + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' # Note: the value specified here must not be zero or be higher # than the corresponding limit. memory: diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index d563962bb..3b0f652ea 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -186,7 +186,7 @@ spec: # only the format of the given number. # # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+\.\d{1,3})$' + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' # Note: the value specified here must not be zero or be lower # than the corresponding request. memory: @@ -217,7 +217,7 @@ spec: # only the format of the given number. # # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+\.\d{1,3})$' + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' # Note: the value specified here must not be zero or be higher # than the corresponding limit. memory: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 2a25c374f..75704afde 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -356,7 +356,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "cpu": { Type: "string", Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", - Pattern: "^(\\d+m|\\d+\\.\\d{1,3})$", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", }, "memory": { Type: "string", @@ -372,7 +372,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "cpu": { Type: "string", Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", - Pattern: "^(\\d+m|\\d+\\.\\d{1,3})$", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", }, "memory": { Type: "string", From cd110aabf48e3630483af7ca4b6647faed7f62e2 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 12 Dec 2019 16:43:55 +0100 Subject: [PATCH 022/104] Enforce minimum cpu and memory limits (#731) * add validation for PG resources and volume size * check resource requests also on UPDATE and SYNC + update docs * if cluster was running don't error on sync --- docs/user.md | 35 +++++++++++++++++-- manifests/standby-manifest.yaml | 2 +- pkg/cluster/cluster.go | 59 +++++++++++++++++++++++++++++++++ pkg/cluster/k8sres.go | 4 +-- pkg/cluster/sync.go | 11 ++++++ pkg/controller/controller.go | 4 +-- pkg/util/util.go | 8 ++--- pkg/util/util_test.go | 12 +++---- 8 files changed, 117 insertions(+), 18 deletions(-) diff --git a/docs/user.md b/docs/user.md index 272defe08..d9e92d5d6 100644 --- a/docs/user.md +++ b/docs/user.md @@ -30,7 +30,7 @@ spec: databases: foo: zalando postgresql: - version: "10" + version: "11" ``` Once you cloned the Postgres Operator [repository](https://github.com/zalando/postgres-operator) @@ -40,6 +40,9 @@ you can find this example also in the manifests folder: kubectl create -f manifests/minimal-postgres-manifest.yaml ``` +Note, that the minimum volume size to run the `postgresql` resource on Elastic +Block Storage (EBS) is `1Gi`. + ## Watch pods being created ```bash @@ -182,6 +185,32 @@ See [infrastructure roles secret](../manifests/infrastructure-roles.yaml) and [infrastructure roles configmap](../manifests/infrastructure-roles-configmap.yaml) for the examples. +## Resource definition + +The compute resources to be used for the Postgres containers in the pods can be +specified in the postgresql cluster manifest. + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-minimal-cluster +spec: + resources: + requests: + cpu: 10m + memory: 100Mi + limits: + cpu: 300m + memory: 300Mi +``` + +The minimum limit to properly run the `postgresql` resource is `256m` for `cpu` +and `256Mi` for `memory`. If a lower value is set in the manifest the operator +will cancel ADD or UPDATE events on this resource with an error. If no +resources are defined in the manifest the operator will obtain the configured +[default requests](reference/operator_parameters.md#kubernetes-resource-requests). + ## Use taints and tolerations for dedicated PostgreSQL nodes To ensure Postgres pods are running on nodes without any other application pods, @@ -305,7 +334,7 @@ Things to note: - There is no way to transform a non-standby cluster to a standby cluster through the operator. Adding the standby section to the manifest of a running Postgres cluster will have no effect. However, it can be done through Patroni - by adding the [standby_cluster] (https://github.com/zalando/patroni/blob/bd2c54581abb42a7d3a3da551edf0b8732eefd27/docs/replica_bootstrap.rst#standby-cluster) + by adding the [standby_cluster](https://github.com/zalando/patroni/blob/bd2c54581abb42a7d3a3da551edf0b8732eefd27/docs/replica_bootstrap.rst#standby-cluster) section using `patronictl edit-config`. Note that the transformed standby cluster will not be doing any streaming. It will be in standby mode and allow read-only transactions only. @@ -384,7 +413,7 @@ specified but globally disabled in the configuration. The ## Increase volume size -PostgreSQL operator supports statefulset volume resize if you're using the +Postgres operator supports statefulset volume resize if you're using the operator on top of AWS. For that you need to change the size field of the volume description in the cluster manifest and apply the change: diff --git a/manifests/standby-manifest.yaml b/manifests/standby-manifest.yaml index 49f2b1a1f..e1bcaf104 100644 --- a/manifests/standby-manifest.yaml +++ b/manifests/standby-manifest.yaml @@ -9,7 +9,7 @@ spec: size: 1Gi numberOfInstances: 1 postgresql: - version: "10" + version: "11" # Make this a standby cluster and provide the s3 bucket path of source cluster for continuous streaming. standby: s3_wal_path: "s3://path/to/bucket/containing/wal/of/source/cluster/" diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 1f8fe203f..0a7377389 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -227,6 +227,10 @@ func (c *Cluster) Create() error { c.setStatus(acidv1.ClusterStatusCreating) + if err = c.validateResources(&c.Spec); err != nil { + return fmt.Errorf("insufficient resource limits specified: %v", err) + } + for _, role := range []PostgresRole{Master, Replica} { if c.Endpoints[role] != nil { @@ -491,6 +495,44 @@ func compareResourcesAssumeFirstNotNil(a *v1.ResourceRequirements, b *v1.Resourc } +func (c *Cluster) validateResources(spec *acidv1.PostgresSpec) error { + + // setting limits too low can cause unnecessary evictions / OOM kills + const ( + cpuMinLimit = "256m" + memoryMinLimit = "256Mi" + ) + + var ( + isSmaller bool + err error + ) + + cpuLimit := spec.Resources.ResourceLimits.CPU + if cpuLimit != "" { + isSmaller, err = util.IsSmallerQuantity(cpuLimit, cpuMinLimit) + if err != nil { + return fmt.Errorf("error validating CPU limit: %v", err) + } + if isSmaller { + return fmt.Errorf("defined CPU limit %s is below required minimum %s to properly run postgresql resource", cpuLimit, cpuMinLimit) + } + } + + memoryLimit := spec.Resources.ResourceLimits.Memory + if memoryLimit != "" { + isSmaller, err = util.IsSmallerQuantity(memoryLimit, memoryMinLimit) + if err != nil { + return fmt.Errorf("error validating memory limit: %v", err) + } + if isSmaller { + return fmt.Errorf("defined memory limit %s is below required minimum %s to properly run postgresql resource", memoryLimit, memoryMinLimit) + } + } + + return nil +} + // Update changes Kubernetes objects according to the new specification. Unlike the sync case, the missing object // (i.e. service) is treated as an error // logical backup cron jobs are an exception: a user-initiated Update can enable a logical backup job @@ -501,6 +543,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { c.mu.Lock() defer c.mu.Unlock() + oldStatus := c.Status c.setStatus(acidv1.ClusterStatusUpdating) c.setSpec(newSpec) @@ -512,6 +555,22 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } }() + if err := c.validateResources(&newSpec.Spec); err != nil { + err = fmt.Errorf("insufficient resource limits specified: %v", err) + + // cancel update only when (already too low) pod resources were edited + // if cluster was successfully running before the update, continue but log a warning + isCPULimitSmaller, err2 := util.IsSmallerQuantity(newSpec.Spec.Resources.ResourceLimits.CPU, oldSpec.Spec.Resources.ResourceLimits.CPU) + isMemoryLimitSmaller, err3 := util.IsSmallerQuantity(newSpec.Spec.Resources.ResourceLimits.Memory, oldSpec.Spec.Resources.ResourceLimits.Memory) + + if oldStatus.Running() && !isCPULimitSmaller && !isMemoryLimitSmaller && err2 == nil && err3 == nil { + c.logger.Warning(err) + } else { + updateFailed = true + return err + } + } + if oldSpec.Spec.PgVersion != newSpec.Spec.PgVersion { // PG versions comparison c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PgVersion, newSpec.Spec.PgVersion) //we need that hack to generate statefulset with the old version diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 73be712ca..c69c7a076 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -741,7 +741,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef limit = c.OpConfig.DefaultMemoryLimit } - isSmaller, err := util.RequestIsSmallerThanLimit(request, limit) + isSmaller, err := util.IsSmallerQuantity(request, limit) if err != nil { return nil, err } @@ -768,7 +768,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef limit = c.OpConfig.DefaultMemoryLimit } - isSmaller, err := util.RequestIsSmallerThanLimit(sidecarRequest, sidecarLimit) + isSmaller, err := util.IsSmallerQuantity(sidecarRequest, sidecarLimit) if err != nil { return nil, err } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index dd55cd04c..abe579fb5 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -23,6 +23,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { c.mu.Lock() defer c.mu.Unlock() + oldStatus := c.Status c.setSpec(newSpec) defer func() { @@ -34,6 +35,16 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } }() + if err = c.validateResources(&c.Spec); err != nil { + err = fmt.Errorf("insufficient resource limits specified: %v", err) + if oldStatus.Running() { + c.logger.Warning(err) + err = nil + } else { + return err + } + } + if err = c.initUsers(); err != nil { err = fmt.Errorf("could not init users: %v", err) return err diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 9db03ceb1..831078f3e 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -111,7 +111,7 @@ func (c *Controller) initOperatorConfig() { if c.opConfig.SetMemoryRequestToLimit { - isSmaller, err := util.RequestIsSmallerThanLimit(c.opConfig.DefaultMemoryRequest, c.opConfig.DefaultMemoryLimit) + isSmaller, err := util.IsSmallerQuantity(c.opConfig.DefaultMemoryRequest, c.opConfig.DefaultMemoryLimit) if err != nil { panic(err) } @@ -120,7 +120,7 @@ func (c *Controller) initOperatorConfig() { c.opConfig.DefaultMemoryRequest = c.opConfig.DefaultMemoryLimit } - isSmaller, err = util.RequestIsSmallerThanLimit(c.opConfig.ScalyrMemoryRequest, c.opConfig.ScalyrMemoryLimit) + isSmaller, err = util.IsSmallerQuantity(c.opConfig.ScalyrMemoryRequest, c.opConfig.ScalyrMemoryLimit) if err != nil { panic(err) } diff --git a/pkg/util/util.go b/pkg/util/util.go index a8ef460db..ad6de14a2 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -141,17 +141,17 @@ func Coalesce(val, defaultVal string) string { return val } -// RequestIsSmallerThanLimit : ... -func RequestIsSmallerThanLimit(requestStr, limitStr string) (bool, error) { +// IsSmallerQuantity : checks if first resource is of a smaller quantity than the second +func IsSmallerQuantity(requestStr, limitStr string) (bool, error) { request, err := resource.ParseQuantity(requestStr) if err != nil { - return false, fmt.Errorf("could not parse memory request %v : %v", requestStr, err) + return false, fmt.Errorf("could not parse request %v : %v", requestStr, err) } limit, err2 := resource.ParseQuantity(limitStr) if err2 != nil { - return false, fmt.Errorf("could not parse memory limit %v : %v", limitStr, err2) + return false, fmt.Errorf("could not parse limit %v : %v", limitStr, err2) } return request.Cmp(limit) == -1, nil diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index a34e57e23..1f86ea1b4 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -69,7 +69,7 @@ var substringMatch = []struct { {regexp.MustCompile(`aaaa (\d+) bbbb`), "aaaa 123 bbbb", nil}, } -var requestIsSmallerThanLimitTests = []struct { +var requestIsSmallerQuantityTests = []struct { request string limit string out bool @@ -155,14 +155,14 @@ func TestMapContains(t *testing.T) { } } -func TestRequestIsSmallerThanLimit(t *testing.T) { - for _, tt := range requestIsSmallerThanLimitTests { - res, err := RequestIsSmallerThanLimit(tt.request, tt.limit) +func TestIsSmallerQuantity(t *testing.T) { + for _, tt := range requestIsSmallerQuantityTests { + res, err := IsSmallerQuantity(tt.request, tt.limit) if err != nil { - t.Errorf("RequestIsSmallerThanLimit returned unexpected error: %#v", err) + t.Errorf("IsSmallerQuantity returned unexpected error: %#v", err) } if res != tt.out { - t.Errorf("RequestIsSmallerThanLimit expected: %#v, got: %#v", tt.out, res) + t.Errorf("IsSmallerQuantity expected: %#v, got: %#v", tt.out, res) } } } From bfe2e709a1f66ba44da8a9abab329e1c356241f8 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Thu, 12 Dec 2019 16:27:40 +0000 Subject: [PATCH 023/104] go.mod: fix dependencies (#754) Fix build issue: go/pkg/mod/k8s.io/client-go@v11.0.0+incompatible/rest/request.go:598:31: not enough arguments in call to watch.NewStreamWatcher --- Makefile | 3 ++- go.mod | 18 +++++++++------- go.sum | 64 +++++++++++++++++++------------------------------------- 3 files changed, 33 insertions(+), 52 deletions(-) diff --git a/Makefile b/Makefile index f42cfe09a..dc1c790fe 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,8 @@ scm-source.json: .git tools: GO111MODULE=on go get -u honnef.co/go/tools/cmd/staticcheck - GO111MODULE=on go get k8s.io/client-go@kubernetes-1.16.0 + GO111MODULE=on go get k8s.io/client-go@kubernetes-1.16.3 + GO111MODULE=on go mod tidy fmt: @gofmt -l -w -s $(DIRS) diff --git a/go.mod b/go.mod index b8c7b1615..36686dcf6 100644 --- a/go.mod +++ b/go.mod @@ -4,20 +4,22 @@ go 1.12 require ( github.com/aws/aws-sdk-go v1.25.44 + github.com/emicklei/go-restful v2.9.6+incompatible // indirect + github.com/evanphx/json-patch v4.5.0+incompatible // indirect + github.com/googleapis/gnostic v0.3.0 // indirect github.com/imdario/mergo v0.3.8 // indirect github.com/lib/pq v1.2.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/sirupsen/logrus v1.4.2 - golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e // indirect - golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 // indirect - golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 // indirect - golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d // indirect + golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect + golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect + golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect + golang.org/x/tools v0.0.0-20191209225234-22774f7dae43 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.2.4 k8s.io/api v0.0.0-20191121015604-11707872ac1c - k8s.io/apiextensions-apiserver v0.0.0-20191121021419-88daf26ec3b8 - k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a - k8s.io/client-go v11.0.0+incompatible + k8s.io/apiextensions-apiserver v0.0.0-20191204090421-cd61debedab5 + k8s.io/apimachinery v0.0.0-20191203211716-adc6f4cd9e7d + k8s.io/client-go v0.0.0-20191204082520-bc9b51d240b2 k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e - sigs.k8s.io/kind v0.5.1 // indirect ) diff --git a/go.sum b/go.sum index c7f7be037..f85dd060f 100644 --- a/go.sum +++ b/go.sum @@ -61,11 +61,9 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w= github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -138,7 +136,6 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -156,8 +153,6 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= @@ -174,7 +169,6 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -184,7 +178,6 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -209,7 +202,6 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= @@ -235,11 +227,9 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -264,21 +254,18 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -315,8 +302,8 @@ golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e h1:egKlR8l7Nu9vHGWbcUV8lqR4987UfUbBd7GbhqGzNYU= -golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 h1:I6A9Ag9FpEKOjcKrRNjQkPHawoXIhKyTGfvvjFAiiAk= @@ -344,8 +331,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 h1:e6HwijUxhDe+hPNjZQQn9bA5PW3vNmnN64U2ZW759Lk= -golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -369,17 +356,15 @@ golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190621203818-d432491b9138/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 h1:ZBzSG/7F4eNKz2L3GE9o300RX0Az1Bw5HF7PDraD+qU= -golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -398,8 +383,8 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d h1:/iIZNFGxc/a7C3yWjGcnboV+Tkc7mxr+p6fDztwoxuM= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191209225234-22774f7dae43 h1:NfPq5mgc5ArFgVLCpeS4z07IoxSAqVfV/gQ5vxdgaxI= +golang.org/x/tools v0.0.0-20191209225234-22774f7dae43/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 h1:OB/uP/Puiu5vS5QMRPrXCDWUPb+kt8f1KW8oQzFejQw= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= @@ -434,7 +419,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -442,33 +426,31 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= -k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20191121015604-11707872ac1c h1:Z87my3sF4WhG0OMxzARkWY/IKBtOr+MhXZAb4ts6qFc= k8s.io/api v0.0.0-20191121015604-11707872ac1c/go.mod h1:R/s4gKT0V/cWEnbQa9taNRJNbWUK57/Dx6cPj6MD3A0= -k8s.io/apiextensions-apiserver v0.0.0-20191121021419-88daf26ec3b8 h1:SrFLwOURsuwzuCi0zJdaBbPF31AcV9JUwpwIVosnnE4= -k8s.io/apiextensions-apiserver v0.0.0-20191121021419-88daf26ec3b8/go.mod h1:NMIy5Wa/or8CsLhYRleOp9CWAHVdcWpzT6Ufx1SNVjA= -k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a h1:9V03T5lHv/iF4fSgvMCd+iB86AgEgmzLpheMqIJy7hs= +k8s.io/apiextensions-apiserver v0.0.0-20191204090421-cd61debedab5 h1:g+GvnbGqLU1Jxb/9iFm/BFcmkqG9HdsGh52+wHirpsM= +k8s.io/apiextensions-apiserver v0.0.0-20191204090421-cd61debedab5/go.mod h1:CPw0IHz1YrWGy0+8mG/76oTHXvChlgCb3EAezKQKB2I= k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apiserver v0.0.0-20191121020624-6eed2f5a3289/go.mod h1:7P+0qMKoaggchirHLUSCVD22ohdkjN19+qQOKcAdfbI= -k8s.io/client-go v0.0.0-20191121015835-571c0ef67034 h1:+/ppGIi1rJThJAz/xJSSOuD82gb6E5jRv2305MSznxQ= -k8s.io/client-go v0.0.0-20191121015835-571c0ef67034/go.mod h1:Adhj+OyDRsEXTnL9BfL7xbLWGWMCqGLWpMqGHkZI4J8= -k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= -k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/apimachinery v0.0.0-20191123233150-4c4803ed55e3/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.0.0-20191128180518-03184f823e28/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.0.0-20191203211716-adc6f4cd9e7d h1:q+OZmYewHJeMCzwpHkXlNTtk5bvaUMPCikKvf77RBlo= +k8s.io/apimachinery v0.0.0-20191203211716-adc6f4cd9e7d/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apiserver v0.0.0-20191204084332-137a9d3b886b/go.mod h1:itgfam5HJbT/4b2BGfpUkkxfheMmDH+Ix+tEAP3uqZk= +k8s.io/client-go v0.0.0-20191204082517-8c19b9f4a642/go.mod h1:HMVIZ0dPop3WCrPEaJ+v5/94cjt56avdDFshpX0Fjvo= +k8s.io/client-go v0.0.0-20191204082519-e9644b2e3edc/go.mod h1:5lSG1yeDZVwDYAHe9VK48SCe5zmcnkAcf2Mx59TuhmM= +k8s.io/client-go v0.0.0-20191204082520-bc9b51d240b2 h1:T2HGghBOPAOEjWuIyFSeCsWEwsxa6unkBvy3PHfqonM= +k8s.io/client-go v0.0.0-20191204082520-bc9b51d240b2/go.mod h1:5lSG1yeDZVwDYAHe9VK48SCe5zmcnkAcf2Mx59TuhmM= k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e h1:HB9Zu5ZUvJfNpLiTPhz+CebVKV8C39qTBMQkAgAZLNw= k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= -k8s.io/component-base v0.0.0-20191121020327-771114ba3383/go.mod h1:tv9ITs6VEFWkF+kHwY4GiFvDr9vUGKJ4X/8+Z+oqVLk= +k8s.io/component-base v0.0.0-20191204083903-0d4d24e738e4/go.mod h1:8VIh1jErItC4bg9hLBkPneyS77Tin8KwSzbYepHJnQI= +k8s.io/component-base v0.0.0-20191204083906-3ac1376c73aa/go.mod h1:mECWvHCPhJudDVDMtBl+AIf/YnTMp5r1F947OYFUwP0= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM= k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= @@ -478,10 +460,6 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= -sigs.k8s.io/kind v0.5.1 h1:BYnHEJ9DC+0Yjlyyehqd3xnKtEmFdLKU8QxqOqvQzdw= -sigs.k8s.io/kind v0.5.1/go.mod h1:L+Kcoo83/D1+ryU5P2VFbvYm0oqbkJn9zTZq0KNxW68= -sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296 h1:iQaIG5Dq+3qSiaFrJ/l/0MjjxKmdwyVNpKRYJwUe/+0= -sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296/go.mod h1:ztX4zYc/QIww3gSripwF7TBOarBTm5BvyAMem0kCzOE= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= From 97e0d6d3888dc901aa3a089293ceb74972a7f5ae Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 12 Dec 2019 17:55:41 +0100 Subject: [PATCH 024/104] extend docs and polish manifest examples (#762) --- docs/administrator.md | 59 +++++++----- docs/developer.md | 12 +-- docs/index.md | 2 +- docs/quickstart.md | 3 + docs/reference/cluster_manifest.md | 14 +-- docs/reference/operator_parameters.md | 4 +- docs/user.md | 110 ++++++++++------------ manifests/complete-postgres-manifest.yaml | 2 +- manifests/minimal-postgres-manifest.yaml | 2 +- manifests/standby-manifest.yaml | 2 +- pkg/apis/acid.zalan.do/v1/util_test.go | 6 +- pkg/util/teams/teams_test.go | 4 +- 12 files changed, 111 insertions(+), 109 deletions(-) diff --git a/docs/administrator.md b/docs/administrator.md index 5cde06ade..5b8769edb 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -3,6 +3,26 @@ Learn how to configure and manage the Postgres Operator in your Kubernetes (K8s) environment. +## Minor and major version upgrade + +Minor version upgrades for PostgreSQL are handled via updating the Spilo Docker +image. The operator will carry out a rolling update of Pods which includes a +switchover (planned failover) of the master to the Pod with new minor version. +The switch should usually take less than 5 seconds, still clients have to +reconnect. + +Major version upgrades are supported via [cloning](user.md#clone-directly). The +new cluster manifest must have a higher `version` string than the source cluster +and will be created from a basebackup. Depending of the cluster size, downtime +in this case can be significant as writes to the database should be stopped and +all WAL files should be archived first before cloning is started. + +Note, that simply changing the version string in the `postgresql` manifest does +not work at present and leads to errors. Neither Patroni nor Postgres Operator +can do in place `pg_upgrade`. Still, it can be executed manually in the Postgres +container, which is tricky (i.e. systems need to be stopped, replicas have to be +synced) but of course faster than cloning. + ## CRD Validation [CustomResourceDefinitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions) @@ -95,8 +115,6 @@ is used by the operator to connect to the clusters after creation. ## Role-based access control for the operator -### Service account and cluster roles - The manifest [`operator-service-account-rbac.yaml`](../manifests/operator-service-account-rbac.yaml) defines the service account, cluster roles and bindings needed for the operator to function under access control restrictions. To deploy the operator with this @@ -109,6 +127,8 @@ kubectl create -f manifests/postgres-operator.yaml kubectl create -f manifests/minimal-postgres-manifest.yaml ``` +### Service account and cluster roles + Note that the service account is named `zalando-postgres-operator`. You may have to change the `service_account_name` in the operator ConfigMap and `serviceAccountName` in the `postgres-operator` deployment appropriately. This @@ -116,12 +136,6 @@ is done intentionally to avoid breaking those setups that already work with the default `operator` account. In the future the operator should ideally be run under the `zalando-postgres-operator` service account. -The service account defined in `operator-service-account-rbac.yaml` acquires -some privileges not used by the operator (i.e. we only need `list` and `watch` -on `configmaps` resources). This is also done intentionally to avoid breaking -things if someone decides to configure the same service account in the -operator's ConfigMap to run Postgres clusters. - ### Give K8s users access to create/list `postgresqls` By default `postgresql` custom resources can only be listed and changed by @@ -157,7 +171,6 @@ metadata: name: postgres-operator data: toleration: "key:postgres,operator:Exists,effect:NoSchedule" - ... ``` For an OperatorConfiguration resource the toleration should be defined like @@ -172,7 +185,6 @@ configuration: kubernetes: toleration: postgres: "key:postgres,operator:Exists,effect:NoSchedule" - ... ``` Note that the K8s version 1.13 brings [taint-based eviction](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#taint-based-evictions) @@ -250,7 +262,6 @@ metadata: name: postgres-operator data: inherited_labels: application,environment - ... ``` **OperatorConfiguration** @@ -265,7 +276,6 @@ configuration: inherited_labels: - application - environment -... ``` **cluster manifest** @@ -279,7 +289,7 @@ metadata: application: my-app environment: demo spec: -... + ... ``` **network policy** @@ -294,7 +304,6 @@ spec: matchLabels: application: my-app environment: demo -... ``` @@ -317,7 +326,6 @@ metadata: data: # referencing config map with custom settings pod_environment_configmap: postgres-pod-config - ... ``` **OperatorConfiguration** @@ -331,7 +339,6 @@ configuration: kubernetes: # referencing config map with custom settings pod_environment_configmap: postgres-pod-config - ... ``` **referenced ConfigMap `postgres-pod-config`** @@ -412,12 +419,12 @@ external systems but defined for an individual Postgres cluster in its manifest. A typical example is a role for connections from an application that uses the database. -* **Human users** originate from the Teams API that returns a list of the team -members given a team id. The operator differentiates between (a) product teams -that own a particular Postgres cluster and are granted admin rights to maintain -it, and (b) Postgres superuser teams that get the superuser access to all -Postgres databases running in a K8s cluster for the purposes of maintaining and -troubleshooting. +* **Human users** originate from the [Teams API](user.md#teams-api-roles) that +returns a list of the team members given a team id. The operator differentiates +between (a) product teams that own a particular Postgres cluster and are granted +admin rights to maintain it, and (b) Postgres superuser teams that get the +superuser access to all Postgres databases running in a K8s cluster for the +purposes of maintaining and troubleshooting. ## Understanding rolling update of Spilo pods @@ -481,7 +488,7 @@ A secret can be pre-provisioned in different ways: With the v1.2 release the Postgres Operator is shipped with a browser-based configuration user interface (UI) that simplifies managing Postgres clusters -with the operator. The UI runs with Node.js and comes with it's own docker +with the operator. The UI runs with Node.js and comes with it's own Docker image. Run NPM to continuously compile `tags/js` code. Basically, it creates an @@ -493,14 +500,14 @@ Run NPM to continuously compile `tags/js` code. Basically, it creates an To build the Docker image open a shell and change to the `ui` folder. Then run: -``` +```bash docker build -t registry.opensource.zalan.do/acid/postgres-operator-ui:v1.2.0 . ``` Apply all manifests for the `ui/manifests` folder to deploy the Postgres Operator UI on K8s. For local tests you don't need the Ingress resource. -``` +```bash kubectl apply -f ui/manifests ``` @@ -510,6 +517,6 @@ to the K8s and Postgres Operator REST API. You can use the provided `run_local.sh` script for this. Make sure it uses the correct URL to your K8s API server, e.g. for minikube it would be `https://192.168.99.100:8443`. -``` +```bash ./run_local.sh ``` diff --git a/docs/developer.md b/docs/developer.md index f8351e28a..6e0fc33c8 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -40,7 +40,7 @@ This would take a while to complete. You have to redo `make deps` every time your dependencies list changes, i.e. after adding a new library dependency. Build the operator with the `make docker` command. You may define the TAG -variable to assign an explicit tag to your docker image and the IMAGE to set +variable to assign an explicit tag to your Docker image and the IMAGE to set the image name. By default, the tag is computed with `git describe --tags --always --dirty` and the image is `registry.opensource.zalan.do/acid/postgres-operator` @@ -60,10 +60,10 @@ The binary will be placed into the build directory. ## Deploying self build image -The fastest way to run and test your docker image locally is to reuse the docker -from [minikube](https://github.com/kubernetes/minikube/releases) or use the -`load docker-image` from [kind](https://kind.sigs.k8s.io/). The following steps -will get you the docker image built and deployed. +The fastest way to run and test your Docker image locally is to reuse the Docker +environment from [minikube](https://github.com/kubernetes/minikube/releases) +or use the `load docker-image` from [kind](https://kind.sigs.k8s.io/). The +following steps will get you the Docker image built and deployed. ```bash # minikube @@ -162,7 +162,7 @@ The operator also supports pprof endpoints listed at the * /debug/pprof/trace It's possible to attach a debugger to troubleshoot postgres-operator inside a -docker container. It's possible with [gdb](https://www.gnu.org/software/gdb/) +Docker container. It's possible with [gdb](https://www.gnu.org/software/gdb/) and [delve](https://github.com/derekparker/delve). Since the latter one is a specialized debugger for Go, we will use it as an example. To use it you need: diff --git a/docs/index.md b/docs/index.md index c0e78ac32..87b08deb2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,7 +13,7 @@ manages PostgreSQL clusters on Kubernetes (K8s): 2. The operator also watches updates to [its own configuration](../manifests/configmap.yaml) and alters running Postgres clusters if necessary. For instance, if the - docker image in a pod is changed, the operator carries out the rolling + Docker image in a pod is changed, the operator carries out the rolling update, which means it re-spawns pods of each managed StatefulSet one-by-one with the new Docker image. diff --git a/docs/quickstart.md b/docs/quickstart.md index 2da2cab7c..8cc5bc0c0 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -155,9 +155,12 @@ export PGPORT=$(echo $HOST_PORT | cut -d: -f 2) ``` Retrieve the password from the K8s Secret that is created in your cluster. +Non-encrypted connections are rejected by default, so set the SSL mode to +require: ```bash export PGPASSWORD=$(kubectl get secret postgres.acid-minimal-cluster.credentials -o 'jsonpath={.data.password}' | base64 -d) +export PGSSLMODE=require psql -U postgres ``` diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index cf522d73d..bf6df681b 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -62,7 +62,7 @@ These parameters are grouped directly under the `spec` key in the manifest. field. * **dockerImage** - custom docker image that overrides the **docker_image** operator parameter. + custom Docker image that overrides the **docker_image** operator parameter. It should be a [Spilo](https://github.com/zalando/spilo) image. Optional. * **spiloFSGroup** @@ -124,7 +124,7 @@ These parameters are grouped directly under the `spec` key in the manifest. * **enableShmVolume** - Start a database pod without limitations on shm memory. By default docker + Start a database pod without limitations on shm memory. By default Docker limit `/dev/shm` to `64M` (see e.g. the [docker issue](https://github.com/docker-library/postgres/issues/416), which could be not enough if PostgreSQL uses parallel workers heavily. If this option is @@ -185,19 +185,19 @@ explanation of `ttl` and `loop_wait` parameters. * **ttl** Patroni `ttl` parameter value, optional. The default is set by the Spilo - docker image. Optional. + Docker image. Optional. * **loop_wait** Patroni `loop_wait` parameter value, optional. The default is set by the - Spilo docker image. Optional. + Spilo Docker image. Optional. * **retry_timeout** Patroni `retry_timeout` parameter value, optional. The default is set by the - Spilo docker image. Optional. + Spilo Docker image. Optional. * **maximum_lag_on_failover** Patroni `maximum_lag_on_failover` parameter value, optional. The default is - set by the Spilo docker image. Optional. + set by the Spilo Docker image. Optional. * **slots** permanent replication slots that Patroni preserves after failover by @@ -320,7 +320,7 @@ defined in the sidecar dictionary: name of the sidecar. Required. * **image** - docker image of the sidecar. Required. + Docker image of the sidecar. Required. * **env** a dictionary of environment variables. Use usual Kubernetes definition diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 3de4a5be4..1055d89b6 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -81,13 +81,13 @@ Those are top-level keys, containing both leaf keys and groups. Kubernetes-native DCS). * **docker_image** - Spilo docker image for Postgres instances. For production, don't rely on the + Spilo Docker image for Postgres instances. For production, don't rely on the default image, as it might be not the most up-to-date one. Instead, build your own Spilo image from the [github repository](https://github.com/zalando/spilo). * **sidecar_docker_images** - a map of sidecar names to docker images to run with Spilo. In case of the name + a map of sidecar names to Docker images to run with Spilo. In case of the name conflict with the definition in the cluster manifest the cluster-specific one is preferred. diff --git a/docs/user.md b/docs/user.md index d9e92d5d6..45f345c87 100644 --- a/docs/user.md +++ b/docs/user.md @@ -13,7 +13,7 @@ kind: postgresql metadata: name: acid-minimal-cluster spec: - teamId: "ACID" + teamId: "acid" volume: size: 1Gi numberOfInstances: 2 @@ -40,8 +40,16 @@ you can find this example also in the manifests folder: kubectl create -f manifests/minimal-postgres-manifest.yaml ``` -Note, that the minimum volume size to run the `postgresql` resource on Elastic -Block Storage (EBS) is `1Gi`. +Make sure, the `spec` section of the manifest contains at least a `teamId`, the +`numberOfInstances` and the `postgresql` object with the `version` specified. +The minimum volume size to run the `postgresql` resource on Elastic Block +Storage (EBS) is `1Gi`. + +Note, that the name of the cluster must start with the `teamId` and `-`. At +Zalando we use team IDs (nicknames) to lower the chance of duplicate cluster +names and colliding entities. The team ID would also be used to query an API to +get all members of a team and create [database roles](#teams-api-roles) for +them. ## Watch pods being created @@ -65,10 +73,12 @@ kubectl port-forward $PGMASTER 6432:5432 Open another CLI and connect to the database. Use the generated secret of the `postgres` robot user to connect to our `acid-minimal-cluster` master running -in Minikube: +in Minikube. As non-encrypted connections are rejected by default set the SSL +mode to require: ```bash export PGPASSWORD=$(kubectl get secret postgres.acid-minimal-cluster.credentials -o 'jsonpath={.data.password}' | base64 -d) +export PGSSLMODE=require psql -U postgres -p 6432 ``` @@ -80,8 +90,7 @@ cluster. It covers three use-cases: * `manifest roles`: create application roles specific to the cluster described in the manifest. * `infrastructure roles`: create application roles that should be automatically -created on every - cluster managed by the operator. +created on every cluster managed by the operator. * `teams API roles`: automatically create users for every member of the team owning the database cluster. @@ -131,9 +140,9 @@ The infrastructure roles secret is specified by the `infrastructure_roles_secret parameter. The role definition looks like this (values are base64 encoded): ```yaml - user1: ZGJ1c2Vy - password1: c2VjcmV0 - inrole1: b3BlcmF0b3I= +user1: ZGJ1c2Vy +password1: c2VjcmV0 +inrole1: b3BlcmF0b3I= ``` The block above describes the infrastructure role 'dbuser' with password @@ -154,19 +163,19 @@ secret and a ConfigMap. The ConfigMap must have the same name as the secret. The secret should contain an entry with 'rolename:rolepassword' for each role. ```yaml - dbuser: c2VjcmV0 +dbuser: c2VjcmV0 ``` And the role description for that user should be specified in the ConfigMap. ```yaml - data: - dbuser: | - inrole: [operator, admin] # following roles will be assigned to the new user - user_flags: - - createdb - db_parameters: # db parameters, applied for this particular user - log_statement: all +data: + dbuser: | + inrole: [operator, admin] # following roles will be assigned to the new user + user_flags: + - createdb + db_parameters: # db parameters, applied for this particular user + log_statement: all ``` One can allow membership in multiple roles via the `inrole` array parameter, @@ -185,16 +194,34 @@ See [infrastructure roles secret](../manifests/infrastructure-roles.yaml) and [infrastructure roles configmap](../manifests/infrastructure-roles-configmap.yaml) for the examples. +### Teams API roles + +These roles are meant for database activity of human users. It's possible to +configure the operator to automatically create database roles for lets say all +employees of one team. They are not listed in the manifest and there are no K8s +secrets created for them. Instead they would use an OAuth2 token to connect. To +get all members of the team the operator queries a defined API endpoint that +returns usernames. A minimal Teams API should work like this: + +``` +/.../ -> ["name","anothername"] +``` + +A ["fake" Teams API](../manifests/fake-teams-api.yaml) deployment is provided +in the manifests folder to set up a basic API around whatever services is used +for user management. The Teams API's URL is set in the operator's +[configuration](reference/operator_parameters.md#automatic-creation-of-human-users-in-the-database) +and `enable_teams_api` must be set to `true`. There are more settings available +to choose superusers, group roles, [PAM configuration](https://github.com/CyberDem0n/pam-oauth2) +etc. An OAuth2 token can be passed to the Teams API via a secret. The name for +this secret is configurable with the `oauth_token_secret_name` parameter. + ## Resource definition The compute resources to be used for the Postgres containers in the pods can be specified in the postgresql cluster manifest. ```yaml -apiVersion: "acid.zalan.do/v1" -kind: postgresql -metadata: - name: acid-minimal-cluster spec: resources: requests: @@ -218,12 +245,7 @@ you can use [taints and tolerations](https://kubernetes.io/docs/concepts/configu and configure the required toleration in the manifest. ```yaml -apiVersion: "acid.zalan.do/v1" -kind: postgresql -metadata: - name: acid-minimal-cluster spec: - teamId: "ACID" tolerations: - key: postgres operator: Exists @@ -241,11 +263,6 @@ section in the spec. There are two options here: ### Clone directly ```yaml -apiVersion: "acid.zalan.do/v1" -kind: postgresql - -metadata: - name: acid-test-cluster spec: clone: cluster: "acid-batman" @@ -261,11 +278,6 @@ means that you can clone only from clusters within the same namespace. ### Clone from S3 ```yaml -apiVersion: "acid.zalan.do/v1" -kind: postgresql - -metadata: - name: acid-test-cluster spec: clone: uid: "efd12e58-5786-11e8-b5a7-06148230260c" @@ -294,10 +306,6 @@ For non AWS S3 following settings can be set to support cloning from other S3 implementations: ```yaml -apiVersion: "acid.zalan.do/v1" -kind: postgresql -metadata: - name: acid-test-cluster spec: clone: uid: "efd12e58-5786-11e8-b5a7-06148230260c" @@ -346,13 +354,7 @@ used for log aggregation, monitoring, backups or other tasks. A sidecar can be specified like this: ```yaml -apiVersion: "acid.zalan.do/v1" -kind: postgresql - -metadata: - name: acid-minimal-cluster spec: - ... sidecars: - name: "container-name" image: "company/image:tag" @@ -390,13 +392,7 @@ be used to run custom actions before any normal and sidecar containers start. An init container can be specified like this: ```yaml -apiVersion: "acid.zalan.do/v1" -kind: postgresql - -metadata: - name: acid-minimal-cluster spec: - ... initContainers: - name: "container-name" image: "company/image:tag" @@ -417,12 +413,7 @@ Postgres operator supports statefulset volume resize if you're using the operator on top of AWS. For that you need to change the size field of the volume description in the cluster manifest and apply the change: -``` -apiVersion: "acid.zalan.do/v1" -kind: postgresql - -metadata: - name: acid-test-cluster +```yaml spec: volume: size: 5Gi # new volume size @@ -451,7 +442,8 @@ size of volumes that correspond to the previously running pods is not changed. You can enable logical backups from the cluster manifest by adding the following parameter in the spec section: -``` +```yaml +spec: enableLogicalBackup: true ``` diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index f8495caad..ba2315753 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -10,7 +10,7 @@ spec: - name: date image: busybox command: [ "/bin/date" ] - teamId: "ACID" + teamId: "acid" volume: size: 1Gi # storageClass: my-sc diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml index 91d297cac..75dfdf07f 100644 --- a/manifests/minimal-postgres-manifest.yaml +++ b/manifests/minimal-postgres-manifest.yaml @@ -4,7 +4,7 @@ metadata: name: acid-minimal-cluster namespace: default spec: - teamId: "ACID" + teamId: "acid" volume: size: 1Gi numberOfInstances: 2 diff --git a/manifests/standby-manifest.yaml b/manifests/standby-manifest.yaml index e1bcaf104..e5299bc9b 100644 --- a/manifests/standby-manifest.yaml +++ b/manifests/standby-manifest.yaml @@ -4,7 +4,7 @@ metadata: name: acid-standby-cluster namespace: default spec: - teamId: "ACID" + teamId: "acid" volume: size: 1Gi numberOfInstances: 1 diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index cf3b080a5..fc068b322 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -180,7 +180,7 @@ var unmarshalCluster = []struct { "name": "acid-testcluster1" }, "spec": { - "teamId": "ACID", + "teamId": "acid", "pod_priority_class_name": "spilo-pod-priority", "volume": { "size": "5Gi", @@ -290,7 +290,7 @@ var unmarshalCluster = []struct { ResourceLimits: ResourceDescription{CPU: "300m", Memory: "3000Mi"}, }, - TeamID: "ACID", + TeamID: "acid", AllowedSourceRanges: []string{"127.0.0.1/32"}, NumberOfInstances: 2, Users: map[string]UserFlags{"zalando": {"superuser", "createdb"}}, @@ -319,7 +319,7 @@ var unmarshalCluster = []struct { }, Error: "", }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"9.6","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"ACID","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"9.6","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, // example with teamId set in input { diff --git a/pkg/util/teams/teams_test.go b/pkg/util/teams/teams_test.go index 637c4e16c..51bbcbc31 100644 --- a/pkg/util/teams/teams_test.go +++ b/pkg/util/teams/teams_test.go @@ -24,7 +24,7 @@ var teamsAPItc = []struct { {`{ "dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", "id": "acid", -"id_name": "ACID", +"id_name": "acid", "team_id": "111222", "type": "official", "name": "Acid team name", @@ -70,7 +70,7 @@ var teamsAPItc = []struct { &Team{ Dn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", ID: "acid", - TeamName: "ACID", + TeamName: "acid", TeamID: "111222", Type: "official", FullName: "Acid team name", From 629feac98f275f01e6cdbfa2ed350cb6ef1f4a20 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 16 Dec 2019 17:07:36 +0100 Subject: [PATCH 025/104] Remove bind verb and explain privileges (#765) Closes #256 --- manifests/operator-service-account-rbac.yaml | 28 +++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index e95fe320b..a37abe476 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -10,6 +10,7 @@ kind: ClusterRole metadata: name: zalando-postgres-operator rules: +# all verbs allowed for custom operator resources - apiGroups: - acid.zalan.do resources: @@ -18,6 +19,7 @@ rules: - operatorconfigurations verbs: - "*" +# to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io resources: @@ -27,12 +29,14 @@ rules: - get - patch - update +# to read configuration from ConfigMaps - apiGroups: - "" resources: - configmaps verbs: - get +# to manage endpoints which are also used by Patroni - apiGroups: - "" resources: @@ -45,6 +49,7 @@ rules: - list - patch - watch # needed if zalando-postgres-operator account is used for pods as well +# to CRUD secrets for database access - apiGroups: - "" resources: @@ -54,6 +59,7 @@ rules: - update - delete - get +# to check nodes for node readiness label - apiGroups: - "" resources: @@ -62,6 +68,7 @@ rules: - get - list - watch +# to read or delete existing PVCs. Creation via StatefulSet - apiGroups: - "" resources: @@ -70,6 +77,7 @@ rules: - delete - get - list + # to read existing PVs. Creation should be done via dynamic provisioning - apiGroups: - "" resources: @@ -78,6 +86,7 @@ rules: - get - list - update # only for resizing AWS volumes +# to watch Spilo pods and do rolling updates. Creation via StatefulSet - apiGroups: - "" resources: @@ -88,12 +97,14 @@ rules: - list - watch - patch +# to resize the filesystem in Spilo pods when increasing volume size - apiGroups: - "" resources: - pods/exec verbs: - create +# to CRUD services to point to Postgres cluster instances - apiGroups: - "" resources: @@ -103,6 +114,7 @@ rules: - delete - get - patch +# to CRUD the StatefulSet which controls the Postgres cluster instances - apiGroups: - apps resources: @@ -113,12 +125,14 @@ rules: - get - list - patch +# to get namespaces operator resources can run in - apiGroups: - "" resources: - namespaces verbs: - get +# to define PDBs. Update happens via delete/create - apiGroups: - policy resources: @@ -127,6 +141,7 @@ rules: - create - delete - get +# to create ServiceAccounts in each namespace the operator watches - apiGroups: - "" resources: @@ -134,6 +149,7 @@ rules: verbs: - get - create +# to create role bindings to the operator service account - apiGroups: - "rbac.authorization.k8s.io" resources: @@ -141,18 +157,11 @@ rules: verbs: - get - create -- apiGroups: - - "rbac.authorization.k8s.io" - resources: - - clusterroles - verbs: - - bind - resourceNames: - - zalando-postgres-operator +# to CRUD cron jobs for logical backups - apiGroups: - batch resources: - - cronjobs # enables logical backups + - cronjobs verbs: - create - delete @@ -160,6 +169,7 @@ rules: - list - patch - update + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding From 182e3bc7db0fd6c7b2617286c324167c7f0c158e Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 16 Dec 2019 17:08:09 +0100 Subject: [PATCH 026/104] add missing fields to OperatorConfiguration CRD validation (#767) --- .../crds/operatorconfigurations.yaml | 30 +++++++----- manifests/complete-postgres-manifest.yaml | 35 ++++++++------ manifests/operatorconfiguration.crd.yaml | 30 +++++++----- ...gresql-operator-default-configuration.yaml | 12 +++-- manifests/standby-manifest.yaml | 4 -- pkg/apis/acid.zalan.do/v1/crds.go | 46 ++++++++++++------- .../v1/operator_configuration_type.go | 32 +++++-------- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 28 ----------- pkg/util/config/config.go | 2 +- 9 files changed, 109 insertions(+), 110 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index d50a2b431..c97e246ab 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -121,6 +121,8 @@ spec: type: array items: type: string + master_pod_move_timeout: + type: string node_readiness_label: type: object additionalProperties: @@ -138,10 +140,16 @@ spec: enum: - "ordered_ready" - "parallel" + pod_priority_class_name: + type: string pod_role_label: type: string + pod_service_account_definition: + type: string pod_service_account_name: type: string + pod_service_account_role_binding_definition: + type: string pod_terminate_grace_period: type: string secret_name_template: @@ -189,16 +197,16 @@ spec: load_balancer: type: object properties: + custom_service_annotations: + type: object + additionalProperties: + type: string db_hosted_zone: type: string enable_master_load_balancer: type: boolean enable_replica_load_balancer: type: boolean - custom_service_annotations: - type: object - additionalProperties: - type: string master_dns_name_format: type: string replica_dns_name_format: @@ -221,21 +229,21 @@ spec: logical_backup: type: object properties: - logical_backup_schedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' logical_backup_docker_image: type: string + logical_backup_s3_access_key_id: + type: string logical_backup_s3_bucket: type: string logical_backup_s3_endpoint: type: string - logical_backup_s3_sse: - type: string - logical_backup_s3_access_key_id: - type: string logical_backup_s3_secret_access_key: type: string + logical_backup_s3_sse: + type: string + logical_backup_schedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' debug: type: object properties: diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index ba2315753..23dd40638 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -6,10 +6,6 @@ metadata: # environment: demo spec: dockerImage: registry.opensource.zalan.do/acid/spilo-11:1.6-p1 - initContainers: - - name: date - image: busybox - command: [ "/bin/date" ] teamId: "acid" volume: size: 1Gi @@ -25,18 +21,22 @@ spec: - 127.0.0.1/32 databases: foo: zalando -# podAnnotations: -# annotation.key: value -# Expert section - - enableShmVolume: true -# spiloFSGroup: 103 postgresql: version: "11" - parameters: + parameters: # Expert section shared_buffers: "32MB" max_connections: "10" log_statement: "all" + + enableShmVolume: true +# spiloFSGroup: 103 +# podAnnotations: +# annotation.key: value +# podPriorityClassName: "spilo-pod-priority" +# tolerations: +# - key: postgres +# operator: Exists +# effect: NoSchedule resources: requests: cpu: 10m @@ -63,6 +63,7 @@ spec: loop_wait: &loop_wait 10 retry_timeout: 10 maximum_lag_on_failover: 33554432 + # restore a Postgres DB with point-in-time-recovery # with a non-empty timestamp, clone from an S3 bucket using the latest backup before the timestamp # with an empty/absent timestamp, clone from an existing alive cluster using pg_basebackup @@ -75,9 +76,15 @@ spec: # run periodic backups with k8s cron jobs # enableLogicalBackup: true # logicalBackupSchedule: "30 00 * * *" - maintenanceWindows: - - 01:00-06:00 #UTC - - Sat:00:00-04:00 + +# maintenanceWindows: +# - 01:00-06:00 #UTC +# - Sat:00:00-04:00 + + initContainers: + - name: date + image: busybox + command: [ "/bin/date" ] # sidecars: # - name: "telegraf-sidecar" # image: "telegraf:latest" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index bed892dc8..810624bc4 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -97,6 +97,8 @@ spec: type: array items: type: string + master_pod_move_timeout: + type: string node_readiness_label: type: object additionalProperties: @@ -114,10 +116,16 @@ spec: enum: - "ordered_ready" - "parallel" + pod_priority_class_name: + type: string pod_role_label: type: string + pod_service_account_definition: + type: string pod_service_account_name: type: string + pod_service_account_role_binding_definition: + type: string pod_terminate_grace_period: type: string secret_name_template: @@ -165,16 +173,16 @@ spec: load_balancer: type: object properties: + custom_service_annotations: + type: object + additionalProperties: + type: string db_hosted_zone: type: string enable_master_load_balancer: type: boolean enable_replica_load_balancer: type: boolean - custom_service_annotations: - type: object - additionalProperties: - type: string master_dns_name_format: type: string replica_dns_name_format: @@ -197,21 +205,21 @@ spec: logical_backup: type: object properties: - logical_backup_schedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' logical_backup_docker_image: type: string + logical_backup_s3_access_key_id: + type: string logical_backup_s3_bucket: type: string logical_backup_s3_endpoint: type: string - logical_backup_s3_sse: - type: string - logical_backup_s3_access_key_id: - type: string logical_backup_s3_secret_access_key: type: string + logical_backup_s3_sse: + type: string + logical_backup_schedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' debug: type: object properties: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 84e12b4ee..cdfe0f573 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -21,7 +21,7 @@ configuration: kubernetes: cluster_domain: cluster.local cluster_labels: - application: spilo + application: spilo cluster_name_label: cluster-name # custom_pod_annotations: # keya: valuea @@ -34,6 +34,7 @@ configuration: # inherited_labels: # - application # - environment + master_pod_move_timeout: 20m # node_readiness_label: # status: ready oauth_token_secret_name: postgresql-operator @@ -41,8 +42,11 @@ configuration: pod_antiaffinity_topology_key: "kubernetes.io/hostname" # pod_environment_configmap: "" pod_management_policy: "ordered_ready" + # pod_priority_class_name: "" pod_role_label: spilo-role + # pod_service_account_definition: "" pod_service_account_name: zalando-postgres-operator + # pod_service_account_role_binding_definition: "" pod_terminate_grace_period: 5m secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" # spilo_fsgroup: 103 @@ -79,10 +83,10 @@ configuration: # wal_s3_bucket: "" logical_backup: logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" - logical_backup_s3_access_key_id: "" + # logical_backup_s3_access_key_id: "" logical_backup_s3_bucket: "my-bucket-url" - logical_backup_s3_endpoint: "" - logical_backup_s3_secret_access_key: "" + # logical_backup_s3_endpoint: "" + # logical_backup_s3_secret_access_key: "" logical_backup_s3_sse: "AES256" logical_backup_schedule: "30 00 * * *" debug: diff --git a/manifests/standby-manifest.yaml b/manifests/standby-manifest.yaml index e5299bc9b..2b621bd10 100644 --- a/manifests/standby-manifest.yaml +++ b/manifests/standby-manifest.yaml @@ -13,7 +13,3 @@ spec: # Make this a standby cluster and provide the s3 bucket path of source cluster for continuous streaming. standby: s3_wal_path: "s3://path/to/bucket/containing/wal/of/source/cluster/" - - maintenanceWindows: - - 01:00-06:00 #UTC - - Sat:00:00-04:00 diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 75704afde..20fa37138 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -717,6 +717,9 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "master_pod_move_timeout": { + Type: "string", + }, "node_readiness_label": { Type: "object", AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ @@ -748,12 +751,21 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "pod_priority_class_name": { + Type: "string", + }, "pod_role_label": { Type: "string", }, + "pod_service_account_definition": { + Type: "string", + }, "pod_service_account_name": { Type: "string", }, + "pod_service_account_role_binding_definition": { + Type: "string", + }, "pod_terminate_grace_period": { Type: "string", }, @@ -826,6 +838,14 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "load_balancer": { Type: "object", Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "custom_service_annotations": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, "db_hosted_zone": { Type: "string", }, @@ -835,14 +855,6 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "enable_replica_load_balancer": { Type: "boolean", }, - "custom_service_annotations": { - Type: "object", - AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ - Schema: &apiextv1beta1.JSONSchemaProps{ - Type: "string", - }, - }, - }, "master_dns_name_format": { Type: "string", }, @@ -877,27 +889,27 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "logical_backup": { Type: "object", Properties: map[string]apiextv1beta1.JSONSchemaProps{ - "logical_backup_schedule": { - Type: "string", - Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$", - }, "logical_backup_docker_image": { Type: "string", }, + "logical_backup_s3_access_key_id": { + Type: "string", + }, "logical_backup_s3_bucket": { Type: "string", }, "logical_backup_s3_endpoint": { Type: "string", }, + "logical_backup_s3_secret_access_key": { + Type: "string", + }, "logical_backup_s3_sse": { Type: "string", }, - "logical_backup_s3_access_key_id": { - Type: "string", - }, - "logical_backup_s3_secret_access_key": { - Type: "string", + "logical_backup_schedule": { + Type: "string", + Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$", }, }, }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index f76790ad5..948c7cbbf 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -118,7 +118,7 @@ type OperatorDebugConfiguration struct { EnableDBAccess bool `json:"enable_database_access,omitempty"` } -// TeamsAPIConfiguration defines the configration of TeamsAPI +// TeamsAPIConfiguration defines the configuration of TeamsAPI type TeamsAPIConfiguration struct { EnableTeamsAPI bool `json:"enable_teams_api,omitempty"` TeamsAPIUrl string `json:"teams_api_url,omitempty"` @@ -150,6 +150,17 @@ type ScalyrConfiguration struct { ScalyrMemoryLimit string `json:"scalyr_memory_limit,omitempty"` } +// OperatorLogicalBackupConfiguration defines configuration for logical backup +type OperatorLogicalBackupConfiguration struct { + Schedule string `json:"logical_backup_schedule,omitempty"` + DockerImage string `json:"logical_backup_docker_image,omitempty"` + S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` + S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"` + S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` + S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"` + S3SSE string `json:"logical_backup_s3_sse,omitempty"` +} + // OperatorConfigurationData defines the operation config type OperatorConfigurationData struct { EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` @@ -176,24 +187,5 @@ type OperatorConfigurationData struct { LogicalBackup OperatorLogicalBackupConfiguration `json:"logical_backup"` } -// OperatorConfigurationUsers defines configration for super user -type OperatorConfigurationUsers struct { - SuperUserName string `json:"superuser_name,omitempty"` - Replication string `json:"replication_user_name,omitempty"` - ProtectedRoles []string `json:"protected_roles,omitempty"` - TeamAPIRoleConfiguration map[string]string `json:"team_api_role_configuration,omitempty"` -} - //Duration shortens this frequently used name type Duration time.Duration - -// OperatorLogicalBackupConfiguration defines configration for logical backup -type OperatorLogicalBackupConfiguration struct { - Schedule string `json:"logical_backup_schedule,omitempty"` - DockerImage string `json:"logical_backup_docker_image,omitempty"` - S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` - S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"` - S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` - S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"` - S3SSE string `json:"logical_backup_s3_sse,omitempty"` -} diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 16f5a9d67..b68a72d1f 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -300,34 +300,6 @@ func (in *OperatorConfigurationList) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OperatorConfigurationUsers) DeepCopyInto(out *OperatorConfigurationUsers) { - *out = *in - if in.ProtectedRoles != nil { - in, out := &in.ProtectedRoles, &out.ProtectedRoles - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.TeamAPIRoleConfiguration != nil { - in, out := &in.TeamAPIRoleConfiguration, &out.TeamAPIRoleConfiguration - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorConfigurationUsers. -func (in *OperatorConfigurationUsers) DeepCopy() *OperatorConfigurationUsers { - if in == nil { - return nil - } - out := new(OperatorConfigurationUsers) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorDebugConfiguration) DeepCopyInto(out *OperatorDebugConfiguration) { *out = *in diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index d46cba2b2..b2a135fad 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -69,7 +69,7 @@ type Scalyr struct { ScalyrMemoryLimit string `name:"scalyr_memory_limit" default:"1Gi"` } -// LogicalBackup defines configration for logical backup +// LogicalBackup defines configuration for logical backup type LogicalBackup struct { LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"` From 7af1de890cbadb02e92d167492d9a02a45ab0a14 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 17 Dec 2019 17:13:56 +0100 Subject: [PATCH 027/104] bump operator v1.3.0 with Spilo 12 image (#770) --- charts/postgres-operator/Chart.yaml | 6 ++-- charts/postgres-operator/index.yaml | 27 ++++++++++++++++-- .../postgres-operator-1.3.0.tgz | Bin 0 -> 19063 bytes charts/postgres-operator/values-crd.yaml | 4 +-- charts/postgres-operator/values.yaml | 4 +-- manifests/complete-postgres-manifest.yaml | 2 +- manifests/configmap.yaml | 2 +- manifests/postgres-operator.yaml | 2 +- ...gresql-operator-default-configuration.yaml | 2 +- pkg/util/config/config.go | 2 +- 10 files changed, 35 insertions(+), 16 deletions(-) create mode 100644 charts/postgres-operator/postgres-operator-1.3.0.tgz diff --git a/charts/postgres-operator/Chart.yaml b/charts/postgres-operator/Chart.yaml index ae9bb855e..08e242a53 100644 --- a/charts/postgres-operator/Chart.yaml +++ b/charts/postgres-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: postgres-operator -version: 1.2.0 -appVersion: 1.2.0 +version: 1.3.0 +appVersion: 1.3.0 home: https://github.com/zalando/postgres-operator description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes keywords: @@ -13,8 +13,6 @@ keywords: maintainers: - name: Zalando email: opensource@zalando.de -- name: kimxogus - email: kgyoo8232@gmail.com sources: - https://github.com/zalando/postgres-operator engine: gotpl diff --git a/charts/postgres-operator/index.yaml b/charts/postgres-operator/index.yaml index b549f1220..84502f6a6 100644 --- a/charts/postgres-operator/index.yaml +++ b/charts/postgres-operator/index.yaml @@ -1,13 +1,34 @@ apiVersion: v1 entries: postgres-operator: + - apiVersion: v1 + appVersion: 1.3.0 + created: "2019-12-17T12:58:49.477140129+01:00" + description: Postgres Operator creates and manages PostgreSQL clusters running + in Kubernetes + digest: 7e788fd37daec76a01f6d6f9fe5be5b54f5035e4eba0041e80a760d656537325 + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + name: postgres-operator + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-1.3.0.tgz + version: 1.3.0 - apiVersion: v1 appVersion: 1.2.0 - created: "2019-08-13T17:33:32.735021423+02:00" + created: "2019-12-17T12:58:49.475844233+01:00" description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes digest: d10710c7cf19f4e266e7704f5d1e98dcfc61bee3919522326c35c22ca7d2f2bf - engine: gotpl home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -26,4 +47,4 @@ entries: urls: - postgres-operator-1.2.0.tgz version: 1.2.0 -generated: "2019-08-13T17:33:32.734335398+02:00" +generated: "2019-12-17T12:58:49.474719294+01:00" diff --git a/charts/postgres-operator/postgres-operator-1.3.0.tgz b/charts/postgres-operator/postgres-operator-1.3.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..460fed53286e21075290ed1d7cdcc1778100f95c GIT binary patch literal 19063 zcmV)XK&`(YiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH;nd=y3ZD4=p62ndL@A)$njyX107AV)JK0Rp6ugc|VJyPdmR zvbTHe?j<=A2wgfz?;usAO9zoIO`3=(y*DWWf(SzXzn$H^+iM{S$oGA}KdV2z?9RM- zGxO%ndvD&nXQf%r#6VU_TOosSG!qNKO{ok z$nhJ!SBFJt!i#~B*M&fSkB#LprdY@;x8>fs|B(l?5}hE!5;Ua+8EOS)wL1P(t2C+* z1rAvQLs&WVDoLakkSJ0MFfb6~APX=G2NsONOpujIrF3Wy45W?aAj1O2Mo|Q10tBT1 zpq)(*847ZcRhVfD)B-cdSy^pxu!-QzHoeL~TY~#wBu3$Mum^!E(xE+*W^h)kPy*?z zLJ1t-mB2vKHe5+z9FYO}LKw%;6roT8)=H4H!h#VLhY=KH5DaL+2vW<}r&!v?7~p4O z+bSF?0H81)!*0SWh47arQbJ7xuTY)Ia2@S8KL4KUKLaCekbRvD;3exnEId5iZT*L< zLqguJ|2OeqMk7HHoLvjD`zj1HWh6{dnQS7{+F;cx0KhU?9Rp2>LZMA*fE34Iq*5&w zje&Zbi7$uqKXfE*G7*$X3;a072K^KOKnl~7P={k2rpH*Q!wd$_cbI>LgPpce2gvm9erIoGBNEhQCB2V({T4@L`GX~3;Yb*NUYicnhB5zb}}46e(- zNCL+M_T`QiYqsbzXwt?nY3Dc2fa}aO%V~ih|F;Fp(h(HPVHCf)KPV?MKEB&n^X#+;e zzLS_9lB^bBRx3#ugnkNK&RK@%SwcZA5c!RB4EWjC8E6MX^cW;7jq5OqBb=1tXT(U> zy$DXQj16rQI=u}yLC(u?6F4+rE^SzGy?~yPW-J&dlldPcCi>kla6r`2#bFs ze&iryAt;Q4IunB#pz9p7(7-@W=m`fcRubc&7SzwSv5?_k_EkB$qcSiMhkQp!wmx5# zZ%Ailhzx>+CWy<2HxuIwW{4xWtQ9lxFazt0^wOTj@Rs~yFhB+^9X+C2i0k^$6x0G- zuYy^a#Y#ec3v$ka1rrE|#&Dfps_!|c3`7LwyPOCIr&!k+tnchB1Yy9)mY5;khL|BKywK^<8(m>}l~`RVRQH6(YDD(PQ?!o2 zMgHSiN^diy^XtoER|-FsHiqmL8CMeD)D3NfT?fNdsLM@1jO$iN;@baA}QyU3+ z_69;iKp_8*IE<3am_*~r&><%ATZR#{k(|z8wdqL0Lhwjx{Mojp=*6;x1=2PS zbpmldNJ7N-GBJYFal%)=S&LWzu@tXg5%&?k5EKWQ42;wQ4eR>FQAZOZd=UmwXD~xU zx@SF?uP*Hg=yzn1-0N6|2z7~3X#u~b;d-7WvNkJZWM3Szwt>OX7KKEAE#P^%LRbZS zhixR8Br@4JV*<@3F_486hZ+=?NdAjOz>H=Ie#vS9S~{L`GXNZB;#C$THd8bXQy>Wq z9L*p}#!BN+6h(8WcdYcPy|`KOe=!8j5S%@l#8@@~_0P{w!RRrAN7on!)$$ct$Yc-( z7-cZfHj48oEa?lBjU*KuP4egpD5Ujx^V0pxA5ZyT+P~P~gv8X?6cv}ny};S)h9tyTs7jFDg1&u$3G#4p7AQ&Jhgq#ES6PjRz*m;&4)}Q>q2kN<0Tt-m*YCe^ z5&aK8`K^DQ8Io4Wuqw{_N~VC9tp8Ar$NFy+8rtaX`hOEoc6M+e$RI3Qz(Nif2@-O4 zD{N-rhs9uqS`ZlAx37XPrHIY4ViXQIGemnFppEFa0ojWcVmT!*h7nXgt61qQ$BSz~ zB^aD}mH||q1mhE5GBE=(41q(C!|$w=0YqrfPr{O7Ga893;HPu~gA|T_3z$Zu9T{Mb zHjFlsw1eI@j3kT%#CdHQ^+BcR3x6J#^%7~ued zGLSa>CFG!L;23^am}oXYe(1|6l;;4V4T+~tM5Q9h3#E{YfoPIx5-Kc^ z!*Gnlv<|hHZ_YQf^2Y0Up(;RX4;Mw_(92M}^dI%QVBmOVlVCYWal)ybfy4-l8+YdV<0U%JgqGBNH6mn&BIll}C0$;Y$%E0}n*?;<|B1iJ_@pw4UYHvT(q|N{3iL zIK+AtVwADc>};fI{I4h)1(g3qqA8jh@Dll79Tw*1{}-wbYZUrc{(lqCYmxswMgNup zy}Z=`UWW!ilQ0++cn1puCk*eeb`Yt#3F1`eqq0`L@9>J^1H&*3QJ=e{90*}lV< z#w)kakEhD4)Bn$zyfkc98rO+jODA$Y-79fAzQc4^%tE?Nu;_FPasWZ$km5jSex;Gq zPyx%zPRj~bL2{@nU~#$QV*#s3F3kmQLFV~az&gkky5R6-&Wi;dCDBFAqqy=_HmQ91 zKZC!PdjK!t{~@6v5$^s!;p(^N|8L}RmH!P4E))NG(qb^m1VJ4u#+ida9lf1j)!Jr2 z)rue?t7=Wsdf>;?NpK*d6hD!I>qrG4Csandx>6}8*H@(6x6kW%@>~Bnw34*l0;yNL z4*0V3zpxMw|DQ%-Z_oeU$Rn5NFsqeyYV$FU*>{oa{@+SyMH6XRD>TTQtywKlqx&MF zuQI~Kf^i0Od)c_Yu8}QDCmq*U1k_|RkKoCis>N4@(;$HOAoq0EW)YQ47x6X#xq^T? zs#FmdRZEP8N&GJ2=cbhUH2^A*16VT_8Xm#gEY2nzem}3HQLo3O1wlaA-ekihjghN^ zx_7zHt&BtNYb=K;Yr=JLzD_bK3L$y#v_R%Q`zwC-?5am+rGHJ#;=B(h zzEL_?zuYrPHbUY(AV0o@*dg>vC0%v^eftJ``6yj=0Db#vJw9`o2{}ExG?FB{V&PH4 zdH+I=9HcT{ZM{nUk4{Wz71vr9AC;s_h>DMOdjztGUP`v04GUbY)Zfj*+;G!UH}<6#c=b8$!u z&QSY_dW2U9Z%xHTS43Qvqc`4~pMe(un{fZLfR^4cp;Ed_Z~_2m%*L5@9Gwm+o$T^^ zQMTPTX7ToBmlnGxK}V4c^4;=%-)Z&$$~o={7u@hIR&~8}Ur0;12m{Mxbu^EytQM>#9M})k+|NBNBSNp%)#USsVKn@F`n-j=e zU!Pas+uyjyiRb!P@k2$JD4Kz-1WuTGBf;h+|I5Vi-Km`&h(hg_})#OnER>%O7prA^jib>I>a5MuI z{vevRSZE4#icSGI!LSOIiQt0Ke+BdvDt#X&82wjzVKxQx|B?P=GpJxEAVJ1Kcb{2B zpo-13Dgsq{EL{<(;w)B0U_XUF=!7u@ZDT=POf0KVSsA()G;j(PfkP}BuaVbM9%7voD3xqCkByJ#R5Cl!75*EmEn8k{8Ynrd;@U+HJ ztTDaIg&hEdOR6a3762JC-2VDIXHg!+Bs`F^S>(m|CVaCTfipVT zbkV@OTW6hT;;z<2ci=9==d0U@4&~7hUB39l(^s5UueJ3wO+t+FR$Ugxwcu49QC$7vWUxt?h2By#i+|3&Q@=} zOFqIx`^3lFDwe_Vh$j6ifIMSP7ZsW#Gp#4l)%p=+`6ByQdW$?ND`8s z1$ZPSXC9q_#8|e7ljs`ey(rb)bv9>Io7V!a$lY=Wgt{r-zkO9CeFVQREGVyG@(T?F z1=Kmzm75NL-SLJES~w7B*r0{@ zDlIlCwRL>E9z;Q~d2K^c6nr>YsC>_s3r$)+g6S89qj?Y7Usy=q;`hG-jH09AJte>^ z!YK+mZk|itY875N6jUO|2KV2f7P)0xDeOq+my4ySI*j*>Ero{WBd<&0X~}z%m%_l2 zyRdnyQfNwYPQIiHSfnfth3cFSI65oFnF|lhnPaQ4Rnu*Hs3R~767~vRo1{&$bTWml zoym8;NVVOa3%yk<^wj2;oTmr_lW$_4SBF=`G0!K%&#Qy-%mwrsV6v}1`ciJ8SC(LM zx(7B02o@xl?rcN1R}Hem`nriPxRl<`v&|xi#qtb!Mdk7CiS!BuN9I}kTDm0Sgy9x~ z64Xs2Em)L`ZXe`v)FIO}59gJ@c_SYnPU`$-n=xcrJG*#&F6Y5(D0r>*XNXt+$#9A6r&KGWfbnkJeI=C*_jLc|Fn1F+vneYeE9#$(gnPF1hD-5e>Li` zFpv0;>bLiQ-^}yx`TxpE=zqfL*NHHC>Gbq_iY4?U6n%F^htKHtUL8L9F;eluX&f$VQSJblf1@J&ln(7>nZ-R@ za1MsbSN_8Q6PU-)VSabn`n&l;>y|yxeXSeqbWomN5(rnZKH58 z_k-6Kg-ug{~{;m<84ws~2HqHPQVDGso9${-3){Ny^x)NvHz(Ir$^C3RkOad$NQ zKj_=cPm=iOYj$aNV}amb^<$RDgYq##-bYD*>y1gszdLy3OFHOV)5J)U&V+ahB7|g& z^6ALx&7eZrND|YN&MedN$A_imB{nL#%94|&Rz~*dj_0Vm1za;s>d(P$z; zijbBJgsUPN2D9J1ZXrqyZob&cBM}v;3(Oxl$&B82Jo)@Vj zBO{dRP^CIDRUM(#L~6quDMK1-Lqfb(OAu2;5Uh|uS6Y&CWIQU`ywkaOm<((-0xt|5 zNarVS@D72q>yocM>zD@|m?48^Ktc@AnY<(gEH;)Cl4Tf3jDa{HC^X&ZbOoeCyCjVA zp!mO4uKh&EpK(RFjN=l~#d}@fS8?Kz?|?ikO<^G<3vL6i0wTq1=~T$uK<&l5P0ezZ^D{Qo|DNjZ(B_&DFqXFvW4>5V-z7cid++>CAfU&7GOD zB<<;B`7dnxF^X)E%fEW$A)mP@pY}D0Q;YJXh|O7SPNcR(xqRMQkUUo=@6UAXP7cPEfX(A zTX^!l`x2?r_`LBFsS3;ITX;U-B7A;Rhxq=a_Wdc;%dH!X!7Px=*PrL#WTlK3VXy$C z=}p2b$}NZds7tbCdijJ7wcS7D>$-?46k&*cBPd`Zn0>u{aads8pZxITAA{9aB(5$z z{vZY#2n$963gc`HMgk)(n8kqJ4lF@{W_XRqR|%g`ahh|1WV7=6D`#dPbil(26Tz`6 zUod>aMj@{V7U-b?v#~;gLRl!$=%kJg;sR!j1r~xN36!RrGkfZ@5>#>906ZZRJl9bH zq~T)*&W4df5OWq_3>1!eAlpRH?rBQI+H-YCU{N}4 z9#he1Au6*bA_G#Qd|CLJGIN}j)dmN$w>pJBy6`b!`H4zOkPf}A9HncR zPWO+{X!O=n{t}cASA~}{SP7FNRnE__$M+<*w+$nGf6XGi8Hl6|Wko?q7`lI@F9~#P z7Btv6UxE@}LOQDRStN;pM#$t_&`G%|&_GjozBWN`tK7!I*I^JWY(T!kSlYcG@8ru& z-5eDCl4%kGBZ-+joCO@a1izm#7)}^C05O9Z(3}}Eo<>}sXG%r1X)A%=ZpB!ZA-&PI z6141^6qS;a+&;zYSBlQRc0ywQ^~G4gdaoa&lVhV&V`EzSXgS}y35ogE7V0vzSDKbu zPEKqen{V4b>r1Wsgdj$R^ZG3=p-pUZT&jyDx)~koYnfjtjS5zov9dicRjA5}WL!{rqbuBzmu( zU);V|pfFMY|Lm*js@hlRW4e^Q4v3JH08|LaXWZi$^_m$^MCL%5UVRsje%xE!z4 zs!)~S*(2_@THqHNqK;ICG*+rLsp>{rjYg{pRW;IthlHxbLNyITLbV~{iM6+@MXVz- zx(Ja)MmRysXXO`=M4xOHFgA*Eu6KfxnH_~@2oq#EEx^?wjnpBAMh2rg(ijFqBO+j< zMj>Iwa9j-|8{-C}AwsQ(P!p++3=a*}ghm)N;fByq1J(!+HHPYq2pFVH1O>IgL~~X_ z7ckQnsC5`FOay1P=~V{W5{#}b;B>IJn{7zL$)p+Fc|S}nCI0DBRtXFwZNrrm^3|0p z2rg9OUr2>Uzmd0_+%-@Mpamm@VryTxKgMIWyj2qKEpJc za7|=mi9Y|+3-w=mJo$e<16jm2^5_5R@KALl_xr!$jo#+}dm~S+kso}Fe&jVM)(~rx z9Aan_AJN_(8LroK3c*vtiEb&85s95TMZ|X|dtnrAPwhZAYXeD3vD?iEw%UgAe>dGW zZ%Ut)`s_pPzPJNB_L>u9?{V(Q(Y6W6&(^PBw7u%Mn6TpQiIeE$sYsHwwSKAWS5>?@BG>0nor8Sccfg|8|QcQ zT=mbr3x8YhZMk@E$K4k@cD%Uo_k}Gxta*R;d{U##-O>YopM9pwqHWdAY+8Qx@bd{x zgDSRQErZAVPc2t`GQ2ux*Ex#*X!n_zZusQSXJt_X%FRv#Kh5k)_22(Va=qUs&Y1V} zZ@F8C1neJu^;G}E3qNKQE#E8F;+kPa-3@0-bWa#^#eZtxjJiA8FU%>SN!Ms@{k3@K z;Rn5glZiY{yQ!l(ZrihBN2IQMK#dt4jt;q$essu#>?-qOuJ-O6+VEQS((0v?K8~u= zzFvh&3);6iIQYu+o_|anjt_pcu*0S?PtNC@-a7UDV&iKM&COi-&86$dha?AQ{FRXx zv8T!~w(FDb3+DVB)@Z=9HD^nnEW57X(iKN$_&w*K^LDogzjGAy2L$$==Ejc}JpVGd`|!Vlu11Yp@+7G(tXg5`s0$Aix~{PWI^rD1DI@B6UPAD2sC-lM5rYu2NWz8ihXx~Rs;Bm9vT{$K zqcwJx1C#vYi;YuMo!qh2(VJy!tY0!Nv)F;=%S%k}txPRrpn@AuX`|aUf9AeL_8$hn zcPRJZrqw?jO0LJ zy?y68%h&<6hkkXW((`o(mXG_u_(Qv0#oGU%sx#p5+7TPo@ZOiI4lBZ^L_`_4B|WXa zy!egV$8(k%D>we#JnBr3vLy9Qm zU4E{4@#D$&vyPRZ9*jy4ik`UZ+-Q4)N_$Ixl>BQ}_5KaJ4C-_$q2|K_t=JoTb9O%1 zz4QJv&9F+J#3>#Qc&y529qMu6t?NFlAE+<2!_+%yX3hSCOVkQ6 z>`r~~Ns|Hoo!8F)VcUaYS!?@G&x+gs?f3+L)0x`%(^LIBJz8F@Q~eIJ+VvU@rgd-U z|HtGi12**1#&4{BZ2y8v+v`zkU^-o#71UD_Z^wGg8dzRXhzUagJQ$ZbZ z=2DA$gPYFO4Cu1G*u#?R(qE)D=0bMWYw>(CH)F__)L#!Bi}_^aCx4gUpfSen z92NON6rhyB`w zXeQNH|Fm!Jh5Kt#<31*)@0}Yt^6c&(F4cZ9{M{cv{-^VSjp?mFKHsnLk9Wu_%e2?W z<2Bej7aJYy+Uvp}S3ca49JFE0IK_$@`Qnyt zAb5DKT202ZoDouUOM~zZTDb4)g*EIq-+%hej6H_Eo4-z8JFLT87}()>$(avJt~)V3 zjOS0Q5_H@HV!*7nu>%d%(QK2it4}m1e2Q3*}E$ zXxZeaTiyFSn^bjN+l+O|>sM)OX4m^@xT5~hxnt*Qs^7Lh-Z-^)6aT>83iZH0 zN=}D*RfXnP)e91+$gfX4TQs6U_@k0*j*Y6;V%9%LD~&joot&b+{_TUsM+4_|>)qx1 zYEjqY7LU7?aB5YZCAU6VJ4;ckq@|N-erIEq_2)ZHUD4|P-IzU#zbiAfXL{@UEh3L^ zdN(O!-h(Sg)2x--e**1<{cdD(OxMrnzVmV3#*zU#y2P#l8@}9HHK*Z%?|R?9|LOOY zZEbV|yB$AO@AkP;)!%7x{OFl@^T4_GuYQ_-^@n}w4FmFyO{o<=@z$PE?Kbw@S$Ry4 zw0@13SAIA+t>&}g@ws2w2X<_z)c!d!t#pl&{v)dVv|vb$3*7QA+UG8Nr{RLDeGZSG zG^X5|fa1%#_F8$=Zsq!%37fwknC|Zk@3g(D$}> z_}6=jhHvUP`23@0s}%F9jy*hU{~*JN$9DCx?>e{M79F^u=HHWc&FFBnp5LQ=ZSSo- zSEA{qtwg!Q%j_dsZNHo`a`b9q-yrpz)ti#8tgbs^Pe`lt&A!Q5vSGxY;O9;Dj=C{2 zr_)dI!v{9}qi%!Vx1x(zcv=Sz>b$8J=KEbLEvS8S zGTgtj>-7b7(v;)7TD}`_x6HtRMx7RZwX*y#V6*BS?(pB2jt^V(^!noKIXlkKow}?$ zrw%x^cI2qwN8|qttJ*(ddFNIX_G{|a$A<@Ppvxz8*-ktut2?^vuWl!n-IPG3$r z+OtN>*u0fHz8(ozv~Ktr{CxGoVYRPa&KXg+^nuRQrtqfIo;0wH+S6+EJG0qlorX5~ zVBFVBmQ-Ig{lnNfyJ9-{H;i0J_8xz5pZ)QTk0+R}EcgGq%;KRf8@As(B5_2cuaE@ z+daqs#&@55_X#(-WkUCE$}FN;rEiIPqv{SBvvKsp{ew^XmwAzXqv3?Ajm-lRhCOR~ z;Zsm0p(TM&S}}ch`IuG}#}93|;8s~~afAMGRU4nZH!3ZxW9Pwbht(*X8(Dne#Ye}6 zFTQqh$lr5yt*6buj8!_+djOTSTUF+lsjaI1JnGioVZFa$h>@yFD^e3e%yKFj#>;=@<`mJ=2L-7NiGfyg)nKQ3Nuj(n!nJ4R?9k55VIbUzPwn?(N zL&H+rzFD0);`HI}krQHfkDn5xJ<_cBn468B^eh`(-|t;6G@{t;3cr-@adB0^sD3+& zy;nSIVBV&`TOYf+^W*T6zh@PTO&@mCpAMVb*$8F~?LFHV6jyV@>2#HjqdwpE$D=JP z`Yn#n-J4mn+M)Q_Ni(@kbHe8>9=YP#*+aIL{@auQ4us&DTMT|KHs zrOHtkl^32R&(tp&w`a+?-2<()$CogYhqwMQ{qW2a+7s=&)nZ1awry3v?X(Fe;i@C* zuR6b9sZHwCkN2#&Ik;`yBF(KC~w z7^=40^~3a@>;7qQCFaDT)6HY^GM=SYYw==B%34*;C&jM}95_DiX>_gl>gz7Vez0ML z--TblZCr2hwpLh+WkX(Y_bV!f>ATLXy+-Z7ypvyi@!toI?f2Ip`u6g%Rhr!0GIhAQ z)0p{J0SIsbd*mjO#SH^^GADV?M)&UF`et=cRhv-rj#H?R|)DsvRr$ zR;s7()@aqOgmUF=_L=DyG#59o+C8AgPyUm;Dw4-^8UMui<^Hsk#UH6kcA2ikR#e=3 zV1B>cormpHn%lb8>(l(7W15FwpX`n+{+t@?zisM|{_3b_%Z9C(^?lQ(-&DVQB(dA8&~_(qzn4K zOI9ts_bIe1%f8pVU--agist1XKJ9tl7(TE@7jJ)5xp$jQE^ zp5-a9o=@+!$jPz&vS!Zh8mBiBdH3$FIlJ(A?vcmuZ#n%ux7_`l)R60^mbVB!eQu^g zV?1M2*K83x$2j=Zr$gG}6>nr5JTa@5D)_6+0~HUKxwkd=+|Cn~+HbNYq%;~(_Taal zr_`xkVp;be4r`Tl`dqmhU2eq+ZQ5{dt-q@0TzCZtA@uuywTqeeP~AHKNRo=eHx~81|RjGkNU!V^b%UZ+d$E9KU7<>sP#A zN_$rE$rSbTjHxvvHB%>9Hr{z}+tr|`XH8qxANurY>3$c>YA0>@^}WY?w@+jSp?V zpSiL6eD!^0LRy|PYS4@d|TH1#Hx$D~Hv|Vc! z3?KB<(fXBEjNY)JRmky{r%G`zdTq`PpHK8XfdtCw&UMc?N()0?w!x--#*i8`wR2%XJa3AU!2$XpO~eC&);l(^uwpQxw-Vs ze~#SknVJ3K9ya^n!Oi`S8Y(}^`RL}Yz^`kSU62^GL9=j=Nzvxd!@6kerA zT22}BdH7g(B4dv&z1q|tADr)Nn0Dyw@AGm;Cw<&+I@|2=Ty3JZ;q)UV^6*VD*Ft7j zJk@36S=F9)$tCr%rN>;paeZRi7r|?ejp}!`>!@03pB&vW_v+&jchyyir|R)O~~u4(mt1#EJ5%O>>*^_+)O_cp%&&x2MQ?qpUOIxyr!WcdxVe%W3! zar~jDmpANMac6Y>yUBYa>P<;i>!NP8%3E@B#(-$kg!O4%Hjp*zwqMZhk10okW;~tu z9<}rAmtRcYgn#>o$u{cADC6^s!KSOXUKmGjfq#GaH2KTr`nex|ar(|L=LTob&{+1A zIP&9&_x@?lu{Bb9oLrv9En46E$+i8rBetxp)5jL@!>r-IeRBWwxHE|-!^>PfZ7$w@ zX~)iM2WR}+WnY(lH&2p_jE|JzgGQe`kTUK_-@jX~JJ*uCJ*{8Wy34Pu>_vX_C~M8y zN0(GTc9}3}%9Jbha-u)qIrQsx}^;?%0VFP=R+GUayO#T_qxd67Hr`scUv*5Azg z>HfiAif=4e?z6i8OiXM=oxGWz+0uN+TB4Yx_3bgc8*MuMbT~Dv__;IVKf2vNb?uO^ z)}NfTeo5A*sLHg>5BIAxc2JFP7oV|jN)7Wbrq|UcWb3jsO^Z*hUTACB`TRib z#!fG)ocp;(@Ssg`FIv>P8@w{FX|>yyt9j8)oBwe*x1sH`=ihw!G~WN9;;RbJV!u4` z(+lmiZX*+NFSq<2q%FF1%Q~)N+QnIQYGibI-l*M#h3`Dxuw(l@fd9B~$l2f99&c8; z+T`WSCtEsw)fn4|-G3Un>cgcUe^D`l`fB~0oLyROqLF>FZbJPT$@}juqbl7y7aP~3 zVP20vSJe1^%fjXxiRZa_f3KGNBk59fEG@(P~@>6$STWL-R zu87?p+pgBR%@^MPZ1R=qb>}af^lmZ#D`AOKsVI$R;hksS>{sE(i*1hyyzC$F;q|Gi zDxWrL*|->WXa`+w_DOOWwV_&0vwGdPe!u$a-uExEBoY!M|nH$K6skfGi%j}ymI%$Vrk4zf^1otLN55HON>5TF{elw2nuRJ*J02!8@Hvjj&9UiyScKv+Vm87Mk zo9;;arE;+iBSHV%8xbE(SloQy&sMtDu}&dR+TEMKci~^-t0wH-(xhF~lEa1@JMQ%~ zl7C&8w*UAy!bS$-2>aY^BYnZ{rT+kpRr%2n-dN{!{-NM z3r@_+AirO~?FGAK>GH#sD~{~9VPLJy56V=W|Mji5*1r~|9eVNf@a|8Cj4+((vNLno z=Z{9PXRWpqy8Ej)##ddjqSf*vljCn5jh_GIgj2Qy%RfEQiL0e|tdR%^+ja_H{nVo1n-8rzG2qyE?2AF`&TPwx5gPSBEs% z(3N=b!ITY|M?0=N+2H=8zecX_f9gTtkB6pwn{(n?$;=JiPd^Cxbx5g6Wfyl?U-Pr< zg-c`R_Iv-R?UQBQpFK#@XJTiDj_g_Tw*%=EwR&U6ubNe2Mns!ezQ3C4aHx1_`F530 zCcPN-eQ?KT4`-A*c{A_YFTVtjedpZVy!Vehp8P29!f!h|<+a#%{7l{kF2wpMVS8xH zjcZGhTTb4)yZQN!%$e)1=a!4ByC|9HmuNW?mRD}*`^*2)4Z8Dp-uw0&r`jJ}x8>r` zA8y&QH*4m|!H+3pkk*uvFnmMv!zHdx-9K^tdq0k?z+&v zs(9w@wS>SM=t+W@@Cm>jkoqR+4}6Lr_Igjwxe?A zoU;oCs@5H>H7EIxJs0*|SX$@f>SfH}#Ku zo4MS(%X0lM*JAeXs`Jn3ceX9*{K;q4?A^u=>b9^#qhYz3%dSFl2)KFUyIM;h;4H|fBKx2pJ^IjM?HZ1?4~%EfeBYuZ1}X!}v0 zj@)(aCwImj-_mU#5&hXx|Ac!hMm9cM?3_5)V)OpfhhhH@AJ6k& z;R=<~z~HaT2`K;b-_WoyO{m-XZ+Jvl=-cz(H}N>4_%qN%upDF8I)15~?jiEH$qcm8 zEWy!yQOrQ#UVSq-%%lYwYLx~#16ge(nIz(-9@jf>AaEKy*5iqNO>~WAc?4Xw(+;K9T4z z6D%a05B3)kBPf#+z2bW$#8PCHG^NCpRvK6Md*I3stiK~Xt;L=!4sAPV7fK{x7N}6T zg?|Crefd}Ig&^7dUkc%hQ!2)oAg2X>l1o&y{N_`1ats^xb1(^hA28mL9+D?wkjS5E= z9s`5xWC3j@zjuGYuQ?82lpRno(;55DO5+GBZR3RdOTvHvngXoZ;=Ft-CG9p_bfW*P z6Ex%uINxDP41Os1a0(HN1qUe=h_zTbJFu~k1??KMq8Am2iB3`Rz&Qhsa{Fn4pC1C5 zVkJmg45=qfu|kTlCuYIOcg+N&Rf7Dnlm^_YREKKSstBc39ib3!w2C)35w%M(wRjSC z4{xmkN;PN(C|VdD1&XGnb5zj+rCI<Cz;f0T}~r zu@D?jxdOF8_iU&-juvx|03$}SP$ABQl?NnVlM|Q+;9O^(6(9;lT(ZmwUjUSXWBQ4| z1PT%V;4qV)(3o?67}~}Wlo-xa;-P@GVwn_b-pbGh$g+;{;YZFO1h6Lg)sRUu=_qJn z7{B_Rt6OxZ)br%;n09pSMO^nCr7JN04(~Zfj7t6oj#iD#uii0tge*$3?7L2?t(ETL zxdL~%$fL34`=*BumaVsoVcSs1Y%64B&>X&yBIZtqP62%}=PigyNC7x);pu`l@@rcn zbC8J~${nLi$h}0Rj7CCDtz&7Bv;kf3@|Ruy5mOfcjAaQE1##iNJan`p`0TSXsBXCq zq_NUEJT(0Um@Xh?7v)8W@^O+AimWA+W>ZLPF1=Io@7%i}%c8)yh2f}4h_0*4d9kqO zgygG2_Ij5PW&VQGDy9{j_c7-yz4%Hmkh;LGS+_8ie4C7o0+>O-L=TNL14U*_u)-dJ zE_8|P-Xln-gXeR!$;3Cn?;jY&5kShZ&+i=kU^&R&D&!gvA-bpJm$-o(W>QQ`9#a-_ z{Qu$!Dk=jb_(29CfM$T&6&g;kj19>%I=u}?o3(Ri5{E(JF9y${AZg0Pf0jl;z@Ft7 zi!dM3&L`dEbwWogN{R`KWDBVQe*F`ak-=Dwu^BiU0|7(xhzJZ0qi|rwSQa@SD};;V z^1>*O8mp7#ZkH#wP`VaYqQb(olLU(b6o@MZ1t8`*xE@NknRpJv&=!{kWWgxHD5NFu zcb1c_B7fwBF2C!;zkyT+!O#@q`NI4O0sRmQV{Itjwb4cb3try2u7;il0>m;zCAfVNz)v>i9qT{eczNwz$ruBmzR|0?+{eHZjV^nL`0br$Y+u zEtr4+{|)7+15rtF2o{amII|8F^f~tq9waeg$2KNq_ZQ@@n3f=&Q8idO;%NH+hC%Cme01N!nyb5s~?}JXnB2h;k@j<~pes z3V;s$Ty+6n^y4w{^B7V=+K$Jpg6JzFIgo)dgs{5#L8YxW)Onf`xvIoWc}!&0#O#C? z%!*`9fs#ZONQkZ=U35T=Sdj{aL=+C=AWzOLuLYruD6V21D|XviTW6INFq%PRhUm)f z=VJr%;F=(SCu1=OHybQPv1c2IEun0S;^rgBy5eO=e>XT7!;=SNAtr4!0fbN2&3$ZyiFeGP)IvQ}e`+M0wWz?;cKxu(rwj_S^RXHe7 zWndrA@}*A6;u`$1V~r;B=#>b zeT)SZO(_|g<|NUtfk+tyN$ZTPn9oQZq7f)2(+oK?8aQd+aO^A+tKp#&yzES55F|7~ z99RV2Cct#Qm+ycMVJTA#H!rL&aV{mXMBJmq2sDzu&5Na|7Or!=(*nOhKWSqS*bhdM z&N?hFzDlYpbOr}<$h~~~p1ZkFLwc7)-Qb=h_^&5#0_7{N)(wTJt2JN+yC?V{|B4qT353nfnRt83T)lD&uX9^ta)}A9%P;_#Pt0seVA`G_+jY+Vz zK*M?nCPLU2q!`NPi-#zsIdTQ&z+4l;x?;}J2*3P$z|lf{DM2XW=ag5G91(I9&M>Ju zRL(c@UZ>ZJjCuyNUt$ggUETx;fGU#;q;PByFBLTKY|&+IT=;<=nz0OmMk+H&V>r-@ z5g;T&bk7;!kJLew;)2Ieq6+>c+SO4XTp;mkhoeDUUW;EidSn%XxQ+*@1-M=%D)r(8 zIDZet%4#L;g7ScNO^+6lWX_60y^Fdf*6^WnWr_M#A8%Tii}oV5%-OBbt*>bNMslIB z+dG9}N#1exT~s?nIhco|(>vmCcAgXf;pc&9Q zI!PSIs7yAIX>G6yBN*j0I|(NqAOOo``5GpJ70ATt20XqN+D36wha{!RX}OXVQB+I| z{P-CuM_c%P zTaw6m5;EH?7$xT7Ld5K_ok_Bfq#W_@9oq?CN{8VVfab|w+ z5Raw;94A-&CXeiv0u#4l7M;rhxRYf$EgbwrU^tEp60!ji=Rn%*1-lM1=kjI6G(Tt? z=Cw?bzlwHJF?5A6QSC)rxIibFX2M{0j5!7Qy}}G-(}JjjpoF+?UXmO;&(xd~A&jDA zA@}?)TID57#GZ*pc`?WmodKClhTu3z@w0_da!Y%VFfzQr=y{r;K%u5MXy6qj$0&$- ztPy>p)=|f#!#{x-3q^u%D@ZE?EjM=<36oq9lNZqH{0oh6j$h#5@I=e&L9*1$r znyTd~1vSf4r3a-;Oh8)nh`4n&h7=r7w86nfEFCJ*CnaVjRH9Wz#by{(uE&oGe~@Cp zNIOzad2PWYHwkkzO$w7JMrskf8c>X2@sLR5L;?^|C`*cS7|fB?P-sA^fL0%Rh0vCm zv!GJT&cX{&U)mzY+74Tso2kkJfLL8_ic|YJKrr6SUHQTAw86#8Y;XkxK=g(KS=qsI(d(c?tQ( Date: Thu, 2 Jan 2020 13:41:58 +0100 Subject: [PATCH 028/104] update operator release image (#777) --- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- manifests/postgres-operator.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 594533747..bcf37f693 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.3.0 + tag: v1.3.0-1-gf2695c7f pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index ab0785a0c..237c58008 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.3.0 + tag: v1.3.0-1-gf2695c7f pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index 498796b61..1f61fe6d4 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -15,7 +15,7 @@ spec: serviceAccountName: zalando-postgres-operator containers: - name: postgres-operator - image: registry.opensource.zalan.do/acid/postgres-operator:v1.3.0 + image: registry.opensource.zalan.do/acid/postgres-operator:v1.3.0-1-gf2695c7f imagePullPolicy: IfNotPresent resources: requests: From 9d7604ecf0f4cd43ec7bb18d40cc3411849a9d38 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 2 Jan 2020 14:06:23 +0100 Subject: [PATCH 029/104] use v1.3.0-dirty tag (#778) --- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- manifests/postgres-operator.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index bcf37f693..4ba6266b2 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.3.0-1-gf2695c7f + tag: v1.3.0-dirty pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 237c58008..b32405690 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.3.0-1-gf2695c7f + tag: v1.3.0-dirty pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index 1f61fe6d4..5ebfa1e8f 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -15,7 +15,7 @@ spec: serviceAccountName: zalando-postgres-operator containers: - name: postgres-operator - image: registry.opensource.zalan.do/acid/postgres-operator:v1.3.0-1-gf2695c7f + image: registry.opensource.zalan.do/acid/postgres-operator:v1.3.0-dirty imagePullPolicy: IfNotPresent resources: requests: From b54458ee3c49dee39d6e365402e455d0717d6faf Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 2 Jan 2020 17:34:49 +0100 Subject: [PATCH 030/104] update copyright in generated code (#779) * update year in copyright of generated code and LICENSE file --- LICENSE | 2 +- pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go | 2 +- pkg/generated/clientset/versioned/clientset.go | 2 +- pkg/generated/clientset/versioned/doc.go | 2 +- pkg/generated/clientset/versioned/fake/clientset_generated.go | 2 +- pkg/generated/clientset/versioned/fake/doc.go | 2 +- pkg/generated/clientset/versioned/fake/register.go | 2 +- pkg/generated/clientset/versioned/scheme/doc.go | 2 +- pkg/generated/clientset/versioned/scheme/register.go | 2 +- .../versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go | 2 +- pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go | 2 +- .../clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go | 2 +- .../typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go | 2 +- .../typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go | 2 +- .../versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go | 2 +- .../versioned/typed/acid.zalan.do/v1/generated_expansion.go | 2 +- .../versioned/typed/acid.zalan.do/v1/operatorconfiguration.go | 2 +- .../clientset/versioned/typed/acid.zalan.do/v1/postgresql.go | 2 +- .../informers/externalversions/acid.zalan.do/interface.go | 2 +- .../informers/externalversions/acid.zalan.do/v1/interface.go | 2 +- .../informers/externalversions/acid.zalan.do/v1/postgresql.go | 2 +- pkg/generated/informers/externalversions/factory.go | 2 +- pkg/generated/informers/externalversions/generic.go | 2 +- .../externalversions/internalinterfaces/factory_interfaces.go | 2 +- pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go | 2 +- pkg/generated/listers/acid.zalan.do/v1/postgresql.go | 2 +- 26 files changed, 26 insertions(+), 26 deletions(-) diff --git a/LICENSE b/LICENSE index 72dcd53e1..da62089ec 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2019 Zalando SE +Copyright (c) 2020 Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index b68a72d1f..dc07aa2cf 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -1,7 +1,7 @@ // +build !ignore_autogenerated /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go index 45c1893a8..cb72ec50f 100644 --- a/pkg/generated/clientset/versioned/clientset.go +++ b/pkg/generated/clientset/versioned/clientset.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/doc.go b/pkg/generated/clientset/versioned/doc.go index d514b90a4..9ec677ac7 100644 --- a/pkg/generated/clientset/versioned/doc.go +++ b/pkg/generated/clientset/versioned/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go index fd63d0ba4..55771905f 100644 --- a/pkg/generated/clientset/versioned/fake/clientset_generated.go +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/fake/doc.go b/pkg/generated/clientset/versioned/fake/doc.go index 960df4951..7c9574952 100644 --- a/pkg/generated/clientset/versioned/fake/doc.go +++ b/pkg/generated/clientset/versioned/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go index 2883940ca..5363e8cc4 100644 --- a/pkg/generated/clientset/versioned/fake/register.go +++ b/pkg/generated/clientset/versioned/fake/register.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/scheme/doc.go b/pkg/generated/clientset/versioned/scheme/doc.go index ea0df2783..02fd3d592 100644 --- a/pkg/generated/clientset/versioned/scheme/doc.go +++ b/pkg/generated/clientset/versioned/scheme/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/scheme/register.go b/pkg/generated/clientset/versioned/scheme/register.go index acdd09c25..381948a4a 100644 --- a/pkg/generated/clientset/versioned/scheme/register.go +++ b/pkg/generated/clientset/versioned/scheme/register.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go index 388ec6178..1879b9514 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go index 8bff3bf2d..55338c4de 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go index c9373b6d8..1ae436a9b 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go index 31f2ec817..8cd4dc9da 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go index 8023d3b07..732b48250 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go index 8cc50e598..1ab20dbfc 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go index e1d824486..fd5707c75 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go index 43a915ca3..e9cc0de77 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go index 88645dab6..78c0fc390 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/interface.go index edf03dce7..4ff4a3d06 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/interface.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go index 77e7c9b34..30090afee 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go index 16a055f50..da7f91669 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/factory.go b/pkg/generated/informers/externalversions/factory.go index c36d22a29..4e6b36614 100644 --- a/pkg/generated/informers/externalversions/factory.go +++ b/pkg/generated/informers/externalversions/factory.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index 3af7bb7ec..562dec419 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go index 712305087..9f4e14a1a 100644 --- a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go +++ b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go index 4c353bec3..1b96a7c76 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go +++ b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go index da41e1358..9a60c8281 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2019 Compose, Zalando SE +Copyright 2020 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 077f9af4e399d7d50177c0c5074672911d88fc1c Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 6 Jan 2020 14:08:47 +0100 Subject: [PATCH 031/104] bump to v1.3.1 (#780) --- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- manifests/postgres-operator.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 4ba6266b2..5b67258fa 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.3.0-dirty + tag: v1.3.1 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index b32405690..60190f49a 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.3.0-dirty + tag: v1.3.1 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index 5ebfa1e8f..fa8682809 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -15,7 +15,7 @@ spec: serviceAccountName: zalando-postgres-operator containers: - name: postgres-operator - image: registry.opensource.zalan.do/acid/postgres-operator:v1.3.0-dirty + image: registry.opensource.zalan.do/acid/postgres-operator:v1.3.1 imagePullPolicy: IfNotPresent resources: requests: From e6ce00050f84dd2d894da1e7718e54e80f9945f4 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 13 Jan 2020 18:23:04 +0100 Subject: [PATCH 032/104] reduce tracing of dumps and include Pg12 (#791) --- docker/logical-backup/Dockerfile | 3 ++- docker/logical-backup/dump.sh | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docker/logical-backup/Dockerfile b/docker/logical-backup/Dockerfile index 1da6f7386..94c524381 100644 --- a/docker/logical-backup/Dockerfile +++ b/docker/logical-backup/Dockerfile @@ -19,6 +19,7 @@ RUN apt-get update \ && curl --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ && apt-get update \ && apt-get install --no-install-recommends -y \ + postgresql-client-12 \ postgresql-client-11 \ postgresql-client-10 \ postgresql-client-9.6 \ @@ -28,6 +29,6 @@ RUN apt-get update \ COPY dump.sh ./ -ENV PG_DIR=/usr/lib/postgresql/ +ENV PG_DIR=/usr/lib/postgresql ENTRYPOINT ["/dump.sh"] diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh index 78217322b..673f09038 100755 --- a/docker/logical-backup/dump.sh +++ b/docker/logical-backup/dump.sh @@ -6,12 +6,10 @@ set -o nounset set -o pipefail IFS=$'\n\t' -# make script trace visible via `kubectl logs` -set -o xtrace - ALL_DB_SIZE_QUERY="select sum(pg_database_size(datname)::numeric) from pg_database;" PG_BIN=$PG_DIR/$PG_VERSION/bin DUMP_SIZE_COEFF=5 +ERRORCOUNT=0 TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1 @@ -42,9 +40,9 @@ function aws_upload { [[ ! -z "$EXPECTED_SIZE" ]] && args+=("--expected-size=$EXPECTED_SIZE") [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT") - [[ ! "$LOGICAL_BACKUP_S3_SSE" == "" ]] && args+=("--sse=$LOGICAL_BACKUP_S3_SSE") + [[ ! -z "$LOGICAL_BACKUP_S3_SSE" ]] && args+=("--sse=$LOGICAL_BACKUP_S3_SSE") - aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}" --debug + aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}" } function get_pods { @@ -93,4 +91,9 @@ for search in "${search_strategy[@]}"; do done +set -x dump | compress | aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF)) +[[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 )) +set +x + +exit $ERRORCOUNT From 7fb163252c0f7477d1d432ceaabbc075c619236c Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 16 Jan 2020 10:47:34 +0100 Subject: [PATCH 033/104] standby clusters can only have 1 pod for now (#797) --- pkg/cluster/k8sres.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index c69c7a076..aed0c6e83 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1051,6 +1051,7 @@ func (c *Cluster) getNumberOfInstances(spec *acidv1.PostgresSpec) int32 { /* Limit the max number of pods to one, if this is standby-cluster */ if spec.StandbyCluster != nil { c.logger.Info("Standby cluster can have maximum of 1 pod") + min = 1 max = 1 } if max >= 0 && newcur > max { From fddaf0fb73731a6234b223596a4cb87eb33e5299 Mon Sep 17 00:00:00 2001 From: Jonathan Juares Beber Date: Mon, 27 Jan 2020 14:43:32 +0100 Subject: [PATCH 034/104] Change error computation on JSON Unmarshal and create subtests on table test scenarios (#801) * Change error computation on JSON Unmarshall The [Unmarshall function][1] on the encoding/JSON default library returns different errors for different go versions. On Go 1.12, the version used currently on the CI system it returns `json: cannot unmarshal number into Go struct field PostgresSpec.teamId of type string`. On Go 1.13.5 it returns `json: cannot unmarshal number into Go struct field PostgresSpec.spec.teamId of type string`. The new version includes more details of the whole structure being unmarshelled. This commit introduces the same error but one level deeper on the JSON structure. It creates consistency across different Go versions. [1]: https://godoc.org/encoding/json#Unmarshal * Create subtests on table test scenarios The Run method of T allows defining subtests creating hierarchical tests. It provides better visibility of tests in case of failure. More details on https://golang.org/pkg/testing/. This commit converts each test scenario on pkg/apis/acid.zalan.do/v1/util_test.go to subtests, providing a better visibility and the debugging environment when working with tests. The following code snippet shows an error during test execution with subtests: ``` --- FAIL: TestUnmarshalMaintenanceWindow (0.00s) --- FAIL: TestUnmarshalMaintenanceWindow/expect_error_as_'From'_is_later_than_'To' (0.00s) ``` It included a `about` field on test scenarios describing the test purpose and/or it expected output. When a description was provided with comments it was moved to the about field. --- pkg/apis/acid.zalan.do/v1/util_test.go | 479 ++++++++++++++----------- 1 file changed, 263 insertions(+), 216 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index fc068b322..a1e01825f 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -13,127 +13,139 @@ import ( ) var parseTimeTests = []struct { - in string - out metav1.Time - err error + about string + in string + out metav1.Time + err error }{ - {"16:08", mustParseTime("16:08"), nil}, - {"11:00", mustParseTime("11:00"), nil}, - {"23:59", mustParseTime("23:59"), nil}, + {"parse common time with minutes", "16:08", mustParseTime("16:08"), nil}, + {"parse time with zeroed minutes", "11:00", mustParseTime("11:00"), nil}, + {"parse corner case last minute of the day", "23:59", mustParseTime("23:59"), nil}, - {"26:09", metav1.Now(), errors.New(`parsing time "26:09": hour out of range`)}, - {"23:69", metav1.Now(), errors.New(`parsing time "23:69": minute out of range`)}, + {"expect error as hour is out of range", "26:09", metav1.Now(), errors.New(`parsing time "26:09": hour out of range`)}, + {"expect error as minute is out of range", "23:69", metav1.Now(), errors.New(`parsing time "23:69": minute out of range`)}, } var parseWeekdayTests = []struct { - in string - out time.Weekday - err error + about string + in string + out time.Weekday + err error }{ - {"Wed", time.Wednesday, nil}, - {"Sunday", time.Weekday(0), errors.New("incorrect weekday")}, - {"", time.Weekday(0), errors.New("incorrect weekday")}, + {"parse common weekday", "Wed", time.Wednesday, nil}, + {"expect error as weekday is invalid", "Sunday", time.Weekday(0), errors.New("incorrect weekday")}, + {"expect error as weekday is empty", "", time.Weekday(0), errors.New("incorrect weekday")}, } var clusterNames = []struct { + about string in string inTeam string clusterName string err error }{ - {"acid-test", "acid", "test", nil}, - {"test-my-name", "test", "my-name", nil}, - {"my-team-another-test", "my-team", "another-test", nil}, - {"------strange-team-cluster", "-----", "strange-team-cluster", + {"common team and cluster name", "acid-test", "acid", "test", nil}, + {"cluster name with hyphen", "test-my-name", "test", "my-name", nil}, + {"cluster and team name with hyphen", "my-team-another-test", "my-team", "another-test", nil}, + {"expect error as cluster name is just hyphens", "------strange-team-cluster", "-----", "strange-team-cluster", errors.New(`name must confirm to DNS-1035, regex used for validation is "^[a-z]([-a-z0-9]*[a-z0-9])?$"`)}, - {"fooobar-fooobarfooobarfooobarfooobarfooobarfooobarfooobarfooobar", "fooobar", "", + {"expect error as cluster name is too long", "fooobar-fooobarfooobarfooobarfooobarfooobarfooobarfooobarfooobar", "fooobar", "", errors.New("name cannot be longer than 58 characters")}, - {"acid-test", "test", "", errors.New("name must match {TEAM}-{NAME} format")}, - {"-test", "", "", errors.New("team name is empty")}, - {"-test", "-", "", errors.New("name must match {TEAM}-{NAME} format")}, - {"", "-", "", errors.New("cluster name must match {TEAM}-{NAME} format. Got cluster name '', team name '-'")}, - {"-", "-", "", errors.New("cluster name must match {TEAM}-{NAME} format. Got cluster name '-', team name '-'")}, + {"expect error as cluster name does not match {TEAM}-{NAME} format", "acid-test", "test", "", errors.New("name must match {TEAM}-{NAME} format")}, + {"expect error as team and cluster name are empty", "-test", "", "", errors.New("team name is empty")}, + {"expect error as cluster name is empty and team name is a hyphen", "-test", "-", "", errors.New("name must match {TEAM}-{NAME} format")}, + {"expect error as cluster name is empty, team name is a hyphen and cluster name is empty", "", "-", "", errors.New("cluster name must match {TEAM}-{NAME} format. Got cluster name '', team name '-'")}, + {"expect error as cluster and team name are hyphens", "-", "-", "", errors.New("cluster name must match {TEAM}-{NAME} format. Got cluster name '-', team name '-'")}, // user may specify the team part of the full cluster name differently from the team name returned by the Teams API // in the case the actual Teams API name is long enough, this will fail the check - {"foo-bar", "qwerty", "", errors.New("cluster name must match {TEAM}-{NAME} format. Got cluster name 'foo-bar', team name 'qwerty'")}, + {"expect error as team name does not match", "foo-bar", "qwerty", "", errors.New("cluster name must match {TEAM}-{NAME} format. Got cluster name 'foo-bar', team name 'qwerty'")}, } var cloneClusterDescriptions = []struct { - in *CloneDescription - err error + about string + in *CloneDescription + err error }{ - {&CloneDescription{"foo+bar", "", "NotEmpty", "", "", "", "", nil}, nil}, - {&CloneDescription{"foo+bar", "", "", "", "", "", "", nil}, + {"cluster name invalid but EndTimeSet is not empty", &CloneDescription{"foo+bar", "", "NotEmpty", "", "", "", "", nil}, nil}, + {"expect error as cluster name does not match DNS-1035", &CloneDescription{"foo+bar", "", "", "", "", "", "", nil}, errors.New(`clone cluster name must confirm to DNS-1035, regex used for validation is "^[a-z]([-a-z0-9]*[a-z0-9])?$"`)}, - {&CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", "", "", "", "", nil}, + {"expect error as cluster name is too long", &CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", "", "", "", "", nil}, errors.New("clone cluster name must be no longer than 63 characters")}, - {&CloneDescription{"foobar", "", "", "", "", "", "", nil}, nil}, + {"common cluster name", &CloneDescription{"foobar", "", "", "", "", "", "", nil}, nil}, } var maintenanceWindows = []struct { - in []byte - out MaintenanceWindow - err error -}{{[]byte(`"Tue:10:00-20:00"`), + about string + in []byte + out MaintenanceWindow + err error +}{{"regular scenario", + []byte(`"Tue:10:00-20:00"`), MaintenanceWindow{ Everyday: false, Weekday: time.Tuesday, StartTime: mustParseTime("10:00"), EndTime: mustParseTime("20:00"), }, nil}, - {[]byte(`"Mon:10:00-10:00"`), + {"starts and ends at the same time", + []byte(`"Mon:10:00-10:00"`), MaintenanceWindow{ Everyday: false, Weekday: time.Monday, StartTime: mustParseTime("10:00"), EndTime: mustParseTime("10:00"), }, nil}, - {[]byte(`"Sun:00:00-00:00"`), + {"starts and ends 00:00 on sunday", + []byte(`"Sun:00:00-00:00"`), MaintenanceWindow{ Everyday: false, Weekday: time.Sunday, StartTime: mustParseTime("00:00"), EndTime: mustParseTime("00:00"), }, nil}, - {[]byte(`"01:00-10:00"`), + {"without day indication should define to sunday", + []byte(`"01:00-10:00"`), MaintenanceWindow{ Everyday: true, Weekday: time.Sunday, StartTime: mustParseTime("01:00"), EndTime: mustParseTime("10:00"), }, nil}, - {[]byte(`"Mon:12:00-11:00"`), MaintenanceWindow{}, errors.New(`'From' time must be prior to the 'To' time`)}, - {[]byte(`"Wed:33:00-00:00"`), MaintenanceWindow{}, errors.New(`could not parse start time: parsing time "33:00": hour out of range`)}, - {[]byte(`"Wed:00:00-26:00"`), MaintenanceWindow{}, errors.New(`could not parse end time: parsing time "26:00": hour out of range`)}, - {[]byte(`"Sunday:00:00-00:00"`), MaintenanceWindow{}, errors.New(`could not parse weekday: incorrect weekday`)}, - {[]byte(`":00:00-10:00"`), MaintenanceWindow{}, errors.New(`could not parse weekday: incorrect weekday`)}, - {[]byte(`"Mon:10:00-00:00"`), MaintenanceWindow{}, errors.New(`'From' time must be prior to the 'To' time`)}, - {[]byte(`"Mon:00:00:00-10:00:00"`), MaintenanceWindow{}, errors.New(`incorrect maintenance window format`)}, - {[]byte(`"Mon:00:00"`), MaintenanceWindow{}, errors.New("incorrect maintenance window format")}, - {[]byte(`"Mon:00:00-00:00:00"`), MaintenanceWindow{}, errors.New("could not parse end time: incorrect time format")}} + {"expect error as 'From' is later than 'To'", []byte(`"Mon:12:00-11:00"`), MaintenanceWindow{}, errors.New(`'From' time must be prior to the 'To' time`)}, + {"expect error as 'From' is later than 'To' with 00:00 corner case", []byte(`"Mon:10:00-00:00"`), MaintenanceWindow{}, errors.New(`'From' time must be prior to the 'To' time`)}, + {"expect error as 'From' time is not valid", []byte(`"Wed:33:00-00:00"`), MaintenanceWindow{}, errors.New(`could not parse start time: parsing time "33:00": hour out of range`)}, + {"expect error as 'To' time is not valid", []byte(`"Wed:00:00-26:00"`), MaintenanceWindow{}, errors.New(`could not parse end time: parsing time "26:00": hour out of range`)}, + {"expect error as weekday is not valid", []byte(`"Sunday:00:00-00:00"`), MaintenanceWindow{}, errors.New(`could not parse weekday: incorrect weekday`)}, + {"expect error as weekday is empty", []byte(`":00:00-10:00"`), MaintenanceWindow{}, errors.New(`could not parse weekday: incorrect weekday`)}, + {"expect error as maintenance window set seconds", []byte(`"Mon:00:00:00-10:00:00"`), MaintenanceWindow{}, errors.New(`incorrect maintenance window format`)}, + {"expect error as 'To' time set seconds", []byte(`"Mon:00:00-00:00:00"`), MaintenanceWindow{}, errors.New("could not parse end time: incorrect time format")}, + {"expect error as 'To' time is missing", []byte(`"Mon:00:00"`), MaintenanceWindow{}, errors.New("incorrect maintenance window format")}} var postgresStatus = []struct { - in []byte - out PostgresStatus - err error + about string + in []byte + out PostgresStatus + err error }{ - {[]byte(`{"PostgresClusterStatus":"Running"}`), + {"cluster running", []byte(`{"PostgresClusterStatus":"Running"}`), PostgresStatus{PostgresClusterStatus: ClusterStatusRunning}, nil}, - {[]byte(`{"PostgresClusterStatus":""}`), + {"cluster status undefined", []byte(`{"PostgresClusterStatus":""}`), PostgresStatus{PostgresClusterStatus: ClusterStatusUnknown}, nil}, - {[]byte(`"Running"`), + {"cluster running without full JSON format", []byte(`"Running"`), PostgresStatus{PostgresClusterStatus: ClusterStatusRunning}, nil}, - {[]byte(`""`), + {"cluster status empty", []byte(`""`), PostgresStatus{PostgresClusterStatus: ClusterStatusUnknown}, nil}} +var tmp postgresqlCopy var unmarshalCluster = []struct { + about string in []byte out Postgresql marshal []byte err error }{ - // example with simple status field { + about: "example with simple status field", in: []byte(`{ "kind": "Postgresql","apiVersion": "acid.zalan.do/v1", "metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), @@ -147,12 +159,14 @@ var unmarshalCluster = []struct { }, Status: PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid}, // This error message can vary between Go versions, so compute it for the current version. - Error: json.Unmarshal([]byte(`{"teamId": 0}`), &PostgresSpec{}).Error(), + Error: json.Unmarshal([]byte(`{ + "kind": "Postgresql","apiVersion": "acid.zalan.do/v1", + "metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), &tmp).Error(), }, marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{}},"status":"Invalid"}`), err: nil}, - // example with /status subresource { + about: "example with /status subresource", in: []byte(`{ "kind": "Postgresql","apiVersion": "acid.zalan.do/v1", "metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), @@ -166,13 +180,14 @@ var unmarshalCluster = []struct { }, Status: PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid}, // This error message can vary between Go versions, so compute it for the current version. - Error: json.Unmarshal([]byte(`{"teamId": 0}`), &PostgresSpec{}).Error(), + Error: json.Unmarshal([]byte(`{ + "kind": "Postgresql","apiVersion": "acid.zalan.do/v1", + "metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), &tmp).Error(), }, marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{}},"status":{"PostgresClusterStatus":"Invalid"}}`), err: nil}, - // example with detailed input manifest - // and deprecated pod_priority_class_name -> podPriorityClassName { + about: "example with detailed input manifest and deprecated pod_priority_class_name -> podPriorityClassName", in: []byte(`{ "kind": "Postgresql", "apiVersion": "acid.zalan.do/v1", @@ -321,9 +336,9 @@ var unmarshalCluster = []struct { }, marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"9.6","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, - // example with teamId set in input { - in: []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "teapot-testcluster1"}, "spec": {"teamId": "acid"}}`), + about: "example with teamId set in input", + in: []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "teapot-testcluster1"}, "spec": {"teamId": "acid"}}`), out: Postgresql{ TypeMeta: metav1.TypeMeta{ Kind: "Postgresql", @@ -338,9 +353,9 @@ var unmarshalCluster = []struct { }, marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"teapot-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null} ,"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{}},"status":{"PostgresClusterStatus":"Invalid"}}`), err: nil}, - // clone example { - in: []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": "acid", "clone": {"cluster": "team-batman"}}}`), + about: "example with clone", + in: []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": "acid", "clone": {"cluster": "team-batman"}}}`), out: Postgresql{ TypeMeta: metav1.TypeMeta{ Kind: "Postgresql", @@ -360,9 +375,9 @@ var unmarshalCluster = []struct { }, marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{"cluster":"team-batman"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, - // standby example { - in: []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": "acid", "standby": {"s3_wal_path": "s3://custom/path/to/bucket/"}}}`), + about: "standby example", + in: []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": "acid", "standby": {"s3_wal_path": "s3://custom/path/to/bucket/"}}}`), out: Postgresql{ TypeMeta: metav1.TypeMeta{ Kind: "Postgresql", @@ -382,24 +397,28 @@ var unmarshalCluster = []struct { }, marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"standby":{"s3_wal_path":"s3://custom/path/to/bucket/"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, - // erroneous examples { + about: "expect error on malformatted JSON", in: []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1"`), out: Postgresql{}, marshal: []byte{}, err: errors.New("unexpected end of JSON input")}, { + about: "expect error on JSON with field's value malformatted", in: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster","creationTimestamp":qaz},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{}},"status":{"PostgresClusterStatus":"Invalid"}}`), out: Postgresql{}, marshal: []byte{}, - err: errors.New("invalid character 'q' looking for beginning of value")}} + err: errors.New("invalid character 'q' looking for beginning of value"), + }, +} var postgresqlList = []struct { - in []byte - out PostgresqlList - err error + about string + in []byte + out PostgresqlList + err error }{ - {[]byte(`{"apiVersion":"v1","items":[{"apiVersion":"acid.zalan.do/v1","kind":"Postgresql","metadata":{"labels":{"team":"acid"},"name":"acid-testcluster42","namespace":"default","resourceVersion":"30446957","selfLink":"/apis/acid.zalan.do/v1/namespaces/default/postgresqls/acid-testcluster42","uid":"857cd208-33dc-11e7-b20a-0699041e4b03"},"spec":{"allowedSourceRanges":["185.85.220.0/22"],"numberOfInstances":1,"postgresql":{"version":"9.6"},"teamId":"acid","volume":{"size":"10Gi"}},"status":{"PostgresClusterStatus":"Running"}}],"kind":"List","metadata":{},"resourceVersion":"","selfLink":""}`), + {"expect success", []byte(`{"apiVersion":"v1","items":[{"apiVersion":"acid.zalan.do/v1","kind":"Postgresql","metadata":{"labels":{"team":"acid"},"name":"acid-testcluster42","namespace":"default","resourceVersion":"30446957","selfLink":"/apis/acid.zalan.do/v1/namespaces/default/postgresqls/acid-testcluster42","uid":"857cd208-33dc-11e7-b20a-0699041e4b03"},"spec":{"allowedSourceRanges":["185.85.220.0/22"],"numberOfInstances":1,"postgresql":{"version":"9.6"},"teamId":"acid","volume":{"size":"10Gi"}},"status":{"PostgresClusterStatus":"Running"}}],"kind":"List","metadata":{},"resourceVersion":"","selfLink":""}`), PostgresqlList{ TypeMeta: metav1.TypeMeta{ Kind: "List", @@ -433,15 +452,17 @@ var postgresqlList = []struct { }}, }, nil}, - {[]byte(`{"apiVersion":"v1","items":[{"apiVersion":"acid.zalan.do/v1","kind":"Postgresql","metadata":{"labels":{"team":"acid"},"name":"acid-testcluster42","namespace"`), + {"expect error on malformatted JSON", []byte(`{"apiVersion":"v1","items":[{"apiVersion":"acid.zalan.do/v1","kind":"Postgresql","metadata":{"labels":{"team":"acid"},"name":"acid-testcluster42","namespace"`), PostgresqlList{}, errors.New("unexpected end of JSON input")}} var annotations = []struct { + about string in []byte annotations map[string]string err error }{{ + about: "common annotations", in: []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "acid-testcluster1"}, "spec": {"podAnnotations": {"foo": "bar"},"teamId": "acid", "clone": {"cluster": "team-batman"}}}`), annotations: map[string]string{"foo": "bar"}, err: nil}, @@ -458,230 +479,256 @@ func mustParseTime(s string) metav1.Time { func TestParseTime(t *testing.T) { for _, tt := range parseTimeTests { - aTime, err := parseTime(tt.in) - if err != nil { - if tt.err == nil || err.Error() != tt.err.Error() { - t.Errorf("ParseTime expected error: %v, got: %v", tt.err, err) + t.Run(tt.about, func(t *testing.T) { + aTime, err := parseTime(tt.in) + if err != nil { + if tt.err == nil || err.Error() != tt.err.Error() { + t.Errorf("ParseTime expected error: %v, got: %v", tt.err, err) + } + return + } else if tt.err != nil { + t.Errorf("Expected error: %v", tt.err) } - continue - } else if tt.err != nil { - t.Errorf("Expected error: %v", tt.err) - } - if aTime != tt.out { - t.Errorf("Expected time: %v, got: %v", tt.out, aTime) - } + if aTime != tt.out { + t.Errorf("Expected time: %v, got: %v", tt.out, aTime) + } + }) } } func TestWeekdayTime(t *testing.T) { for _, tt := range parseWeekdayTests { - aTime, err := parseWeekday(tt.in) - if err != nil { - if tt.err == nil || err.Error() != tt.err.Error() { - t.Errorf("ParseWeekday expected error: %v, got: %v", tt.err, err) + t.Run(tt.about, func(t *testing.T) { + aTime, err := parseWeekday(tt.in) + if err != nil { + if tt.err == nil || err.Error() != tt.err.Error() { + t.Errorf("ParseWeekday expected error: %v, got: %v", tt.err, err) + } + return + } else if tt.err != nil { + t.Errorf("Expected error: %v", tt.err) } - continue - } else if tt.err != nil { - t.Errorf("Expected error: %v", tt.err) - } - if aTime != tt.out { - t.Errorf("Expected weekday: %v, got: %v", tt.out, aTime) - } + if aTime != tt.out { + t.Errorf("Expected weekday: %v, got: %v", tt.out, aTime) + } + }) } } func TestClusterAnnotations(t *testing.T) { for _, tt := range annotations { - var cluster Postgresql - err := cluster.UnmarshalJSON(tt.in) - if err != nil { - if tt.err == nil || err.Error() != tt.err.Error() { - t.Errorf("Unable to marshal cluster with annotations: expected %v got %v", tt.err, err) + t.Run(tt.about, func(t *testing.T) { + var cluster Postgresql + err := cluster.UnmarshalJSON(tt.in) + if err != nil { + if tt.err == nil || err.Error() != tt.err.Error() { + t.Errorf("Unable to marshal cluster with annotations: expected %v got %v", tt.err, err) + } + return } - continue - } - for k, v := range cluster.Spec.PodAnnotations { - found, expected := v, tt.annotations[k] - if found != expected { - t.Errorf("Didn't find correct value for key %v in for podAnnotations: Expected %v found %v", k, expected, found) + for k, v := range cluster.Spec.PodAnnotations { + found, expected := v, tt.annotations[k] + if found != expected { + t.Errorf("Didn't find correct value for key %v in for podAnnotations: Expected %v found %v", k, expected, found) + } } - } + }) } } func TestClusterName(t *testing.T) { for _, tt := range clusterNames { - name, err := extractClusterName(tt.in, tt.inTeam) - if err != nil { - if tt.err == nil || err.Error() != tt.err.Error() { - t.Errorf("extractClusterName expected error: %v, got: %v", tt.err, err) + t.Run(tt.about, func(t *testing.T) { + name, err := extractClusterName(tt.in, tt.inTeam) + if err != nil { + if tt.err == nil || err.Error() != tt.err.Error() { + t.Errorf("extractClusterName expected error: %v, got: %v", tt.err, err) + } + return + } else if tt.err != nil { + t.Errorf("Expected error: %v", tt.err) } - continue - } else if tt.err != nil { - t.Errorf("Expected error: %v", tt.err) - } - if name != tt.clusterName { - t.Errorf("Expected cluserName: %q, got: %q", tt.clusterName, name) - } + if name != tt.clusterName { + t.Errorf("Expected cluserName: %q, got: %q", tt.clusterName, name) + } + }) } } func TestCloneClusterDescription(t *testing.T) { for _, tt := range cloneClusterDescriptions { - if err := validateCloneClusterDescription(tt.in); err != nil { - if tt.err == nil || err.Error() != tt.err.Error() { - t.Errorf("testCloneClusterDescription expected error: %v, got: %v", tt.err, err) + t.Run(tt.about, func(t *testing.T) { + if err := validateCloneClusterDescription(tt.in); err != nil { + if tt.err == nil || err.Error() != tt.err.Error() { + t.Errorf("testCloneClusterDescription expected error: %v, got: %v", tt.err, err) + } + } else if tt.err != nil { + t.Errorf("Expected error: %v", tt.err) } - } else if tt.err != nil { - t.Errorf("Expected error: %v", tt.err) - } + }) } } func TestUnmarshalMaintenanceWindow(t *testing.T) { for _, tt := range maintenanceWindows { - var m MaintenanceWindow - err := m.UnmarshalJSON(tt.in) - if err != nil { - if tt.err == nil || err.Error() != tt.err.Error() { - t.Errorf("MaintenanceWindow unmarshal expected error: %v, got %v", tt.err, err) + t.Run(tt.about, func(t *testing.T) { + var m MaintenanceWindow + err := m.UnmarshalJSON(tt.in) + if err != nil { + if tt.err == nil || err.Error() != tt.err.Error() { + t.Errorf("MaintenanceWindow unmarshal expected error: %v, got %v", tt.err, err) + } + return + } else if tt.err != nil { + t.Errorf("Expected error: %v", tt.err) } - continue - } else if tt.err != nil { - t.Errorf("Expected error: %v", tt.err) - } - if !reflect.DeepEqual(m, tt.out) { - t.Errorf("Expected maintenance window: %#v, got: %#v", tt.out, m) - } + if !reflect.DeepEqual(m, tt.out) { + t.Errorf("Expected maintenance window: %#v, got: %#v", tt.out, m) + } + }) } } func TestMarshalMaintenanceWindow(t *testing.T) { for _, tt := range maintenanceWindows { - if tt.err != nil { - continue - } + t.Run(tt.about, func(t *testing.T) { + if tt.err != nil { + return + } - s, err := tt.out.MarshalJSON() - if err != nil { - t.Errorf("Marshal Error: %v", err) - } + s, err := tt.out.MarshalJSON() + if err != nil { + t.Errorf("Marshal Error: %v", err) + } - if !bytes.Equal(s, tt.in) { - t.Errorf("Expected Marshal: %q, got: %q", string(tt.in), string(s)) - } + if !bytes.Equal(s, tt.in) { + t.Errorf("Expected Marshal: %q, got: %q", string(tt.in), string(s)) + } + }) } } func TestUnmarshalPostgresStatus(t *testing.T) { for _, tt := range postgresStatus { - var ps PostgresStatus - err := ps.UnmarshalJSON(tt.in) - if err != nil { - if tt.err == nil || err.Error() != tt.err.Error() { - t.Errorf("CR status unmarshal expected error: %v, got %v", tt.err, err) - } - continue - //} else if tt.err != nil { - //t.Errorf("Expected error: %v", tt.err) - } + t.Run(tt.about, func(t *testing.T) { - if !reflect.DeepEqual(ps, tt.out) { - t.Errorf("Expected status: %#v, got: %#v", tt.out, ps) - } + var ps PostgresStatus + err := ps.UnmarshalJSON(tt.in) + if err != nil { + if tt.err == nil || err.Error() != tt.err.Error() { + t.Errorf("CR status unmarshal expected error: %v, got %v", tt.err, err) + } + return + } + + if !reflect.DeepEqual(ps, tt.out) { + t.Errorf("Expected status: %#v, got: %#v", tt.out, ps) + } + }) } } func TestPostgresUnmarshal(t *testing.T) { for _, tt := range unmarshalCluster { - var cluster Postgresql - err := cluster.UnmarshalJSON(tt.in) - if err != nil { - if tt.err == nil || err.Error() != tt.err.Error() { - t.Errorf("Unmarshal expected error: %v, got: %v", tt.err, err) + t.Run(tt.about, func(t *testing.T) { + var cluster Postgresql + err := cluster.UnmarshalJSON(tt.in) + if err != nil { + if tt.err == nil || err.Error() != tt.err.Error() { + t.Errorf("Unmarshal expected error: %v, got: %v", tt.err, err) + } + return + } else if tt.err != nil { + t.Errorf("Expected error: %v", tt.err) } - continue - } else if tt.err != nil { - t.Errorf("Expected error: %v", tt.err) - } - if !reflect.DeepEqual(cluster, tt.out) { - t.Errorf("Expected Postgresql: %#v, got %#v", tt.out, cluster) - } + if !reflect.DeepEqual(cluster, tt.out) { + t.Errorf("Expected Postgresql: %#v, got %#v", tt.out, cluster) + } + }) } } func TestMarshal(t *testing.T) { for _, tt := range unmarshalCluster { - if tt.err != nil { - continue - } + t.Run(tt.about, func(t *testing.T) { - // Unmarshal and marshal example to capture api changes - var cluster Postgresql - err := cluster.UnmarshalJSON(tt.marshal) - if err != nil { - if tt.err == nil || err.Error() != tt.err.Error() { - t.Errorf("Backwards compatibility unmarshal expected error: %v, got: %v", tt.err, err) + if tt.err != nil { + return } - continue - } - expected, err := json.Marshal(cluster) - if err != nil { - t.Errorf("Backwards compatibility marshal error: %v", err) - } - m, err := json.Marshal(tt.out) - if err != nil { - t.Errorf("Marshal error: %v", err) - } - if !bytes.Equal(m, expected) { - t.Errorf("Marshal Postgresql \nexpected: %q, \ngot: %q", string(expected), string(m)) - } + // Unmarshal and marshal example to capture api changes + var cluster Postgresql + err := cluster.UnmarshalJSON(tt.marshal) + if err != nil { + if tt.err == nil || err.Error() != tt.err.Error() { + t.Errorf("Backwards compatibility unmarshal expected error: %v, got: %v", tt.err, err) + } + return + } + expected, err := json.Marshal(cluster) + if err != nil { + t.Errorf("Backwards compatibility marshal error: %v", err) + } + + m, err := json.Marshal(tt.out) + if err != nil { + t.Errorf("Marshal error: %v", err) + } + if !bytes.Equal(m, expected) { + t.Errorf("Marshal Postgresql \nexpected: %q, \ngot: %q", string(expected), string(m)) + } + }) } } func TestPostgresMeta(t *testing.T) { for _, tt := range unmarshalCluster { - if a := tt.out.GetObjectKind(); a != &tt.out.TypeMeta { - t.Errorf("GetObjectKindMeta \nexpected: %v, \ngot: %v", tt.out.TypeMeta, a) - } + t.Run(tt.about, func(t *testing.T) { - if a := tt.out.GetObjectMeta(); reflect.DeepEqual(a, tt.out.ObjectMeta) { - t.Errorf("GetObjectMeta \nexpected: %v, \ngot: %v", tt.out.ObjectMeta, a) - } + if a := tt.out.GetObjectKind(); a != &tt.out.TypeMeta { + t.Errorf("GetObjectKindMeta \nexpected: %v, \ngot: %v", tt.out.TypeMeta, a) + } + + if a := tt.out.GetObjectMeta(); reflect.DeepEqual(a, tt.out.ObjectMeta) { + t.Errorf("GetObjectMeta \nexpected: %v, \ngot: %v", tt.out.ObjectMeta, a) + } + }) } } func TestPostgresListMeta(t *testing.T) { for _, tt := range postgresqlList { - if tt.err != nil { - continue - } + t.Run(tt.about, func(t *testing.T) { + if tt.err != nil { + return + } - if a := tt.out.GetObjectKind(); a != &tt.out.TypeMeta { - t.Errorf("GetObjectKindMeta expected: %v, got: %v", tt.out.TypeMeta, a) - } + if a := tt.out.GetObjectKind(); a != &tt.out.TypeMeta { + t.Errorf("GetObjectKindMeta expected: %v, got: %v", tt.out.TypeMeta, a) + } - if a := tt.out.GetListMeta(); reflect.DeepEqual(a, tt.out.ListMeta) { - t.Errorf("GetObjectMeta expected: %v, got: %v", tt.out.ListMeta, a) - } + if a := tt.out.GetListMeta(); reflect.DeepEqual(a, tt.out.ListMeta) { + t.Errorf("GetObjectMeta expected: %v, got: %v", tt.out.ListMeta, a) + } - return + return + }) } } func TestPostgresqlClone(t *testing.T) { for _, tt := range unmarshalCluster { - cp := &tt.out - cp.Error = "" - clone := cp.Clone() - if !reflect.DeepEqual(clone, cp) { - t.Errorf("TestPostgresqlClone expected: \n%#v\n, got \n%#v", cp, clone) - } - + t.Run(tt.about, func(t *testing.T) { + cp := &tt.out + cp.Error = "" + clone := cp.Clone() + if !reflect.DeepEqual(clone, cp) { + t.Errorf("TestPostgresqlClone expected: \n%#v\n, got \n%#v", cp, clone) + } + }) } } From 1f0312a01451b544231089dd6b703b93e68bd2c5 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 3 Feb 2020 11:43:18 +0100 Subject: [PATCH 035/104] make minimum limits boundaries configurable (#808) * make minimum limits boundaries configurable * add e2e test --- .../crds/operatorconfigurations.yaml | 6 ++ charts/postgres-operator/values-crd.yaml | 26 ++++--- charts/postgres-operator/values.yaml | 24 ++++--- docs/reference/operator_parameters.md | 14 +++- docs/user.md | 10 +-- e2e/tests/test_e2e.py | 68 ++++++++++++++++--- manifests/complete-postgres-manifest.yaml | 4 +- manifests/configmap.yaml | 6 +- manifests/operatorconfiguration.crd.yaml | 6 ++ manifests/postgres-operator.yaml | 4 +- ...gresql-operator-default-configuration.yaml | 8 ++- pkg/apis/acid.zalan.do/v1/crds.go | 8 +++ .../v1/operator_configuration_type.go | 2 + pkg/cluster/cluster.go | 56 +++++++-------- pkg/cluster/sync.go | 16 ++--- pkg/controller/operator_config.go | 2 + pkg/util/config/config.go | 8 ++- 17 files changed, 175 insertions(+), 93 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index c97e246ab..52d03df9c 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -179,6 +179,12 @@ spec: default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + min_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + min_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' timeouts: type: object properties: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 5b67258fa..61cab3d06 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -115,13 +115,17 @@ configKubernetes: # configure resource requests for the Postgres pods configPostgresPodResources: # CPU limits for the postgres containers - default_cpu_limit: "3" - # cpu request value for the postgres containers + default_cpu_limit: "1" + # CPU request value for the postgres containers default_cpu_request: 100m # memory limits for the postgres containers - default_memory_limit: 1Gi + default_memory_limit: 500Mi # memory request value for the postgres containers default_memory_request: 100Mi + # hard CPU minimum required to properly run a Postgres cluster + min_cpu_limit: 250m + # hard memory minimum required to properly run a Postgres cluster + min_memory_limit: 250Mi # timeouts related to some operator actions configTimeouts: @@ -251,7 +255,7 @@ configScalyr: # CPU rquest value for the Scalyr sidecar scalyr_cpu_request: 100m # Memory limit value for the Scalyr sidecar - scalyr_memory_limit: 1Gi + scalyr_memory_limit: 500Mi # Memory request value for the Scalyr sidecar scalyr_memory_request: 50Mi @@ -272,13 +276,13 @@ serviceAccount: priorityClassName: "" -resources: {} - # limits: - # cpu: 100m - # memory: 300Mi - # requests: - # cpu: 100m - # memory: 300Mi +resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 250Mi # Affinity for pod assignment # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 60190f49a..deb506329 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -108,13 +108,17 @@ configKubernetes: # configure resource requests for the Postgres pods configPostgresPodResources: # CPU limits for the postgres containers - default_cpu_limit: "3" - # cpu request value for the postgres containers + default_cpu_limit: "1" + # CPU request value for the postgres containers default_cpu_request: 100m # memory limits for the postgres containers - default_memory_limit: 1Gi + default_memory_limit: 500Mi # memory request value for the postgres containers default_memory_request: 100Mi + # hard CPU minimum required to properly run a Postgres cluster + min_cpu_limit: 250m + # hard memory minimum required to properly run a Postgres cluster + min_memory_limit: 250Mi # timeouts related to some operator actions configTimeouts: @@ -248,13 +252,13 @@ serviceAccount: priorityClassName: "" -resources: {} - # limits: - # cpu: 100m - # memory: 300Mi - # requests: - # cpu: 100m - # memory: 300Mi +resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 250Mi # Affinity for pod assignment # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 1055d89b6..d6dde8c0e 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -318,11 +318,19 @@ CRD-based configuration. * **default_cpu_limit** CPU limits for the Postgres containers, unless overridden by cluster-specific - settings. The default is `3`. + settings. The default is `1`. * **default_memory_limit** memory limits for the Postgres containers, unless overridden by cluster-specific - settings. The default is `1Gi`. + settings. The default is `500Mi`. + +* **min_cpu_limit** + hard CPU minimum what we consider to be required to properly run Postgres + clusters with Patroni on Kubernetes. The default is `250m`. + +* **min_memory_limit** + hard memory minimum what we consider to be required to properly run Postgres + clusters with Patroni on Kubernetes. The default is `250Mi`. ## Operator timeouts @@ -579,4 +587,4 @@ scalyr sidecar. In the CRD-based configuration they are grouped under the CPU limit value for the Scalyr sidecar. The default is `1`. * **scalyr_memory_limit** - Memory limit value for the Scalyr sidecar. The default is `1Gi`. + Memory limit value for the Scalyr sidecar. The default is `500Mi`. diff --git a/docs/user.md b/docs/user.md index 45f345c87..f81e11ede 100644 --- a/docs/user.md +++ b/docs/user.md @@ -232,11 +232,11 @@ spec: memory: 300Mi ``` -The minimum limit to properly run the `postgresql` resource is `256m` for `cpu` -and `256Mi` for `memory`. If a lower value is set in the manifest the operator -will cancel ADD or UPDATE events on this resource with an error. If no -resources are defined in the manifest the operator will obtain the configured -[default requests](reference/operator_parameters.md#kubernetes-resource-requests). +The minimum limits to properly run the `postgresql` resource are configured to +`250m` for `cpu` and `250Mi` for `memory`. If a lower value is set in the +manifest the operator will raise the limits to the configured minimum values. +If no resources are defined in the manifest they will be obtained from the +configured [default requests](reference/operator_parameters.md#kubernetes-resource-requests). ## Use taints and tolerations for dedicated PostgreSQL nodes diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 88a7f1f34..fc87c9887 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -58,6 +58,57 @@ class EndToEndTestCase(unittest.TestCase): k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") k8s.wait_for_pod_start('spilo-role=master') + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_min_resource_limits(self): + ''' + Lower resource limits below configured minimum and let operator fix it + ''' + k8s = self.k8s + cluster_label = 'version=acid-minimal-cluster' + _, failover_targets = k8s.get_pg_nodes(cluster_label) + + # configure minimum boundaries for CPU and memory limits + minCPULimit = '250m' + minMemoryLimit = '250Mi' + patch_min_resource_limits = { + "data": { + "min_cpu_limit": minCPULimit, + "min_memory_limit": minMemoryLimit + } + } + k8s.update_config(patch_min_resource_limits) + + # lower resource limits below minimum + pg_patch_resources = { + "spec": { + "resources": { + "requests": { + "cpu": "10m", + "memory": "50Mi" + }, + "limits": { + "cpu": "200m", + "memory": "200Mi" + } + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) + k8s.wait_for_master_failover(failover_targets) + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector='spilo-role=master,' + cluster_label).items + self.assert_master_is_unique() + masterPod = pods[0] + + self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, + "Expected CPU limit {}, found {}" + .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) + self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, + "Expected memory limit {}, found {}" + .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_multi_namespace_support(self): ''' @@ -76,10 +127,9 @@ class EndToEndTestCase(unittest.TestCase): @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_scaling(self): - """ + ''' Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. - """ - + ''' k8s = self.k8s labels = "version=acid-minimal-cluster" @@ -93,9 +143,9 @@ class EndToEndTestCase(unittest.TestCase): @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): - """ + ''' Add taint "postgres=:NoExecute" to node with master. This must cause a failover. - """ + ''' k8s = self.k8s cluster_label = 'version=acid-minimal-cluster' @@ -145,7 +195,7 @@ class EndToEndTestCase(unittest.TestCase): @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_logical_backup_cron_job(self): - """ + ''' Ensure we can (a) create the cron job at user request for a specific PG cluster (b) update the cluster-wide image for the logical backup pod (c) delete the job at user request @@ -153,7 +203,7 @@ class EndToEndTestCase(unittest.TestCase): Limitations: (a) Does not run the actual batch job because there is no S3 mock to upload backups to (b) Assumes 'acid-minimal-cluster' exists as defined in setUp - """ + ''' k8s = self.k8s @@ -208,10 +258,10 @@ class EndToEndTestCase(unittest.TestCase): "Expected 0 logical backup jobs, found {}".format(len(jobs))) def assert_master_is_unique(self, namespace='default', version="acid-minimal-cluster"): - """ + ''' Check that there is a single pod in the k8s cluster with the label "spilo-role=master" To be called manually after operations that affect pods - """ + ''' k8s = self.k8s labels = 'spilo-role=master,version=' + version diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index cf450ef94..2478156d6 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -42,8 +42,8 @@ spec: cpu: 10m memory: 100Mi limits: - cpu: 300m - memory: 300Mi + cpu: 500m + memory: 500Mi patroni: initdb: encoding: "UTF8" diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index afb5957da..7d11198da 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,9 +15,9 @@ data: # custom_pod_annotations: "keya:valuea,keyb:valueb" db_hosted_zone: db.example.com debug_logging: "true" - # default_cpu_limit: "3" + # default_cpu_limit: "1" # default_cpu_request: 100m - # default_memory_limit: 1Gi + # default_memory_limit: 500Mi # default_memory_request: 100Mi docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p16 # enable_admin_role_for_users: "true" @@ -48,6 +48,8 @@ data: # master_pod_move_timeout: 10m # max_instances: "-1" # min_instances: "-1" + # min_cpu_limit: 250m + # min_memory_limit: 250Mi # node_readiness_label: "" # oauth_token_secret_name: postgresql-operator # pam_configuration: | diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 810624bc4..509d9aefc 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -155,6 +155,12 @@ spec: default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + min_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + min_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' timeouts: type: object properties: diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index fa8682809..a06abfc68 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -19,10 +19,10 @@ spec: imagePullPolicy: IfNotPresent resources: requests: - cpu: 500m + cpu: 100m memory: 250Mi limits: - cpu: 2000m + cpu: 500m memory: 500Mi securityContext: runAsUser: 1000 diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 5e10ff66e..f13a1eed9 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -54,10 +54,12 @@ configuration: # toleration: {} # watched_namespace: "" postgres_pod_resources: - default_cpu_limit: "3" + default_cpu_limit: "1" default_cpu_request: 100m - default_memory_limit: 1Gi + default_memory_limit: 500Mi default_memory_request: 100Mi + # min_cpu_limit: 250m + # min_memory_limit: 250Mi timeouts: pod_label_wait_timeout: 10m pod_deletion_wait_timeout: 10m @@ -115,6 +117,6 @@ configuration: scalyr_cpu_limit: "1" scalyr_cpu_request: 100m # scalyr_image: "" - scalyr_memory_limit: 1Gi + scalyr_memory_limit: 500Mi scalyr_memory_request: 50Mi # scalyr_server_url: "" diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 20fa37138..4d5a6f024 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -810,6 +810,14 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation Type: "string", Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", }, + "min_cpu_limit": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "min_memory_limit": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, }, }, "timeouts": { diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 948c7cbbf..1e6a3b459 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -79,6 +79,8 @@ type PostgresPodResourcesDefaults struct { DefaultMemoryRequest string `json:"default_memory_request,omitempty"` DefaultCPULimit string `json:"default_cpu_limit,omitempty"` DefaultMemoryLimit string `json:"default_memory_limit,omitempty"` + MinCPULimit string `json:"min_cpu_limit,omitempty"` + MinMemoryLimit string `json:"min_memory_limit,omitempty"` } // OperatorTimeouts defines the timeout of ResourceCheck, PodWait, ReadyWait diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 0a7377389..c560c4cdf 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -227,8 +227,8 @@ func (c *Cluster) Create() error { c.setStatus(acidv1.ClusterStatusCreating) - if err = c.validateResources(&c.Spec); err != nil { - return fmt.Errorf("insufficient resource limits specified: %v", err) + if err = c.enforceMinResourceLimits(&c.Spec); err != nil { + return fmt.Errorf("could not enforce minimum resource limits: %v", err) } for _, role := range []PostgresRole{Master, Replica} { @@ -495,38 +495,38 @@ func compareResourcesAssumeFirstNotNil(a *v1.ResourceRequirements, b *v1.Resourc } -func (c *Cluster) validateResources(spec *acidv1.PostgresSpec) error { - - // setting limits too low can cause unnecessary evictions / OOM kills - const ( - cpuMinLimit = "256m" - memoryMinLimit = "256Mi" - ) +func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error { var ( isSmaller bool err error ) + // setting limits too low can cause unnecessary evictions / OOM kills + minCPULimit := c.OpConfig.MinCPULimit + minMemoryLimit := c.OpConfig.MinMemoryLimit + cpuLimit := spec.Resources.ResourceLimits.CPU if cpuLimit != "" { - isSmaller, err = util.IsSmallerQuantity(cpuLimit, cpuMinLimit) + isSmaller, err = util.IsSmallerQuantity(cpuLimit, minCPULimit) if err != nil { - return fmt.Errorf("error validating CPU limit: %v", err) + return fmt.Errorf("could not compare defined CPU limit %s with configured minimum value %s: %v", cpuLimit, minCPULimit, err) } if isSmaller { - return fmt.Errorf("defined CPU limit %s is below required minimum %s to properly run postgresql resource", cpuLimit, cpuMinLimit) + c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit) + spec.Resources.ResourceLimits.CPU = minCPULimit } } memoryLimit := spec.Resources.ResourceLimits.Memory if memoryLimit != "" { - isSmaller, err = util.IsSmallerQuantity(memoryLimit, memoryMinLimit) + isSmaller, err = util.IsSmallerQuantity(memoryLimit, minMemoryLimit) if err != nil { - return fmt.Errorf("error validating memory limit: %v", err) + return fmt.Errorf("could not compare defined memory limit %s with configured minimum value %s: %v", memoryLimit, minMemoryLimit, err) } if isSmaller { - return fmt.Errorf("defined memory limit %s is below required minimum %s to properly run postgresql resource", memoryLimit, memoryMinLimit) + c.logger.Warningf("defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit) + spec.Resources.ResourceLimits.Memory = minMemoryLimit } } @@ -543,7 +543,6 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { c.mu.Lock() defer c.mu.Unlock() - oldStatus := c.Status c.setStatus(acidv1.ClusterStatusUpdating) c.setSpec(newSpec) @@ -555,22 +554,6 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } }() - if err := c.validateResources(&newSpec.Spec); err != nil { - err = fmt.Errorf("insufficient resource limits specified: %v", err) - - // cancel update only when (already too low) pod resources were edited - // if cluster was successfully running before the update, continue but log a warning - isCPULimitSmaller, err2 := util.IsSmallerQuantity(newSpec.Spec.Resources.ResourceLimits.CPU, oldSpec.Spec.Resources.ResourceLimits.CPU) - isMemoryLimitSmaller, err3 := util.IsSmallerQuantity(newSpec.Spec.Resources.ResourceLimits.Memory, oldSpec.Spec.Resources.ResourceLimits.Memory) - - if oldStatus.Running() && !isCPULimitSmaller && !isMemoryLimitSmaller && err2 == nil && err3 == nil { - c.logger.Warning(err) - } else { - updateFailed = true - return err - } - } - if oldSpec.Spec.PgVersion != newSpec.Spec.PgVersion { // PG versions comparison c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PgVersion, newSpec.Spec.PgVersion) //we need that hack to generate statefulset with the old version @@ -616,6 +599,12 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { // Statefulset func() { + if err := c.enforceMinResourceLimits(&c.Spec); err != nil { + c.logger.Errorf("could not sync resources: %v", err) + updateFailed = true + return + } + oldSs, err := c.generateStatefulSet(&oldSpec.Spec) if err != nil { c.logger.Errorf("could not generate old statefulset spec: %v", err) @@ -623,6 +612,9 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { return } + // update newSpec to for latter comparison with oldSpec + c.enforceMinResourceLimits(&newSpec.Spec) + newSs, err := c.generateStatefulSet(&newSpec.Spec) if err != nil { c.logger.Errorf("could not generate new statefulset spec: %v", err) diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index abe579fb5..fa4fc9ec1 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -23,7 +23,6 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { c.mu.Lock() defer c.mu.Unlock() - oldStatus := c.Status c.setSpec(newSpec) defer func() { @@ -35,16 +34,6 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } }() - if err = c.validateResources(&c.Spec); err != nil { - err = fmt.Errorf("insufficient resource limits specified: %v", err) - if oldStatus.Running() { - c.logger.Warning(err) - err = nil - } else { - return err - } - } - if err = c.initUsers(); err != nil { err = fmt.Errorf("could not init users: %v", err) return err @@ -76,6 +65,11 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return err } + if err = c.enforceMinResourceLimits(&c.Spec); err != nil { + err = fmt.Errorf("could not enforce minimum resource limits: %v", err) + return err + } + c.logger.Debugf("syncing statefulsets") if err = c.syncStatefulSet(); err != nil { if !k8sutil.ResourceAlreadyExists(err) { diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 56ba91d02..d0357d222 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -75,6 +75,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.DefaultMemoryRequest = fromCRD.PostgresPodResources.DefaultMemoryRequest result.DefaultCPULimit = fromCRD.PostgresPodResources.DefaultCPULimit result.DefaultMemoryLimit = fromCRD.PostgresPodResources.DefaultMemoryLimit + result.MinCPULimit = fromCRD.PostgresPodResources.MinCPULimit + result.MinMemoryLimit = fromCRD.PostgresPodResources.MinMemoryLimit // timeout config result.ResourceCheckInterval = time.Duration(fromCRD.Timeouts.ResourceCheckInterval) diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 224691120..339f06ce0 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -37,8 +37,10 @@ type Resources struct { PodToleration map[string]string `name:"toleration" default:""` DefaultCPURequest string `name:"default_cpu_request" default:"100m"` DefaultMemoryRequest string `name:"default_memory_request" default:"100Mi"` - DefaultCPULimit string `name:"default_cpu_limit" default:"3"` - DefaultMemoryLimit string `name:"default_memory_limit" default:"1Gi"` + DefaultCPULimit string `name:"default_cpu_limit" default:"1"` + DefaultMemoryLimit string `name:"default_memory_limit" default:"500Mi"` + MinCPULimit string `name:"min_cpu_limit" default:"250m"` + MinMemoryLimit string `name:"min_memory_limit" default:"250Mi"` PodEnvironmentConfigMap string `name:"pod_environment_configmap" default:""` NodeReadinessLabel map[string]string `name:"node_readiness_label" default:""` MaxInstances int32 `name:"max_instances" default:"-1"` @@ -66,7 +68,7 @@ type Scalyr struct { ScalyrCPURequest string `name:"scalyr_cpu_request" default:"100m"` ScalyrMemoryRequest string `name:"scalyr_memory_request" default:"50Mi"` ScalyrCPULimit string `name:"scalyr_cpu_limit" default:"1"` - ScalyrMemoryLimit string `name:"scalyr_memory_limit" default:"1Gi"` + ScalyrMemoryLimit string `name:"scalyr_memory_limit" default:"500Mi"` } // LogicalBackup defines configuration for logical backup From 8794e4f9acad87c8acbb0e4dd13b952fcd38f89e Mon Sep 17 00:00:00 2001 From: siku4 <44839490+siku4@users.noreply.github.com> Date: Mon, 3 Feb 2020 15:30:26 +0100 Subject: [PATCH 036/104] add service for exposing operator api (#794) --- .../postgres-operator/templates/service.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 charts/postgres-operator/templates/service.yaml diff --git a/charts/postgres-operator/templates/service.yaml b/charts/postgres-operator/templates/service.yaml new file mode 100644 index 000000000..52990c5d4 --- /dev/null +++ b/charts/postgres-operator/templates/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: {{ template "postgres-operator.name" . }} + helm.sh/chart: {{ template "postgres-operator.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "postgres-operator.fullname" . }} +spec: + ports: + - port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: {{ template "postgres-operator.name" . }} + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} \ No newline at end of file From 1ee99b81944752a0d6c3e6cc4d71f4bd662601e6 Mon Sep 17 00:00:00 2001 From: siku4 <44839490+siku4@users.noreply.github.com> Date: Wed, 5 Feb 2020 16:54:46 +0100 Subject: [PATCH 037/104] add first version of ui helm chart (#795) * add first version of ui helm chart * add postgres versions 12 and 9.5 --- charts/postgres-operator-ui/.helmignore | 22 +++++ charts/postgres-operator-ui/Chart.yaml | 19 +++++ .../postgres-operator-ui/templates/NOTES.txt | 3 + .../templates/_helpers.tpl | 32 ++++++++ .../templates/deployment.yaml | 69 ++++++++++++++++ .../templates/ingress.yaml | 44 ++++++++++ .../templates/service.yaml | 20 +++++ .../templates/serviceaccount.yaml | 81 +++++++++++++++++++ charts/postgres-operator-ui/values.yaml | 47 +++++++++++ 9 files changed, 337 insertions(+) create mode 100644 charts/postgres-operator-ui/.helmignore create mode 100644 charts/postgres-operator-ui/Chart.yaml create mode 100644 charts/postgres-operator-ui/templates/NOTES.txt create mode 100644 charts/postgres-operator-ui/templates/_helpers.tpl create mode 100644 charts/postgres-operator-ui/templates/deployment.yaml create mode 100644 charts/postgres-operator-ui/templates/ingress.yaml create mode 100644 charts/postgres-operator-ui/templates/service.yaml create mode 100644 charts/postgres-operator-ui/templates/serviceaccount.yaml create mode 100644 charts/postgres-operator-ui/values.yaml diff --git a/charts/postgres-operator-ui/.helmignore b/charts/postgres-operator-ui/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/charts/postgres-operator-ui/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/postgres-operator-ui/Chart.yaml b/charts/postgres-operator-ui/Chart.yaml new file mode 100644 index 000000000..4be7d8af1 --- /dev/null +++ b/charts/postgres-operator-ui/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +name: postgres-operator-ui +version: 0.1.0 +appVersion: 1.2.0 +home: https://github.com/zalando/postgres-operator +description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience +keywords: +- postgres +- operator +- ui +- cloud-native +- patroni +- spilo +maintainers: +- name: siku4 + email: sk@sik-net.de +sources: +- https://github.com/zalando/postgres-operator +engine: gotpl diff --git a/charts/postgres-operator-ui/templates/NOTES.txt b/charts/postgres-operator-ui/templates/NOTES.txt new file mode 100644 index 000000000..7e9142891 --- /dev/null +++ b/charts/postgres-operator-ui/templates/NOTES.txt @@ -0,0 +1,3 @@ +To verify that postgres-operator has started, run: + + kubectl --namespace={{ .Release.Namespace }} get pods -l "app.kubernetes.io/name={{ template "postgres-operator-ui.name" . }}" \ No newline at end of file diff --git a/charts/postgres-operator-ui/templates/_helpers.tpl b/charts/postgres-operator-ui/templates/_helpers.tpl new file mode 100644 index 000000000..a5e97081d --- /dev/null +++ b/charts/postgres-operator-ui/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "postgres-operator-ui.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "postgres-operator-ui.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "postgres-operator-ui.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml new file mode 100644 index 000000000..924bcf000 --- /dev/null +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + labels: + app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} + helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "postgres-operator-ui.fullname" . }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + team: "acid" # Parameterize? + spec: + serviceAccountName: {{ template "postgres-operator-ui.name" . }} + containers: + - name: "service" + image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 8081 + protocol: "TCP" + readinessProbe: + httpGet: + path: "/health" + port: 8081 + initialDelaySeconds: 5 + timeoutSeconds: 1 + resources: + {{- toYaml .Values.resources | nindent 12 }} + env: + - name: "APP_URL" + value: "http://localhost:8081" + - name: "OPERATOR_API_URL" + value: {{ .Values.envs.operatorApiUrl }} + - name: "TARGET_NAMESPACE" + value: {{ .Values.envs.targetNamespace }} + - name: "TEAMS" + value: |- + [ + "acid" + ] + - name: "OPERATOR_UI_CONFIG" + value: |- + { + "docs_link":"https://postgres-operator.readthedocs.io/en/latest/", + "dns_format_string": "{1}-{0}.{2}", + "databases_visible": true, + "master_load_balancer_visible": true, + "nat_gateways_visible": false, + "replica_load_balancer_visible": true, + "resources_visible": true, + "users_visible": true, + "postgresql_versions": [ + "12", + "11", + "10", + "9.6", + "9.5" + ] + } \ No newline at end of file diff --git a/charts/postgres-operator-ui/templates/ingress.yaml b/charts/postgres-operator-ui/templates/ingress.yaml new file mode 100644 index 000000000..73fa2e817 --- /dev/null +++ b/charts/postgres-operator-ui/templates/ingress.yaml @@ -0,0 +1,44 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "postgres-operator-ui.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} + helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ . }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/postgres-operator-ui/templates/service.yaml b/charts/postgres-operator-ui/templates/service.yaml new file mode 100644 index 000000000..09adff26f --- /dev/null +++ b/charts/postgres-operator-ui/templates/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} + helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "postgres-operator-ui.fullname" . }} +spec: + ports: + - port: {{ .Values.service.port }} + targetPort: 8081 + protocol: TCP + selector: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} + type: {{ .Values.service.type }} + + diff --git a/charts/postgres-operator-ui/templates/serviceaccount.yaml b/charts/postgres-operator-ui/templates/serviceaccount.yaml new file mode 100644 index 000000000..4148938b0 --- /dev/null +++ b/charts/postgres-operator-ui/templates/serviceaccount.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "postgres-operator-ui.name" . }} + labels: + app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} + helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ template "postgres-operator-ui.name" . }} + labels: + app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} + helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +rules: +- apiGroups: + - acid.zalan.do + resources: + - postgresqls + verbs: + - create + - delete + - get + - list + - patch + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "postgres-operator-ui.name" . }} + labels: + app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} + helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "postgres-operator-ui.name" . }} +subjects: +- kind: ServiceAccount +# note: the cluster role binding needs to be defined +# for every namespace the operator-ui service account lives in. + name: {{ template "postgres-operator-ui.name" . }} + namespace: {{ .Release.Namespace }} \ No newline at end of file diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml new file mode 100644 index 000000000..9351d470e --- /dev/null +++ b/charts/postgres-operator-ui/values.yaml @@ -0,0 +1,47 @@ +# Default values for postgres-operator-ui. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +# configure ui image +image: + registry: registry.opensource.zalan.do + repository: acid/postgres-operator-ui + tag: v1.2.0 + pullPolicy: "IfNotPresent" + +# configure UI pod resources +resources: + limits: + cpu: 300m + memory: 3000Mi + requests: + cpu: 100m + memory: 100Mi + +# configure UI ENVs +envs: + # IMPORTANT: While operator chart and UI chart are idendependent, this is the interface between + # UI and operator API. Insert the service name of the operator API here! + operatorApiUrl: "http://postgres-operator:8080" + targetNamespace: "default" + +# configure UI service +service: + type: "ClusterIP" + port: "8080" + +# configure UI ingress. If needed: "enabled: true" +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: ui.example.org + paths: [""] + tls: [] + # - secretName: ui-tls + # hosts: + # - ui.exmaple.org \ No newline at end of file From a660d758a5850530bb7f7d3e1a049dd01f2bdda2 Mon Sep 17 00:00:00 2001 From: Vito Botta Date: Mon, 10 Feb 2020 12:48:24 +0200 Subject: [PATCH 038/104] Add region setting for logical backups to non-AWS storage (#813) * Add region setting for logical backups to non-AWS storage --- charts/postgres-operator/crds/operatorconfigurations.yaml | 2 ++ charts/postgres-operator/values-crd.yaml | 2 ++ charts/postgres-operator/values.yaml | 2 ++ docker/logical-backup/dump.sh | 1 + docs/reference/operator_parameters.md | 5 ++++- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 2 ++ manifests/postgresql-operator-default-configuration.yaml | 1 + pkg/apis/acid.zalan.do/v1/crds.go | 3 +++ pkg/apis/acid.zalan.do/v1/operator_configuration_type.go | 1 + pkg/cluster/k8sres.go | 4 ++++ pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 1 + 13 files changed, 25 insertions(+), 1 deletion(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 52d03df9c..9725c2708 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -243,6 +243,8 @@ spec: type: string logical_backup_s3_endpoint: type: string + logical_backup_s3_region: + type: string logical_backup_s3_secret_access_key: type: string logical_backup_s3_sse: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 61cab3d06..1f9b5e495 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -204,6 +204,8 @@ configLogicalBackup: logical_backup_s3_access_key_id: "" # S3 bucket to store backup results logical_backup_s3_bucket: "my-bucket-url" + # S3 region of bucket + logical_backup_s3_region: "" # S3 endpoint url when not using AWS logical_backup_s3_endpoint: "" # S3 Secret Access Key diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index deb506329..1be5851d2 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -195,6 +195,8 @@ configLogicalBackup: logical_backup_s3_access_key_id: "" # S3 bucket to store backup results logical_backup_s3_bucket: "my-bucket-url" + # S3 region of bucket + logical_backup_s3_region: "" # S3 endpoint url when not using AWS logical_backup_s3_endpoint: "" # S3 Secret Access Key diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh index 673f09038..2d9a39e02 100755 --- a/docker/logical-backup/dump.sh +++ b/docker/logical-backup/dump.sh @@ -40,6 +40,7 @@ function aws_upload { [[ ! -z "$EXPECTED_SIZE" ]] && args+=("--expected-size=$EXPECTED_SIZE") [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT") + [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION") [[ ! -z "$LOGICAL_BACKUP_S3_SSE" ]] && args+=("--sse=$LOGICAL_BACKUP_S3_SSE") aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}" diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index d6dde8c0e..7a8acc232 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -461,8 +461,11 @@ grouped under the `logical_backup` key. S3 bucket to store backup results. The bucket has to be present and accessible by Postgres pods. Default: empty. +* **logical_backup_s3_region** + Specifies the region of the bucket which is required with some non-AWS S3 storage services. The default is empty. + * **logical_backup_s3_endpoint** - When using non-AWS S3 storage, endpoint can be set as a ENV variable. + When using non-AWS S3 storage, endpoint can be set as a ENV variable. The default is empty. * **logical_backup_s3_sse** Specify server side encription that S3 storage is using. If empty string diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 7d11198da..d26c83edf 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -40,6 +40,7 @@ data: # logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" # logical_backup_s3_access_key_id: "" # logical_backup_s3_bucket: "my-bucket-url" + # logical_backup_s3_region: "" # logical_backup_s3_endpoint: "" # logical_backup_s3_secret_access_key: "" # logical_backup_s3_sse: "AES256" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 509d9aefc..7bd5c529c 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -219,6 +219,8 @@ spec: type: string logical_backup_s3_endpoint: type: string + logical_backup_s3_region: + type: string logical_backup_s3_secret_access_key: type: string logical_backup_s3_sse: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index f13a1eed9..efd1a5396 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -88,6 +88,7 @@ configuration: # logical_backup_s3_access_key_id: "" logical_backup_s3_bucket: "my-bucket-url" # logical_backup_s3_endpoint: "" + # logical_backup_s3_region: "" # logical_backup_s3_secret_access_key: "" logical_backup_s3_sse: "AES256" logical_backup_schedule: "30 00 * * *" diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 4d5a6f024..bc33f11f6 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -909,6 +909,9 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "logical_backup_s3_endpoint": { Type: "string", }, + "logical_backup_s3_region": { + Type: "string", + }, "logical_backup_s3_secret_access_key": { Type: "string", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 1e6a3b459..35c51e08d 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -157,6 +157,7 @@ type OperatorLogicalBackupConfiguration struct { Schedule string `json:"logical_backup_schedule,omitempty"` DockerImage string `json:"logical_backup_docker_image,omitempty"` S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` + S3Region string `json:"logical_backup_s3_region,omitempty"` S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"` S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"` diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index aed0c6e83..4c2f50296 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1589,6 +1589,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { Name: "LOGICAL_BACKUP_S3_BUCKET", Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket, }, + { + Name: "LOGICAL_BACKUP_S3_REGION", + Value: c.OpConfig.LogicalBackup.LogicalBackupS3Region, + }, { Name: "LOGICAL_BACKUP_S3_ENDPOINT", Value: c.OpConfig.LogicalBackup.LogicalBackupS3Endpoint, diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index d0357d222..98b56a298 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -106,6 +106,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.LogicalBackupSchedule = fromCRD.LogicalBackup.Schedule result.LogicalBackupDockerImage = fromCRD.LogicalBackup.DockerImage result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket + result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 339f06ce0..e4e429abb 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -76,6 +76,7 @@ type LogicalBackup struct { LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"` LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` + LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""` LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` From ba60e15d073e0b254df142cf71adb5725d6b1a16 Mon Sep 17 00:00:00 2001 From: Jonathan Juares Beber Date: Mon, 10 Feb 2020 12:03:25 +0100 Subject: [PATCH 039/104] Add ServiceAnnotations cluster config (#803) The [operator parameters][1] already support the `custom_service_annotations` config.With this parameter is possible to define custom annotations that will be used on the services created by the operator. The `custom_service_annotations` as all the other [operator parameters][1] are defined on the operator level and do not allow customization on the cluster level. A cluster may require different service annotations, as for example, set up different cloud load balancers timeouts, different ingress annotations, and/or enable more customizable environments. This commit introduces a new parameter on the cluster level, called `serviceAnnotations`, responsible for defining custom annotations just for the services created by the operator to the specifically defined cluster. It allows a mix of configuration between `custom_service_annotations` and `serviceAnnotations` where the latest one will have priority. In order to allow custom service annotations to be used on services without LoadBalancers (as for example, service mesh services annotations) both `custom_service_annotations` and `serviceAnnotations` are applied independently of load-balancing configuration. For retro-compatibility purposes, `custom_service_annotations` is still under [Load balancer related options][2]. The two default annotations when using LoadBalancer services, `external-dns.alpha.kubernetes.io/hostname` and `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` are still defined by the operator. `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` can be overridden by `custom_service_annotations` or `serviceAnnotations`, allowing a more customizable environment. `external-dns.alpha.kubernetes.io/hostname` can not be overridden once there is no differentiation between custom service annotations for replicas and masters. It updates the documentation and creates the necessary unit and e2e tests to the above-described feature too. [1]: https://github.com/zalando/postgres-operator/blob/master/docs/reference/operator_parameters.md [2]: https://github.com/zalando/postgres-operator/blob/master/docs/reference/operator_parameters.md#load-balancer-related-options --- .../postgres-operator/crds/postgresqls.yaml | 4 + docs/administrator.md | 11 + docs/reference/cluster_manifest.md | 5 + docs/reference/operator_parameters.md | 5 +- e2e/README.md | 1 + e2e/tests/test_e2e.py | 51 ++- manifests/complete-postgres-manifest.yaml | 2 + ...res-manifest-with-service-annotations.yaml | 20 ++ manifests/postgresql.crd.yaml | 4 + pkg/apis/acid.zalan.do/v1/crds.go | 8 + pkg/apis/acid.zalan.do/v1/postgresql_type.go | 1 + pkg/apis/acid.zalan.do/v1/util_test.go | 101 +++++- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 7 + pkg/cluster/cluster_test.go | 322 ++++++++++++++++++ pkg/cluster/k8sres.go | 60 ++-- 15 files changed, 565 insertions(+), 37 deletions(-) create mode 100644 manifests/postgres-manifest-with-service-annotations.yaml diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 198afe119..b4b676236 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -266,6 +266,10 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' # Note: the value specified here must not be zero or be higher # than the corresponding limit. + serviceAnnotations: + type: object + additionalProperties: + type: string sidecars: type: array nullable: true diff --git a/docs/administrator.md b/docs/administrator.md index 5b8769edb..2e86193c0 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -376,6 +376,17 @@ cluster manifest. In the case any of these variables are omitted from the manifest, the operator configuration settings `enable_master_load_balancer` and `enable_replica_load_balancer` apply. Note that the operator settings affect all Postgresql services running in all namespaces watched by the operator. +If load balancing is enabled two default annotations will be applied to its +services: + +- `external-dns.alpha.kubernetes.io/hostname` with the value defined by the + operator configs `master_dns_name_format` and `replica_dns_name_format`. + This value can't be overwritten. If any changing in its value is needed, it + MUST be done changing the DNS format operator config parameters; and +- `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` with + a default value of "3600". This value can be overwritten with the operator + config parameter `custom_service_annotations` or the cluster parameter + `serviceAnnotations`. To limit the range of IP addresses that can reach a load balancer, specify the desired ranges in the `allowedSourceRanges` field (applies to both master and diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index bf6df681b..7b049b6fa 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -122,6 +122,11 @@ These parameters are grouped directly under the `spec` key in the manifest. A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) to each pod created for the database. +* **serviceAnnotations** + A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) + to the services created for the database cluster. Check the + [administrator docs](https://github.com/zalando/postgres-operator/blob/master/docs/administrator.md#load-balancers-and-allowed-ip-ranges) + for more information regarding default values and overwrite rules. * **enableShmVolume** Start a database pod without limitations on shm memory. By default Docker diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 7a8acc232..e3893ea31 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -388,8 +388,9 @@ In the CRD-based configuration they are grouped under the `load_balancer` key. `false`. * **custom_service_annotations** - when load balancing is enabled, LoadBalancer service is created and - this parameter takes service annotations that are applied to service. + This key/value map provides a list of annotations that get attached to each + service of a cluster created by the operator. If the annotation key is also + provided by the cluster definition, the manifest value is used. Optional. * **master_dns_name_format** defines the DNS name string template for the diff --git a/e2e/README.md b/e2e/README.md index 1d611bcd0..f1bc5f9ed 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -44,3 +44,4 @@ The current tests are all bundled in [`test_e2e.py`](tests/test_e2e.py): * taint-based eviction of Postgres pods * invoking logical backup cron job * uniqueness of master pod +* custom service annotations diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index fc87c9887..5f34dcb16 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -211,8 +211,8 @@ class EndToEndTestCase(unittest.TestCase): schedule = "7 7 7 7 *" pg_patch_enable_backup = { "spec": { - "enableLogicalBackup": True, - "logicalBackupSchedule": schedule + "enableLogicalBackup": True, + "logicalBackupSchedule": schedule } } k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -234,7 +234,7 @@ class EndToEndTestCase(unittest.TestCase): image = "test-image-name" patch_logical_backup_image = { "data": { - "logical_backup_docker_image": image, + "logical_backup_docker_image": image, } } k8s.update_config(patch_logical_backup_image) @@ -247,7 +247,7 @@ class EndToEndTestCase(unittest.TestCase): # delete the logical backup cron job pg_patch_disable_backup = { "spec": { - "enableLogicalBackup": False, + "enableLogicalBackup": False, } } k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -257,6 +257,37 @@ class EndToEndTestCase(unittest.TestCase): self.assertEqual(0, len(jobs), "Expected 0 logical backup jobs, found {}".format(len(jobs))) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_service_annotations(self): + ''' + Create a Postgres cluster with service annotations and check them. + ''' + k8s = self.k8s + patch_custom_service_annotations = { + "data": { + "custom_service_annotations": "foo:bar", + } + } + k8s.update_config(patch_custom_service_annotations) + + k8s.create_with_kubectl("manifests/postgres-manifest-with-service-annotations.yaml") + annotations = { + "annotation.key": "value", + "foo": "bar", + } + self.assertTrue(k8s.check_service_annotations( + "version=acid-service-annotations,spilo-role=master", annotations)) + self.assertTrue(k8s.check_service_annotations( + "version=acid-service-annotations,spilo-role=replica", annotations)) + + # clean up + unpatch_custom_service_annotations = { + "data": { + "custom_service_annotations": "", + } + } + k8s.update_config(unpatch_custom_service_annotations) + def assert_master_is_unique(self, namespace='default', version="acid-minimal-cluster"): ''' Check that there is a single pod in the k8s cluster with the label "spilo-role=master" @@ -322,6 +353,16 @@ class K8s: pod_phase = pods[0].status.phase time.sleep(self.RETRY_TIMEOUT_SEC) + def check_service_annotations(self, svc_labels, annotations, namespace='default'): + svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items + for svc in svcs: + if len(svc.metadata.annotations) != len(annotations): + return False + for key in svc.metadata.annotations: + if svc.metadata.annotations[key] != annotations[key]: + return False + return True + def wait_for_pg_to_scale(self, number_of_instances, namespace='default'): body = { @@ -330,7 +371,7 @@ class K8s: } } _ = self.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body) + "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body) labels = 'version=acid-minimal-cluster' while self.count_pods_with_label(labels) != number_of_instances: diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 2478156d6..9e3b891c3 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -32,6 +32,8 @@ spec: # spiloFSGroup: 103 # podAnnotations: # annotation.key: value +# serviceAnnotations: +# annotation.key: value # podPriorityClassName: "spilo-pod-priority" # tolerations: # - key: postgres diff --git a/manifests/postgres-manifest-with-service-annotations.yaml b/manifests/postgres-manifest-with-service-annotations.yaml new file mode 100644 index 000000000..ab3096740 --- /dev/null +++ b/manifests/postgres-manifest-with-service-annotations.yaml @@ -0,0 +1,20 @@ +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-service-annotations +spec: + teamId: "acid" + volume: + size: 1Gi + numberOfInstances: 2 + users: + zalando: # database owner + - superuser + - createdb + foo_user: [] # role for application foo + databases: + foo: zalando # dbname: owner + postgresql: + version: "11" + serviceAnnotations: + annotation.key: value diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 3b0f652ea..276bc94b8 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -230,6 +230,10 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' # Note: the value specified here must not be zero or be higher # than the corresponding limit. + serviceAnnotations: + type: object + additionalProperties: + type: string sidecars: type: array nullable: true diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index bc33f11f6..4cfc9a9e6 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -383,6 +383,14 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "serviceAnnotations": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, "sidecars": { Type: "array", Items: &apiextv1beta1.JSONSchemaPropsOrArray{ diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 515a73ff0..07b42d4d4 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -60,6 +60,7 @@ type PostgresSpec struct { LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` StandbyCluster *StandbyDescription `json:"standby"` PodAnnotations map[string]string `json:"podAnnotations"` + ServiceAnnotations map[string]string `json:"serviceAnnotations"` // deprecated json tags InitContainersOld []v1.Container `json:"init_containers,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index a1e01825f..28e9e8ca4 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -456,18 +456,84 @@ var postgresqlList = []struct { PostgresqlList{}, errors.New("unexpected end of JSON input")}} -var annotations = []struct { +var podAnnotations = []struct { about string in []byte annotations map[string]string err error }{{ - about: "common annotations", - in: []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "acid-testcluster1"}, "spec": {"podAnnotations": {"foo": "bar"},"teamId": "acid", "clone": {"cluster": "team-batman"}}}`), + about: "common annotations", + in: []byte(`{ + "kind": "Postgresql", + "apiVersion": "acid.zalan.do/v1", + "metadata": { + "name": "acid-testcluster1" + }, + "spec": { + "podAnnotations": { + "foo": "bar" + }, + "teamId": "acid", + "clone": { + "cluster": "team-batman" + } + } + }`), annotations: map[string]string{"foo": "bar"}, err: nil}, } +var serviceAnnotations = []struct { + about string + in []byte + annotations map[string]string + err error +}{ + { + about: "common single annotation", + in: []byte(`{ + "kind": "Postgresql", + "apiVersion": "acid.zalan.do/v1", + "metadata": { + "name": "acid-testcluster1" + }, + "spec": { + "serviceAnnotations": { + "foo": "bar" + }, + "teamId": "acid", + "clone": { + "cluster": "team-batman" + } + } + }`), + annotations: map[string]string{"foo": "bar"}, + err: nil, + }, + { + about: "common two annotations", + in: []byte(`{ + "kind": "Postgresql", + "apiVersion": "acid.zalan.do/v1", + "metadata": { + "name": "acid-testcluster1" + }, + "spec": { + "serviceAnnotations": { + "foo": "bar", + "post": "gres" + }, + "teamId": "acid", + "clone": { + "cluster": "team-batman" + } + } + }`), + annotations: map[string]string{"foo": "bar", "post": "gres"}, + err: nil, + }, +} + func mustParseTime(s string) metav1.Time { v, err := time.Parse("15:04", s) if err != nil { @@ -517,21 +583,42 @@ func TestWeekdayTime(t *testing.T) { } } -func TestClusterAnnotations(t *testing.T) { - for _, tt := range annotations { +func TestPodAnnotations(t *testing.T) { + for _, tt := range podAnnotations { t.Run(tt.about, func(t *testing.T) { var cluster Postgresql err := cluster.UnmarshalJSON(tt.in) if err != nil { if tt.err == nil || err.Error() != tt.err.Error() { - t.Errorf("Unable to marshal cluster with annotations: expected %v got %v", tt.err, err) + t.Errorf("Unable to marshal cluster with podAnnotations: expected %v got %v", tt.err, err) } return } for k, v := range cluster.Spec.PodAnnotations { found, expected := v, tt.annotations[k] if found != expected { - t.Errorf("Didn't find correct value for key %v in for podAnnotations: Expected %v found %v", k, expected, found) + t.Errorf("Didn't find correct value for key %v in for podAnnotations: Expected %v found %v", k, expected, found) + } + } + }) + } +} + +func TestServiceAnnotations(t *testing.T) { + for _, tt := range serviceAnnotations { + t.Run(tt.about, func(t *testing.T) { + var cluster Postgresql + err := cluster.UnmarshalJSON(tt.in) + if err != nil { + if tt.err == nil || err.Error() != tt.err.Error() { + t.Errorf("Unable to marshal cluster with serviceAnnotations: expected %v got %v", tt.err, err) + } + return + } + for k, v := range cluster.Spec.ServiceAnnotations { + found, expected := v, tt.annotations[k] + if found != expected { + t.Errorf("Didn't find correct value for key %v in for serviceAnnotations: Expected %v found %v", k, expected, found) } } }) diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index dc07aa2cf..aaae1f04b 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -514,6 +514,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*out)[key] = val } } + if in.ServiceAnnotations != nil { + in, out := &in.ServiceAnnotations, &out.ServiceAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.InitContainersOld != nil { in, out := &in.InitContainersOld, &out.InitContainersOld *out = make([]corev1.Container, len(*in)) diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 85d014d8a..9efbc51c6 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -355,6 +355,12 @@ func TestPodAnnotations(t *testing.T) { database: map[string]string{"foo": "bar"}, merged: map[string]string{"foo": "bar"}, }, + { + subTest: "Both Annotations", + operator: map[string]string{"foo": "bar"}, + database: map[string]string{"post": "gres"}, + merged: map[string]string{"foo": "bar", "post": "gres"}, + }, { subTest: "Database Config overrides Operator Config Annotations", operator: map[string]string{"foo": "bar", "global": "foo"}, @@ -382,3 +388,319 @@ func TestPodAnnotations(t *testing.T) { } } } + +func TestServiceAnnotations(t *testing.T) { + enabled := true + disabled := false + tests := []struct { + about string + role PostgresRole + enableMasterLoadBalancerSpec *bool + enableMasterLoadBalancerOC bool + enableReplicaLoadBalancerSpec *bool + enableReplicaLoadBalancerOC bool + operatorAnnotations map[string]string + clusterAnnotations map[string]string + expect map[string]string + }{ + //MASTER + { + about: "Master with no annotations and EnableMasterLoadBalancer disabled on spec and OperatorConfig", + role: "master", + enableMasterLoadBalancerSpec: &disabled, + enableMasterLoadBalancerOC: false, + operatorAnnotations: make(map[string]string), + clusterAnnotations: make(map[string]string), + expect: make(map[string]string), + }, + { + about: "Master with no annotations and EnableMasterLoadBalancer enabled on spec", + role: "master", + enableMasterLoadBalancerSpec: &enabled, + enableMasterLoadBalancerOC: false, + operatorAnnotations: make(map[string]string), + clusterAnnotations: make(map[string]string), + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + }, + }, + { + about: "Master with no annotations and EnableMasterLoadBalancer enabled only on operator config", + role: "master", + enableMasterLoadBalancerSpec: &disabled, + enableMasterLoadBalancerOC: true, + operatorAnnotations: make(map[string]string), + clusterAnnotations: make(map[string]string), + expect: make(map[string]string), + }, + { + about: "Master with no annotations and EnableMasterLoadBalancer defined only on operator config", + role: "master", + enableMasterLoadBalancerOC: true, + operatorAnnotations: make(map[string]string), + clusterAnnotations: make(map[string]string), + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + }, + }, + { + about: "Master with cluster annotations and load balancer enabled", + role: "master", + enableMasterLoadBalancerOC: true, + operatorAnnotations: make(map[string]string), + clusterAnnotations: map[string]string{"foo": "bar"}, + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "foo": "bar", + }, + }, + { + about: "Master with cluster annotations and load balancer disabled", + role: "master", + enableMasterLoadBalancerSpec: &disabled, + enableMasterLoadBalancerOC: true, + operatorAnnotations: make(map[string]string), + clusterAnnotations: map[string]string{"foo": "bar"}, + expect: map[string]string{"foo": "bar"}, + }, + { + about: "Master with operator annotations and load balancer enabled", + role: "master", + enableMasterLoadBalancerOC: true, + operatorAnnotations: map[string]string{"foo": "bar"}, + clusterAnnotations: make(map[string]string), + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "foo": "bar", + }, + }, + { + about: "Master with operator annotations override default annotations", + role: "master", + enableMasterLoadBalancerOC: true, + operatorAnnotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", + }, + clusterAnnotations: make(map[string]string), + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", + }, + }, + { + about: "Master with cluster annotations override default annotations", + role: "master", + enableMasterLoadBalancerOC: true, + operatorAnnotations: make(map[string]string), + clusterAnnotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", + }, + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", + }, + }, + { + about: "Master with cluster annotations do not override external-dns annotations", + role: "master", + enableMasterLoadBalancerOC: true, + operatorAnnotations: make(map[string]string), + clusterAnnotations: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "wrong.external-dns-name.example.com", + }, + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + }, + }, + { + about: "Master with operator annotations do not override external-dns annotations", + role: "master", + enableMasterLoadBalancerOC: true, + clusterAnnotations: make(map[string]string), + operatorAnnotations: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "wrong.external-dns-name.example.com", + }, + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + }, + }, + // REPLICA + { + about: "Replica with no annotations and EnableReplicaLoadBalancer disabled on spec and OperatorConfig", + role: "replica", + enableReplicaLoadBalancerSpec: &disabled, + enableReplicaLoadBalancerOC: false, + operatorAnnotations: make(map[string]string), + clusterAnnotations: make(map[string]string), + expect: make(map[string]string), + }, + { + about: "Replica with no annotations and EnableReplicaLoadBalancer enabled on spec", + role: "replica", + enableReplicaLoadBalancerSpec: &enabled, + enableReplicaLoadBalancerOC: false, + operatorAnnotations: make(map[string]string), + clusterAnnotations: make(map[string]string), + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + }, + }, + { + about: "Replica with no annotations and EnableReplicaLoadBalancer enabled only on operator config", + role: "replica", + enableReplicaLoadBalancerSpec: &disabled, + enableReplicaLoadBalancerOC: true, + operatorAnnotations: make(map[string]string), + clusterAnnotations: make(map[string]string), + expect: make(map[string]string), + }, + { + about: "Replica with no annotations and EnableReplicaLoadBalancer defined only on operator config", + role: "replica", + enableReplicaLoadBalancerOC: true, + operatorAnnotations: make(map[string]string), + clusterAnnotations: make(map[string]string), + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + }, + }, + { + about: "Replica with cluster annotations and load balancer enabled", + role: "replica", + enableReplicaLoadBalancerOC: true, + operatorAnnotations: make(map[string]string), + clusterAnnotations: map[string]string{"foo": "bar"}, + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "foo": "bar", + }, + }, + { + about: "Replica with cluster annotations and load balancer disabled", + role: "replica", + enableReplicaLoadBalancerSpec: &disabled, + enableReplicaLoadBalancerOC: true, + operatorAnnotations: make(map[string]string), + clusterAnnotations: map[string]string{"foo": "bar"}, + expect: map[string]string{"foo": "bar"}, + }, + { + about: "Replica with operator annotations and load balancer enabled", + role: "replica", + enableReplicaLoadBalancerOC: true, + operatorAnnotations: map[string]string{"foo": "bar"}, + clusterAnnotations: make(map[string]string), + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "foo": "bar", + }, + }, + { + about: "Replica with operator annotations override default annotations", + role: "replica", + enableReplicaLoadBalancerOC: true, + operatorAnnotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", + }, + clusterAnnotations: make(map[string]string), + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", + }, + }, + { + about: "Replica with cluster annotations override default annotations", + role: "replica", + enableReplicaLoadBalancerOC: true, + operatorAnnotations: make(map[string]string), + clusterAnnotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", + }, + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", + }, + }, + { + about: "Replica with cluster annotations do not override external-dns annotations", + role: "replica", + enableReplicaLoadBalancerOC: true, + operatorAnnotations: make(map[string]string), + clusterAnnotations: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "wrong.external-dns-name.example.com", + }, + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + }, + }, + { + about: "Replica with operator annotations do not override external-dns annotations", + role: "replica", + enableReplicaLoadBalancerOC: true, + clusterAnnotations: make(map[string]string), + operatorAnnotations: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "wrong.external-dns-name.example.com", + }, + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + }, + }, + // COMMON + { + about: "cluster annotations append to operator annotations", + role: "replica", + enableReplicaLoadBalancerOC: false, + operatorAnnotations: map[string]string{"foo": "bar"}, + clusterAnnotations: map[string]string{"post": "gres"}, + expect: map[string]string{"foo": "bar", "post": "gres"}, + }, + { + about: "cluster annotations override operator annotations", + role: "replica", + enableReplicaLoadBalancerOC: false, + operatorAnnotations: map[string]string{"foo": "bar", "post": "gres"}, + clusterAnnotations: map[string]string{"post": "greSQL"}, + expect: map[string]string{"foo": "bar", "post": "greSQL"}, + }, + } + + for _, tt := range tests { + t.Run(tt.about, func(t *testing.T) { + cl.OpConfig.CustomServiceAnnotations = tt.operatorAnnotations + cl.OpConfig.EnableMasterLoadBalancer = tt.enableMasterLoadBalancerOC + cl.OpConfig.EnableReplicaLoadBalancer = tt.enableReplicaLoadBalancerOC + cl.OpConfig.MasterDNSNameFormat = "{cluster}.{team}.{hostedzone}" + cl.OpConfig.ReplicaDNSNameFormat = "{cluster}-repl.{team}.{hostedzone}" + cl.OpConfig.DbHostedZone = "db.example.com" + + cl.Postgresql.Spec.ClusterName = "test" + cl.Postgresql.Spec.TeamID = "acid" + cl.Postgresql.Spec.ServiceAnnotations = tt.clusterAnnotations + cl.Postgresql.Spec.EnableMasterLoadBalancer = tt.enableMasterLoadBalancerSpec + cl.Postgresql.Spec.EnableReplicaLoadBalancer = tt.enableReplicaLoadBalancerSpec + + got := cl.generateServiceAnnotations(tt.role, &cl.Postgresql.Spec) + if len(tt.expect) != len(got) { + t.Errorf("expected %d annotation(s), got %d", len(tt.expect), len(got)) + return + } + for k, v := range got { + if tt.expect[k] != v { + t.Errorf("expected annotation '%v' with value '%v', got value '%v'", k, tt.expect[k], v) + } + } + }) + } +} diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 4c2f50296..e6561e0f3 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1230,14 +1230,6 @@ func (c *Cluster) shouldCreateLoadBalancerForService(role PostgresRole, spec *ac } func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) *v1.Service { - var dnsName string - - if role == Master { - dnsName = c.masterDNSName() - } else { - dnsName = c.replicaDNSName() - } - serviceSpec := v1.ServiceSpec{ Ports: []v1.ServicePort{{Name: "postgresql", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}}, Type: v1.ServiceTypeClusterIP, @@ -1247,8 +1239,6 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) serviceSpec.Selector = c.roleLabelsSet(false, role) } - var annotations map[string]string - if c.shouldCreateLoadBalancerForService(role, spec) { // spec.AllowedSourceRanges evaluates to the empty slice of zero length @@ -1262,18 +1252,6 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) c.logger.Debugf("final load balancer source ranges as seen in a service spec (not necessarily applied): %q", serviceSpec.LoadBalancerSourceRanges) serviceSpec.Type = v1.ServiceTypeLoadBalancer - - annotations = map[string]string{ - constants.ZalandoDNSNameAnnotation: dnsName, - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - } - - if len(c.OpConfig.CustomServiceAnnotations) != 0 { - c.logger.Debugf("There are custom annotations defined, creating them.") - for customAnnotationKey, customAnnotationValue := range c.OpConfig.CustomServiceAnnotations { - annotations[customAnnotationKey] = customAnnotationValue - } - } } else if role == Replica { // before PR #258, the replica service was only created if allocated a LB // now we always create the service but warn if the LB is absent @@ -1285,7 +1263,7 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) Name: c.serviceName(role), Namespace: c.Namespace, Labels: c.roleLabelsSet(true, role), - Annotations: annotations, + Annotations: c.generateServiceAnnotations(role, spec), }, Spec: serviceSpec, } @@ -1293,6 +1271,42 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) return service } +func (c *Cluster) generateServiceAnnotations(role PostgresRole, spec *acidv1.PostgresSpec) map[string]string { + annotations := make(map[string]string) + + for k, v := range c.OpConfig.CustomServiceAnnotations { + annotations[k] = v + } + if spec != nil || spec.ServiceAnnotations != nil { + for k, v := range spec.ServiceAnnotations { + annotations[k] = v + } + } + + if c.shouldCreateLoadBalancerForService(role, spec) { + var dnsName string + if role == Master { + dnsName = c.masterDNSName() + } else { + dnsName = c.replicaDNSName() + } + + // Just set ELB Timeout annotation with default value, if it does not + // have a cutom value + if _, ok := annotations[constants.ElbTimeoutAnnotationName]; !ok { + annotations[constants.ElbTimeoutAnnotationName] = constants.ElbTimeoutAnnotationValue + } + // External DNS name annotation is not customizable + annotations[constants.ZalandoDNSNameAnnotation] = dnsName + } + + if len(annotations) == 0 { + return nil + } + + return annotations +} + func (c *Cluster) generateEndpoint(role PostgresRole, subsets []v1.EndpointSubset) *v1.Endpoints { endpoints := &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ From be6c8cd5737f21a03ed7d671ddd31bc1e9abbef9 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 10 Feb 2020 16:41:51 +0100 Subject: [PATCH 040/104] specify cluster in e2e taint test (#823) --- e2e/tests/test_e2e.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 5f34dcb16..e3f00da9c 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -68,8 +68,8 @@ class EndToEndTestCase(unittest.TestCase): _, failover_targets = k8s.get_pg_nodes(cluster_label) # configure minimum boundaries for CPU and memory limits - minCPULimit = '250m' - minMemoryLimit = '250Mi' + minCPULimit = '500m' + minMemoryLimit = '500Mi' patch_min_resource_limits = { "data": { "min_cpu_limit": minCPULimit, @@ -176,7 +176,7 @@ class EndToEndTestCase(unittest.TestCase): # patch node and test if master is failing over to one of the expected nodes k8s.api.core_v1.patch_node(current_master_node, body) k8s.wait_for_master_failover(failover_targets) - k8s.wait_for_pod_start('spilo-role=replica') + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) new_master_node, new_replica_nodes = k8s.get_pg_nodes(cluster_label) self.assertNotEqual(current_master_node, new_master_node, From 00f00af2e84786eb4a87a35f2b987f5f946af3c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20=C3=98strem?= Date: Tue, 11 Feb 2020 17:16:38 +0100 Subject: [PATCH 041/104] Fix MasterPodMoveTimeout field that cannot be unmarshalled (#816) * Update operator_configuration_type.go * Update operator_config.go --- pkg/apis/acid.zalan.do/v1/operator_configuration_type.go | 2 +- pkg/controller/operator_config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 35c51e08d..ded5261fb 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -67,7 +67,7 @@ type KubernetesMetaConfiguration struct { // TODO: use namespacedname PodEnvironmentConfigMap string `json:"pod_environment_configmap,omitempty"` PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` - MasterPodMoveTimeout time.Duration `json:"master_pod_move_timeout,omitempty"` + MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` PodManagementPolicy string `json:"pod_management_policy,omitempty"` diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 98b56a298..c6f10faa0 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -66,7 +66,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName result.PodManagementPolicy = fromCRD.Kubernetes.PodManagementPolicy - result.MasterPodMoveTimeout = fromCRD.Kubernetes.MasterPodMoveTimeout + result.MasterPodMoveTimeout = time.Duration(fromCRD.Kubernetes.MasterPodMoveTimeout) result.EnablePodAntiAffinity = fromCRD.Kubernetes.EnablePodAntiAffinity result.PodAntiAffinityTopologyKey = fromCRD.Kubernetes.PodAntiAffinityTopologyKey From 744c71d16bd93b2d94b74fabc5e1f37a1eaa58c2 Mon Sep 17 00:00:00 2001 From: Jonathan Juares Beber Date: Thu, 13 Feb 2020 10:55:30 +0100 Subject: [PATCH 042/104] Allow services update when changing annotations (#818) The current implementations for `pkg.util.k8sutil.SameService` considers only service annotations change on the default annotations created by the operator. Custom annotations are not compared and consequently not applied after the first service creation. This commit introduces a complete annotations comparison between the current service created by the operator and the new one generated based on the configs. Also, it adds tests on the above-mentioned function. --- e2e/tests/test_e2e.py | 11 +- ...res-manifest-with-service-annotations.yaml | 20 -- pkg/util/k8sutil/k8sutil.go | 41 ++- pkg/util/k8sutil/k8sutil_test.go | 310 ++++++++++++++++++ 4 files changed, 348 insertions(+), 34 deletions(-) delete mode 100644 manifests/postgres-manifest-with-service-annotations.yaml create mode 100644 pkg/util/k8sutil/k8sutil_test.go diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index e3f00da9c..e92aba11f 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -270,7 +270,16 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(patch_custom_service_annotations) - k8s.create_with_kubectl("manifests/postgres-manifest-with-service-annotations.yaml") + pg_patch_custom_annotations = { + "spec": { + "serviceAnnotations": { + "annotation.key": "value" + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) + annotations = { "annotation.key": "value", "foo": "bar", diff --git a/manifests/postgres-manifest-with-service-annotations.yaml b/manifests/postgres-manifest-with-service-annotations.yaml deleted file mode 100644 index ab3096740..000000000 --- a/manifests/postgres-manifest-with-service-annotations.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: "acid.zalan.do/v1" -kind: postgresql -metadata: - name: acid-service-annotations -spec: - teamId: "acid" - volume: - size: 1Gi - numberOfInstances: 2 - users: - zalando: # database owner - - superuser - - createdb - foo_user: [] # role for application foo - databases: - foo: zalando # dbname: owner - postgresql: - version: "11" - serviceAnnotations: - annotation.key: value diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 118d1df53..c7b2366b0 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -9,7 +9,6 @@ import ( batchv1beta1 "k8s.io/api/batch/v1beta1" clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" - "github.com/zalando/postgres-operator/pkg/util/constants" v1 "k8s.io/api/core/v1" policybeta1 "k8s.io/api/policy/v1beta1" apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" @@ -136,21 +135,37 @@ func SameService(cur, new *v1.Service) (match bool, reason string) { } } - oldDNSAnnotation := cur.Annotations[constants.ZalandoDNSNameAnnotation] - newDNSAnnotation := new.Annotations[constants.ZalandoDNSNameAnnotation] - oldELBAnnotation := cur.Annotations[constants.ElbTimeoutAnnotationName] - newELBAnnotation := new.Annotations[constants.ElbTimeoutAnnotationName] + match = true - if oldDNSAnnotation != newDNSAnnotation { - return false, fmt.Sprintf("new service's %q annotation value %q doesn't match the current one %q", - constants.ZalandoDNSNameAnnotation, newDNSAnnotation, oldDNSAnnotation) - } - if oldELBAnnotation != newELBAnnotation { - return false, fmt.Sprintf("new service's %q annotation value %q doesn't match the current one %q", - constants.ElbTimeoutAnnotationName, oldELBAnnotation, newELBAnnotation) + reasonPrefix := "new service's annotations doesn't match the current one:" + for ann := range cur.Annotations { + if _, ok := new.Annotations[ann]; !ok { + match = false + if len(reason) == 0 { + reason = reasonPrefix + } + reason += fmt.Sprintf(" Removed '%s'.", ann) + } } - return true, "" + for ann := range new.Annotations { + v, ok := cur.Annotations[ann] + if !ok { + if len(reason) == 0 { + reason = reasonPrefix + } + reason += fmt.Sprintf(" Added '%s' with value '%s'.", ann, new.Annotations[ann]) + match = false + } else if v != new.Annotations[ann] { + if len(reason) == 0 { + reason = reasonPrefix + } + reason += fmt.Sprintf(" '%s' changed from '%s' to '%s'.", ann, v, new.Annotations[ann]) + match = false + } + } + + return match, reason } // SamePDB compares the PodDisruptionBudgets diff --git a/pkg/util/k8sutil/k8sutil_test.go b/pkg/util/k8sutil/k8sutil_test.go new file mode 100644 index 000000000..12288243e --- /dev/null +++ b/pkg/util/k8sutil/k8sutil_test.go @@ -0,0 +1,310 @@ +package k8sutil + +import ( + "strings" + "testing" + + "github.com/zalando/postgres-operator/pkg/util/constants" + + v1 "k8s.io/api/core/v1" +) + +func newsService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service { + svc := &v1.Service{ + Spec: v1.ServiceSpec{ + Type: svcT, + LoadBalancerSourceRanges: lbSr, + }, + } + svc.Annotations = ann + return svc +} + +func TestServiceAnnotations(t *testing.T) { + tests := []struct { + about string + current *v1.Service + new *v1.Service + reason string + match bool + }{ + { + about: "two equal services", + current: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: true, + }, + { + about: "services differ on service type", + current: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's type "LoadBalancer" doesn't match the current one "ClusterIP"`, + }, + { + about: "services differ on lb source ranges", + current: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"185.249.56.0/22"}), + match: false, + reason: `new service's LoadBalancerSourceRange doesn't match the current one`, + }, + { + about: "new service doesn't have lb source ranges", + current: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{}), + match: false, + reason: `new service's LoadBalancerSourceRange doesn't match the current one`, + }, + { + about: "services differ on DNS annotation", + current: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "new_clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations doesn't match the current one: 'external-dns.alpha.kubernetes.io/hostname' changed from 'clstr.acid.zalan.do' to 'new_clstr.acid.zalan.do'.`, + }, + { + about: "services differ on AWS ELB annotation", + current: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: "1800", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations doesn't match the current one: 'service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout' changed from '3600' to '1800'.`, + }, + { + about: "service changes existing annotation", + current: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "baz", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations doesn't match the current one: 'foo' changed from 'bar' to 'baz'.`, + }, + { + about: "service changes multiple existing annotations", + current: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + "bar": "foo", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "baz", + "bar": "fooz", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + // Test just the prefix to avoid flakiness and map sorting + reason: `new service's annotations doesn't match the current one:`, + }, + { + about: "service adds a new custom annotation", + current: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations doesn't match the current one: Added 'foo' with value 'bar'.`, + }, + { + about: "service removes a custom annotation", + current: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations doesn't match the current one: Removed 'foo'.`, + }, + { + about: "service removes a custom annotation and adds a new one", + current: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "bar": "foo", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations doesn't match the current one: Removed 'foo'. Added 'bar' with value 'foo'.`, + }, + { + about: "service removes a custom annotation, adds a new one and change another", + current: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + "zalan": "do", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "bar": "foo", + "zalan": "do.com", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations doesn't match the current one: Removed 'foo'. Added 'bar' with value 'foo'. 'zalan' changed from 'do' to 'do.com'`, + }, + { + about: "service add annotations", + current: newsService( + map[string]string{}, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + // Test just the prefix to avoid flakiness and map sorting + reason: `new service's annotations doesn't match the current one: Added `, + }, + } + for _, tt := range tests { + t.Run(tt.about, func(t *testing.T) { + match, reason := SameService(tt.current, tt.new) + if match && !tt.match { + t.Errorf("expected services to do not match: '%q' and '%q'", tt.current, tt.new) + return + } + if !match && tt.match { + t.Errorf("expected services to be the same: '%q' and '%q'", tt.current, tt.new) + return + } + if !match && !tt.match { + if !strings.HasPrefix(reason, tt.reason) { + t.Errorf("expected reason '%s', found '%s'", tt.reason, reason) + return + } + } + }) + } +} From 3b10dc645dd3f4112df7d15042f335b86c01b3fc Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 13 Feb 2020 16:24:15 +0100 Subject: [PATCH 043/104] patch/update services on type change (#824) * use Update when disabling LoadBalancer + added e2e test --- e2e/tests/test_e2e.py | 56 ++++++++++ manifests/operator-service-account-rbac.yaml | 1 + pkg/cluster/resources.go | 103 +++++-------------- pkg/cluster/sync.go | 2 +- 4 files changed, 86 insertions(+), 76 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index e92aba11f..2d81a0647 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -58,6 +58,55 @@ class EndToEndTestCase(unittest.TestCase): k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") k8s.wait_for_pod_start('spilo-role=master') + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_enable_load_balancer(self): + ''' + Test if services are updated when enabling/disabling load balancers + ''' + + k8s = self.k8s + cluster_label = 'version=acid-minimal-cluster' + + # enable load balancer services + pg_patch_enable_lbs = { + "spec": { + "enableMasterLoadBalancer": True, + "enableReplicaLoadBalancer": True + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) + # wait for service recreation + time.sleep(60) + + master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + self.assertEqual(master_svc_type, 'LoadBalancer', + "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) + + repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + self.assertEqual(repl_svc_type, 'LoadBalancer', + "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) + + # disable load balancer services again + pg_patch_disable_lbs = { + "spec": { + "enableMasterLoadBalancer": False, + "enableReplicaLoadBalancer": False + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) + # wait for service recreation + time.sleep(60) + + master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + self.assertEqual(master_svc_type, 'ClusterIP', + "Expected ClusterIP service type for master, found {}".format(master_svc_type)) + + repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + self.assertEqual(repl_svc_type, 'ClusterIP', + "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_min_resource_limits(self): ''' @@ -362,6 +411,13 @@ class K8s: pod_phase = pods[0].status.phase time.sleep(self.RETRY_TIMEOUT_SEC) + def get_service_type(self, svc_labels, namespace='default'): + svc_type = '' + svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items + for svc in svcs: + svc_type = svc.spec.type + return svc_type + def check_service_annotations(self, svc_labels, annotations, namespace='default'): svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items for svc in svcs: diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index a37abe476..4761c145e 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -114,6 +114,7 @@ rules: - delete - get - patch + - update # to CRUD the StatefulSet which controls the Postgres cluster instances - apiGroups: - apps diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index c94a7bb46..d6c2149bf 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -366,6 +366,11 @@ func (c *Cluster) createService(role PostgresRole) (*v1.Service, error) { } func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error { + var ( + svc *v1.Service + err error + ) + c.setProcessName("updating %v service", role) if c.Services[role] == nil { @@ -373,70 +378,6 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error } serviceName := util.NameFromMeta(c.Services[role].ObjectMeta) - endpointName := util.NameFromMeta(c.Endpoints[role].ObjectMeta) - // TODO: check if it possible to change the service type with a patch in future versions of Kubernetes - if newService.Spec.Type != c.Services[role].Spec.Type { - // service type has changed, need to replace the service completely. - // we cannot use just patch the current service, since it may contain attributes incompatible with the new type. - var ( - currentEndpoint *v1.Endpoints - err error - ) - - if role == Master { - // for the master service we need to re-create the endpoint as well. Get the up-to-date version of - // the addresses stored in it before the service is deleted (deletion of the service removes the endpoint) - currentEndpoint, err = c.KubeClient.Endpoints(c.Namespace).Get(c.endpointName(role), metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("could not get current cluster %s endpoints: %v", role, err) - } - } - err = c.KubeClient.Services(serviceName.Namespace).Delete(serviceName.Name, c.deleteOptions) - if err != nil { - return fmt.Errorf("could not delete service %q: %v", serviceName, err) - } - - // wait until the service is truly deleted - c.logger.Debugf("waiting for service to be deleted") - - err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, - func() (bool, error) { - _, err2 := c.KubeClient.Services(serviceName.Namespace).Get(serviceName.Name, metav1.GetOptions{}) - if err2 == nil { - return false, nil - } - if k8sutil.ResourceNotFound(err2) { - return true, nil - } - return false, err2 - }) - if err != nil { - return fmt.Errorf("could not delete service %q: %v", serviceName, err) - } - - // make sure we clear the stored service and endpoint status if the subsequent create fails. - c.Services[role] = nil - c.Endpoints[role] = nil - if role == Master { - // create the new endpoint using the addresses obtained from the previous one - endpointSpec := c.generateEndpoint(role, currentEndpoint.Subsets) - ep, err := c.KubeClient.Endpoints(endpointSpec.Namespace).Create(endpointSpec) - if err != nil { - return fmt.Errorf("could not create endpoint %q: %v", endpointName, err) - } - - c.Endpoints[role] = ep - } - - svc, err := c.KubeClient.Services(serviceName.Namespace).Create(newService) - if err != nil { - return fmt.Errorf("could not create service %q: %v", serviceName, err) - } - - c.Services[role] = svc - - return nil - } // update the service annotation in order to propagate ELB notation. if len(newService.ObjectMeta.Annotations) > 0 { @@ -454,18 +395,30 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error } } - patchData, err := specPatch(newService.Spec) - if err != nil { - return fmt.Errorf("could not form patch for the service %q: %v", serviceName, err) - } + // now, patch the service spec, but when disabling LoadBalancers do update instead + // patch does not work because of LoadBalancerSourceRanges field (even if set to nil) + oldServiceType := c.Services[role].Spec.Type + newServiceType := newService.Spec.Type + if newServiceType == "ClusterIP" && newServiceType != oldServiceType { + newService.ResourceVersion = c.Services[role].ResourceVersion + newService.Spec.ClusterIP = c.Services[role].Spec.ClusterIP + svc, err = c.KubeClient.Services(serviceName.Namespace).Update(newService) + if err != nil { + return fmt.Errorf("could not update service %q: %v", serviceName, err) + } + } else { + patchData, err := specPatch(newService.Spec) + if err != nil { + return fmt.Errorf("could not form patch for the service %q: %v", serviceName, err) + } - // update the service spec - svc, err := c.KubeClient.Services(serviceName.Namespace).Patch( - serviceName.Name, - types.MergePatchType, - patchData, "") - if err != nil { - return fmt.Errorf("could not patch service %q: %v", serviceName, err) + svc, err = c.KubeClient.Services(serviceName.Namespace).Patch( + serviceName.Name, + types.MergePatchType, + patchData, "") + if err != nil { + return fmt.Errorf("could not patch service %q: %v", serviceName, err) + } } c.Services[role] = svc diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index fa4fc9ec1..053db9ff7 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -116,7 +116,7 @@ func (c *Cluster) syncServices() error { c.logger.Debugf("syncing %s service", role) if err := c.syncEndpoint(role); err != nil { - return fmt.Errorf("could not sync %s endpont: %v", role, err) + return fmt.Errorf("could not sync %s endpoint: %v", role, err) } if err := c.syncService(role); err != nil { From 702a194c414f3fb8ceeb70b4b0cd35d56bd1c5bb Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 17 Feb 2020 11:25:07 +0100 Subject: [PATCH 044/104] switch to rbac/v1 (#829) * switch to rbac/v1 --- charts/postgres-operator-ui/templates/serviceaccount.yaml | 6 +++--- charts/postgres-operator/templates/clusterrole.yaml | 2 +- manifests/operator-service-account-rbac.yaml | 2 +- pkg/cluster/cluster.go | 4 ++-- pkg/controller/controller.go | 8 ++++---- pkg/util/k8sutil/k8sutil.go | 6 +++--- ui/manifests/ui-service-account-rbac.yaml | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/charts/postgres-operator-ui/templates/serviceaccount.yaml b/charts/postgres-operator-ui/templates/serviceaccount.yaml index 4148938b0..7bb715167 100644 --- a/charts/postgres-operator-ui/templates/serviceaccount.yaml +++ b/charts/postgres-operator-ui/templates/serviceaccount.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ template "postgres-operator-ui.name" . }} @@ -17,7 +17,7 @@ metadata: app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/instance: {{ .Release.Name }} rules: - apiGroups: - acid.zalan.do @@ -78,4 +78,4 @@ subjects: # note: the cluster role binding needs to be defined # for every namespace the operator-ui service account lives in. name: {{ template "postgres-operator-ui.name" . }} - namespace: {{ .Release.Namespace }} \ No newline at end of file + namespace: {{ .Release.Namespace }} diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index f8550a539..f7fe1634c 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -1,5 +1,5 @@ {{ if .Values.rbac.create }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ include "postgres-operator.serviceAccountName" . }} diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 4761c145e..5e43cc03b 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -5,7 +5,7 @@ metadata: namespace: default --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: zalando-postgres-operator diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index c560c4cdf..91e7a5195 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -29,7 +29,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/patroni" "github.com/zalando/postgres-operator/pkg/util/teams" "github.com/zalando/postgres-operator/pkg/util/users" - rbacv1beta1 "k8s.io/api/rbac/v1beta1" + rbacv1 "k8s.io/api/rbac/v1" ) var ( @@ -45,7 +45,7 @@ type Config struct { RestConfig *rest.Config InfrastructureRoles map[string]spec.PgUser // inherited from the controller PodServiceAccount *v1.ServiceAccount - PodServiceAccountRoleBinding *rbacv1beta1.RoleBinding + PodServiceAccountRoleBinding *rbacv1.RoleBinding } type kubeResources struct { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 831078f3e..f67d99c1d 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -7,7 +7,7 @@ import ( "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" - rbacv1beta1 "k8s.io/api/rbac/v1beta1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" @@ -57,7 +57,7 @@ type Controller struct { workerLogs map[uint32]ringlog.RingLogger PodServiceAccount *v1.ServiceAccount - PodServiceAccountRoleBinding *rbacv1beta1.RoleBinding + PodServiceAccountRoleBinding *rbacv1.RoleBinding } // NewController creates a new controller @@ -198,7 +198,7 @@ func (c *Controller) initRoleBinding() { if c.opConfig.PodServiceAccountRoleBindingDefinition == "" { c.opConfig.PodServiceAccountRoleBindingDefinition = fmt.Sprintf(` { - "apiVersion": "rbac.authorization.k8s.io/v1beta1", + "apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", "metadata": { "name": "%s" @@ -227,7 +227,7 @@ func (c *Controller) initRoleBinding() { case groupVersionKind.Kind != "RoleBinding": panic(fmt.Errorf("role binding definition in the operator config map defines another type of resource: %v", groupVersionKind.Kind)) default: - c.PodServiceAccountRoleBinding = obj.(*rbacv1beta1.RoleBinding) + c.PodServiceAccountRoleBinding = obj.(*rbacv1.RoleBinding) c.PodServiceAccountRoleBinding.Namespace = "" c.logger.Info("successfully parsed") diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index c7b2366b0..509b12c19 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -18,7 +18,7 @@ import ( appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" policyv1beta1 "k8s.io/client-go/kubernetes/typed/policy/v1beta1" - rbacv1beta1 "k8s.io/client-go/kubernetes/typed/rbac/v1beta1" + rbacv1 "k8s.io/client-go/kubernetes/typed/rbac/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -39,7 +39,7 @@ type KubernetesClient struct { corev1.NamespacesGetter corev1.ServiceAccountsGetter appsv1.StatefulSetsGetter - rbacv1beta1.RoleBindingsGetter + rbacv1.RoleBindingsGetter policyv1beta1.PodDisruptionBudgetsGetter apiextbeta1.CustomResourceDefinitionsGetter clientbatchv1beta1.CronJobsGetter @@ -103,7 +103,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.StatefulSetsGetter = client.AppsV1() kubeClient.PodDisruptionBudgetsGetter = client.PolicyV1beta1() kubeClient.RESTClient = client.CoreV1().RESTClient() - kubeClient.RoleBindingsGetter = client.RbacV1beta1() + kubeClient.RoleBindingsGetter = client.RbacV1() kubeClient.CronJobsGetter = client.BatchV1beta1() apiextClient, err := apiextclient.NewForConfig(cfg) diff --git a/ui/manifests/ui-service-account-rbac.yaml b/ui/manifests/ui-service-account-rbac.yaml index 4ae218e74..f0a6e8bb7 100644 --- a/ui/manifests/ui-service-account-rbac.yaml +++ b/ui/manifests/ui-service-account-rbac.yaml @@ -5,7 +5,7 @@ metadata: namespace: default --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: postgres-operator-ui From 4b440e59de984e83cd01af3431e283d03e20d341 Mon Sep 17 00:00:00 2001 From: Jonathan Juares Beber Date: Tue, 18 Feb 2020 16:45:44 +0100 Subject: [PATCH 045/104] Fix test flakiness on TestSameService (#833) The code added on #818 depends on map sorting to return a static reason for service annotation changes. To avoid tests flakiness and map sorting the tests include a `strings.HasPrefix` instead of comparing the whole string. One of the test cases, `service_removes_a_custom_annotation,_adds_a_new_one_and_change_another`, is trying to test the whole reason string. This commit replaces the test case reason, for only the reason prefix. It removes the flakiness from the tests. As all the cases (annotation adding, removing and value changing) are tested before, it's safe to test only prefixes. Also, it renames the test name from `TestServiceAnnotations` to `TestSameService` and introduces a better description in case of test failure, describing that only prefixes are tested. --- pkg/util/k8sutil/k8sutil_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/util/k8sutil/k8sutil_test.go b/pkg/util/k8sutil/k8sutil_test.go index 12288243e..9b4f2eac3 100644 --- a/pkg/util/k8sutil/k8sutil_test.go +++ b/pkg/util/k8sutil/k8sutil_test.go @@ -20,7 +20,7 @@ func newsService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1. return svc } -func TestServiceAnnotations(t *testing.T) { +func TestSameService(t *testing.T) { tests := []struct { about string current *v1.Service @@ -267,8 +267,9 @@ func TestServiceAnnotations(t *testing.T) { }, v1.ServiceTypeLoadBalancer, []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations doesn't match the current one: Removed 'foo'. Added 'bar' with value 'foo'. 'zalan' changed from 'do' to 'do.com'`, + match: false, + // Test just the prefix to avoid flakiness and map sorting + reason: `new service's annotations doesn't match the current one: Removed 'foo'.`, }, { about: "service add annotations", @@ -301,7 +302,7 @@ func TestServiceAnnotations(t *testing.T) { } if !match && !tt.match { if !strings.HasPrefix(reason, tt.reason) { - t.Errorf("expected reason '%s', found '%s'", tt.reason, reason) + t.Errorf("expected reason prefix '%s', found '%s'", tt.reason, reason) return } } From aea9e9bd330c3fa934ca5fabcb8cb389a6356485 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 19 Feb 2020 12:32:54 +0100 Subject: [PATCH 046/104] postgres-pod clusterrole (#832) * define postgres-pod clusterrole and align rbac in chart * align UI chart rbac with operator and update doc * operator RBAC needs podsecuritypolicy to grant it to postgres-pod --- .../templates/_helpers.tpl | 7 ++ .../templates/clusterrole.yaml | 52 +++++++++ .../templates/clusterrolebinding.yaml | 19 ++++ .../templates/deployment.yaml | 8 +- .../templates/serviceaccount.yaml | 76 +------------ charts/postgres-operator-ui/values.yaml | 15 ++- .../templates/clusterrole-postgres-pod.yaml | 53 +++++++++ .../templates/clusterrole.yaml | 64 ++++++++--- .../templates/clusterrolebinding.yaml | 2 - .../templates/configmap.yaml | 1 - .../templates/operatorconfiguration.yaml | 1 - charts/postgres-operator/values-crd.yaml | 2 + charts/postgres-operator/values.yaml | 2 + docs/administrator.md | 49 +++------ docs/reference/operator_parameters.md | 17 +-- manifests/configmap.yaml | 2 +- manifests/operator-service-account-rbac.yaml | 104 ++++++++++++++---- manifests/postgres-operator.yaml | 2 +- ...gresql-operator-default-configuration.yaml | 2 +- manifests/user-facing-clusterroles.yaml | 10 +- pkg/controller/controller.go | 21 ++-- pkg/util/config/config.go | 2 +- ui/manifests/ui-service-account-rbac.yaml | 2 - 23 files changed, 333 insertions(+), 180 deletions(-) create mode 100644 charts/postgres-operator-ui/templates/clusterrole.yaml create mode 100644 charts/postgres-operator-ui/templates/clusterrolebinding.yaml create mode 100644 charts/postgres-operator/templates/clusterrole-postgres-pod.yaml diff --git a/charts/postgres-operator-ui/templates/_helpers.tpl b/charts/postgres-operator-ui/templates/_helpers.tpl index a5e97081d..d83b9291a 100644 --- a/charts/postgres-operator-ui/templates/_helpers.tpl +++ b/charts/postgres-operator-ui/templates/_helpers.tpl @@ -24,6 +24,13 @@ If release name contains chart name it will be used as a full name. {{- end -}} {{- end -}} +{{/* +Create a service account name. +*/}} +{{- define "postgres-operator-ui.serviceAccountName" -}} +{{ default (include "postgres-operator-ui.fullname" .) .Values.serviceAccount.name }} +{{- end -}} + {{/* Create chart name and version as used by the chart label. */}} diff --git a/charts/postgres-operator-ui/templates/clusterrole.yaml b/charts/postgres-operator-ui/templates/clusterrole.yaml new file mode 100644 index 000000000..4f76400ec --- /dev/null +++ b/charts/postgres-operator-ui/templates/clusterrole.yaml @@ -0,0 +1,52 @@ +{{ if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "postgres-operator-ui.serviceAccountName" . }} + labels: + app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} + helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +rules: +- apiGroups: + - acid.zalan.do + resources: + - postgresqls + verbs: + - create + - delete + - get + - list + - patch + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list +{{ end }} diff --git a/charts/postgres-operator-ui/templates/clusterrolebinding.yaml b/charts/postgres-operator-ui/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..7c874d3f7 --- /dev/null +++ b/charts/postgres-operator-ui/templates/clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{ if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "postgres-operator-ui.serviceAccountName" . }} + labels: + app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} + helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "postgres-operator-ui.serviceAccountName" . }} +subjects: +- kind: ServiceAccount + name: {{ include "postgres-operator-ui.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{ end }} diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml index 924bcf000..da0280e61 100644 --- a/charts/postgres-operator-ui/templates/deployment.yaml +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -20,7 +20,7 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} team: "acid" # Parameterize? spec: - serviceAccountName: {{ template "postgres-operator-ui.name" . }} + serviceAccountName: {{ include "postgres-operator-ui.serviceAccountName" . }} containers: - name: "service" image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -45,8 +45,8 @@ spec: value: {{ .Values.envs.targetNamespace }} - name: "TEAMS" value: |- - [ - "acid" + [ + "acid" ] - name: "OPERATOR_UI_CONFIG" value: |- @@ -66,4 +66,4 @@ spec: "9.6", "9.5" ] - } \ No newline at end of file + } diff --git a/charts/postgres-operator-ui/templates/serviceaccount.yaml b/charts/postgres-operator-ui/templates/serviceaccount.yaml index 7bb715167..4c5a25543 100644 --- a/charts/postgres-operator-ui/templates/serviceaccount.yaml +++ b/charts/postgres-operator-ui/templates/serviceaccount.yaml @@ -1,81 +1,11 @@ +{{ if .Values.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: - name: {{ template "postgres-operator-ui.name" . }} + name: {{ include "postgres-operator-ui.serviceAccountName" . }} labels: app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ template "postgres-operator-ui.name" . }} - labels: - app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} - helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} -rules: -- apiGroups: - - acid.zalan.do - resources: - - postgresqls - verbs: - - create - - delete - - get - - list - - patch - - update -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list -- apiGroups: - - apps - resources: - - statefulsets - verbs: - - get - - list -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ template "postgres-operator-ui.name" . }} - labels: - app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} - helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ template "postgres-operator-ui.name" . }} -subjects: -- kind: ServiceAccount -# note: the cluster role binding needs to be defined -# for every namespace the operator-ui service account lives in. - name: {{ template "postgres-operator-ui.name" . }} - namespace: {{ .Release.Namespace }} +{{ end }} diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml index 9351d470e..dca093410 100644 --- a/charts/postgres-operator-ui/values.yaml +++ b/charts/postgres-operator-ui/values.yaml @@ -11,6 +11,17 @@ image: tag: v1.2.0 pullPolicy: "IfNotPresent" +rbac: + # Specifies whether RBAC resources should be created + create: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # configure UI pod resources resources: limits: @@ -22,7 +33,7 @@ resources: # configure UI ENVs envs: - # IMPORTANT: While operator chart and UI chart are idendependent, this is the interface between + # IMPORTANT: While operator chart and UI chart are idendependent, this is the interface between # UI and operator API. Insert the service name of the operator API here! operatorApiUrl: "http://postgres-operator:8080" targetNamespace: "default" @@ -44,4 +55,4 @@ ingress: tls: [] # - secretName: ui-tls # hosts: - # - ui.exmaple.org \ No newline at end of file + # - ui.exmaple.org diff --git a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml new file mode 100644 index 000000000..c327d9101 --- /dev/null +++ b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml @@ -0,0 +1,53 @@ +{{ if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: postgres-pod + labels: + app.kubernetes.io/name: {{ template "postgres-operator.name" . }} + helm.sh/chart: {{ template "postgres-operator.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +rules: +# Patroni needs to watch and manage endpoints +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# Patroni needs to watch pods +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch +# to let Patroni create a headless service +- apiGroups: + - "" + resources: + - services + verbs: + - create +# to run privileged pods +- apiGroups: + - extensions + resources: + - podsecuritypolicies + resourceNames: + - privileged + verbs: + - use +{{ end }} diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index f7fe1634c..9a4165797 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -9,6 +9,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} rules: +# all verbs allowed for custom operator resources - apiGroups: - acid.zalan.do resources: @@ -16,7 +17,15 @@ rules: - postgresqls/status - operatorconfigurations verbs: - - "*" + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io resources: @@ -26,12 +35,14 @@ rules: - get - patch - update +# to read configuration from ConfigMaps - apiGroups: - "" resources: - configmaps verbs: - get +# to manage endpoints which are also used by Patroni - apiGroups: - "" resources: @@ -43,7 +54,9 @@ rules: - get - list - patch - - watch # needed if zalando-postgres-operator account is used for pods as well + - update + - watch +# to CRUD secrets for database access - apiGroups: - "" resources: @@ -53,6 +66,7 @@ rules: - update - delete - get +# to check nodes for node readiness label - apiGroups: - "" resources: @@ -61,6 +75,7 @@ rules: - get - list - watch +# to read or delete existing PVCs. Creation via StatefulSet - apiGroups: - "" resources: @@ -69,6 +84,7 @@ rules: - delete - get - list + # to read existing PVs. Creation should be done via dynamic provisioning - apiGroups: - "" resources: @@ -77,6 +93,7 @@ rules: - get - list - update # only for resizing AWS volumes +# to watch Spilo pods and do rolling updates. Creation via StatefulSet - apiGroups: - "" resources: @@ -86,13 +103,16 @@ rules: - get - list - watch + - update - patch +# to resize the filesystem in Spilo pods when increasing volume size - apiGroups: - "" resources: - pods/exec verbs: - create +# to CRUD services to point to Postgres cluster instances - apiGroups: - "" resources: @@ -102,6 +122,8 @@ rules: - delete - get - patch + - update +# to CRUD the StatefulSet which controls the Postgres cluster instances - apiGroups: - apps resources: @@ -112,12 +134,26 @@ rules: - get - list - patch +# to CRUD cron jobs for logical backups +- apiGroups: + - batch + resources: + - cronjobs + verbs: + - create + - delete + - get + - list + - patch + - update +# to get namespaces operator resources can run in - apiGroups: - "" resources: - namespaces verbs: - get +# to define PDBs. Update happens via delete/create - apiGroups: - policy resources: @@ -126,6 +162,7 @@ rules: - create - delete - get +# to create ServiceAccounts in each namespace the operator watches - apiGroups: - "" resources: @@ -133,30 +170,21 @@ rules: verbs: - get - create +# to create role bindings to the postgres-pod service account - apiGroups: - - "rbac.authorization.k8s.io" + - rbac.authorization.k8s.io resources: - rolebindings verbs: - get - create +# to grant privilege to run privileged pods - apiGroups: - - "rbac.authorization.k8s.io" + - extensions resources: - - clusterroles - verbs: - - bind + - podsecuritypolicies resourceNames: - - {{ include "postgres-operator.serviceAccountName" . }} -- apiGroups: - - batch - resources: - - cronjobs # enables logical backups + - privileged verbs: - - create - - delete - - get - - list - - patch - - update + - use {{ end }} diff --git a/charts/postgres-operator/templates/clusterrolebinding.yaml b/charts/postgres-operator/templates/clusterrolebinding.yaml index bfa21b42f..dbf65d00e 100644 --- a/charts/postgres-operator/templates/clusterrolebinding.yaml +++ b/charts/postgres-operator/templates/clusterrolebinding.yaml @@ -14,8 +14,6 @@ roleRef: name: {{ include "postgres-operator.serviceAccountName" . }} subjects: - kind: ServiceAccount -# note: the cluster role binding needs to be defined -# for every namespace the operator service account lives in. name: {{ include "postgres-operator.serviceAccountName" . }} namespace: {{ .Release.Namespace }} {{ end }} diff --git a/charts/postgres-operator/templates/configmap.yaml b/charts/postgres-operator/templates/configmap.yaml index 95eeb9546..0b976294e 100644 --- a/charts/postgres-operator/templates/configmap.yaml +++ b/charts/postgres-operator/templates/configmap.yaml @@ -9,7 +9,6 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} data: - pod_service_account_name: {{ include "postgres-operator.serviceAccountName" . }} {{ toYaml .Values.configGeneral | indent 2 }} {{ toYaml .Values.configUsers | indent 2 }} {{ toYaml .Values.configKubernetes | indent 2 }} diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index 6a301c1fb..06e9c7605 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -14,7 +14,6 @@ configuration: {{ toYaml .Values.configUsers | indent 4 }} kubernetes: oauth_token_secret_name: {{ template "postgres-operator.fullname" . }} - pod_service_account_name: {{ include "postgres-operator.serviceAccountName" . }} {{ toYaml .Values.configKubernetes | indent 4 }} postgres_pod_resources: {{ toYaml .Values.configPostgresPodResources | indent 4 }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 1f9b5e495..08c255a04 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -100,6 +100,8 @@ configKubernetes: pod_management_policy: "ordered_ready" # label assigned to the Postgres pods (and services/endpoints) pod_role_label: spilo-role + # name of service account to be used by postgres cluster pods + pod_service_account_name: "postgres-pod" # Postgres pods are terminated forcefully after this timeout pod_terminate_grace_period: 5m # template for database user secrets generated by the operator diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 1be5851d2..60a0a1f04 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -93,6 +93,8 @@ configKubernetes: pod_management_policy: "ordered_ready" # label assigned to the Postgres pods (and services/endpoints) pod_role_label: spilo-role + # name of service account to be used by postgres cluster pods + pod_service_account_name: "postgres-pod" # Postgres pods are terminated forcefully after this timeout pod_terminate_grace_period: 5m # template for database user secrets generated by the operator diff --git a/docs/administrator.md b/docs/administrator.md index 2e86193c0..2175ae595 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -47,6 +47,12 @@ patching the CRD manifest: zk8 patch crd postgresqls.acid.zalan.do -p '{"spec":{"validation": null}}' ``` +## Non-default cluster domain + +If your cluster uses a DNS domain other than the default `cluster.local`, this +needs to be set in the operator configuration (`cluster_domain` variable). This +is used by the operator to connect to the clusters after creation. + ## Namespaces ### Select the namespace to deploy to @@ -89,36 +95,13 @@ lacks access rights to any of them (except K8s system namespaces like 'list pods' execute at the cluster scope and fail at the first violation of access rights. -The watched namespace also needs to have a (possibly different) service account -in the case database pods need to talk to the K8s API (e.g. when using -K8s-native configuration of Patroni). The operator checks that the -`pod_service_account_name` exists in the target namespace, and, if not, deploys -there the `pod_service_account_definition` from the operator -[`Config`](../pkg/util/config/config.go) with the default value of: - -```yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: operator -``` - -In this definition, the operator overwrites the account's name to match -`pod_service_account_name` and the `default` namespace to match the target -namespace. The operator performs **no** further syncing of this account. - -## Non-default cluster domain - -If your cluster uses a DNS domain other than the default `cluster.local`, this -needs to be set in the operator configuration (`cluster_domain` variable). This -is used by the operator to connect to the clusters after creation. - ## Role-based access control for the operator The manifest [`operator-service-account-rbac.yaml`](../manifests/operator-service-account-rbac.yaml) defines the service account, cluster roles and bindings needed for the operator -to function under access control restrictions. To deploy the operator with this -RBAC policy use: +to function under access control restrictions. The file also includes a cluster +role `postgres-pod` with privileges for Patroni to watch and manage pods and +endpoints. To deploy the operator with this RBAC policies use: ```bash kubectl create -f manifests/configmap.yaml @@ -127,14 +110,14 @@ kubectl create -f manifests/postgres-operator.yaml kubectl create -f manifests/minimal-postgres-manifest.yaml ``` -### Service account and cluster roles +### Namespaced service account and role binding -Note that the service account is named `zalando-postgres-operator`. You may have -to change the `service_account_name` in the operator ConfigMap and -`serviceAccountName` in the `postgres-operator` deployment appropriately. This -is done intentionally to avoid breaking those setups that already work with the -default `operator` account. In the future the operator should ideally be run -under the `zalando-postgres-operator` service account. +For each namespace the operator watches it creates (or reads) a service account +and role binding to be used by the Postgres Pods. The service account is bound +to the `postgres-pod` cluster role. The name and definitions of these resources +can be [configured](reference/operator_parameters.md#kubernetes-resources). +Note, that the operator performs **no** further syncing of namespaced service +accounts and role bindings. ### Give K8s users access to create/list `postgresqls` diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index e3893ea31..ca972c22b 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -152,21 +152,22 @@ configuration they are grouped under the `kubernetes` key. service account used by Patroni running on individual Pods to communicate with the operator. Required even if native Kubernetes support in Patroni is not used, because Patroni keeps pod labels in sync with the instance role. - The default is `operator`. + The default is `postgres-pod`. * **pod_service_account_definition** - The operator tries to create the pod Service Account in the namespace that - doesn't define such an account using the YAML definition provided by this - option. If not defined, a simple definition that contains only the name will - be used. The default is empty. + On Postgres cluster creation the operator tries to create the service account + for the Postgres pods if it does not exist in the namespace. The internal + default service account definition (defines only the name) can be overwritten + with this parameter. Make sure to provide a valid YAML or JSON string. The + default is empty. * **pod_service_account_role_binding_definition** - This definition must bind pod service account to a role with permission + This definition must bind the pod service account to a role with permission sufficient for the pods to start and for Patroni to access K8s endpoints; service account on its own lacks any such rights starting with K8s v1.8. If not explicitly defined by the user, a simple definition that binds the - account to the operator's own 'zalando-postgres-operator' cluster role will - be used. The default is empty. + account to the 'postgres-pod' [cluster role](../../manifests/operator-service-account-rbac.yaml#L198) + will be used. The default is empty. * **pod_terminate_grace_period** Postgres pods are [terminated forcefully](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods) diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index d26c83edf..6b00c30f4 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -63,7 +63,7 @@ data: pod_label_wait_timeout: 10m pod_management_policy: "ordered_ready" pod_role_label: spilo-role - pod_service_account_name: "zalando-postgres-operator" + pod_service_account_name: "postgres-pod" pod_terminate_grace_period: 5m # postgres_superuser_teams: "postgres_superusers" # protected_role_names: "admin" diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 5e43cc03b..80fcd89ef 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: zalando-postgres-operator + name: postgres-operator namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: zalando-postgres-operator + name: postgres-operator rules: # all verbs allowed for custom operator resources - apiGroups: @@ -18,7 +18,14 @@ rules: - postgresqls/status - operatorconfigurations verbs: - - "*" + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch # to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io @@ -48,7 +55,8 @@ rules: - get - list - patch - - watch # needed if zalando-postgres-operator account is used for pods as well + - update + - watch # to CRUD secrets for database access - apiGroups: - "" @@ -96,6 +104,7 @@ rules: - get - list - watch + - update - patch # to resize the filesystem in Spilo pods when increasing volume size - apiGroups: @@ -126,6 +135,18 @@ rules: - get - list - patch +# to CRUD cron jobs for logical backups +- apiGroups: + - batch + resources: + - cronjobs + verbs: + - create + - delete + - get + - list + - patch + - update # to get namespaces operator resources can run in - apiGroups: - "" @@ -150,39 +171,82 @@ rules: verbs: - get - create -# to create role bindings to the operator service account +# to create role bindings to the postgres-pod service account - apiGroups: - - "rbac.authorization.k8s.io" + - rbac.authorization.k8s.io resources: - rolebindings verbs: - get - create -# to CRUD cron jobs for logical backups +# to grant privilege to run privileged pods - apiGroups: - - batch + - extensions resources: - - cronjobs + - podsecuritypolicies + resourceNames: + - privileged verbs: - - create - - delete - - get - - list - - patch - - update + - use --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: zalando-postgres-operator + name: postgres-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: zalando-postgres-operator + name: postgres-operator subjects: - kind: ServiceAccount -# note: the cluster role binding needs to be defined -# for every namespace the operator service account lives in. - name: zalando-postgres-operator + name: postgres-operator namespace: default + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: postgres-pod +rules: +# Patroni needs to watch and manage endpoints +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# Patroni needs to watch pods +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch +# to let Patroni create a headless service +- apiGroups: + - "" + resources: + - services + verbs: + - create +# to run privileged pods +- apiGroups: + - extensions + resources: + - podsecuritypolicies + resourceNames: + - privileged + verbs: + - use diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index a06abfc68..e3bc3e3e4 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -12,7 +12,7 @@ spec: labels: name: postgres-operator spec: - serviceAccountName: zalando-postgres-operator + serviceAccountName: postgres-operator containers: - name: postgres-operator image: registry.opensource.zalan.do/acid/postgres-operator:v1.3.1 diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index efd1a5396..695a4e9c5 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -45,7 +45,7 @@ configuration: # pod_priority_class_name: "" pod_role_label: spilo-role # pod_service_account_definition: "" - pod_service_account_name: zalando-postgres-operator + pod_service_account_name: postgres-pod # pod_service_account_role_binding_definition: "" pod_terminate_grace_period: 5m secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" diff --git a/manifests/user-facing-clusterroles.yaml b/manifests/user-facing-clusterroles.yaml index 800aafdb9..64392a19a 100644 --- a/manifests/user-facing-clusterroles.yaml +++ b/manifests/user-facing-clusterroles.yaml @@ -11,7 +11,14 @@ rules: - postgresqls - postgresqls/status verbs: - - "*" + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch --- apiVersion: rbac.authorization.k8s.io/v1 @@ -48,4 +55,3 @@ rules: - get - list - watch - diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index f67d99c1d..3c49b9a13 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -161,11 +161,12 @@ func (c *Controller) initPodServiceAccount() { if c.opConfig.PodServiceAccountDefinition == "" { c.opConfig.PodServiceAccountDefinition = ` - { "apiVersion": "v1", - "kind": "ServiceAccount", - "metadata": { - "name": "operator" - } + { + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": { + "name": "postgres-pod" + } }` } @@ -175,13 +176,13 @@ func (c *Controller) initPodServiceAccount() { switch { case err != nil: - panic(fmt.Errorf("Unable to parse pod service account definition from the operator config map: %v", err)) + panic(fmt.Errorf("Unable to parse pod service account definition from the operator configuration: %v", err)) case groupVersionKind.Kind != "ServiceAccount": - panic(fmt.Errorf("pod service account definition in the operator config map defines another type of resource: %v", groupVersionKind.Kind)) + panic(fmt.Errorf("pod service account definition in the operator configuration defines another type of resource: %v", groupVersionKind.Kind)) default: c.PodServiceAccount = obj.(*v1.ServiceAccount) if c.PodServiceAccount.Name != c.opConfig.PodServiceAccountName { - c.logger.Warnf("in the operator config map, the pod service account name %v does not match the name %v given in the account definition; using the former for consistency", c.opConfig.PodServiceAccountName, c.PodServiceAccount.Name) + c.logger.Warnf("in the operator configuration, the pod service account name %v does not match the name %v given in the account definition; using the former for consistency", c.opConfig.PodServiceAccountName, c.PodServiceAccount.Name) c.PodServiceAccount.Name = c.opConfig.PodServiceAccountName } c.PodServiceAccount.Namespace = "" @@ -223,9 +224,9 @@ func (c *Controller) initRoleBinding() { switch { case err != nil: - panic(fmt.Errorf("Unable to parse the definition of the role binding for the pod service account definition from the operator config map: %v", err)) + panic(fmt.Errorf("unable to parse the definition of the role binding for the pod service account definition from the operator configuration: %v", err)) case groupVersionKind.Kind != "RoleBinding": - panic(fmt.Errorf("role binding definition in the operator config map defines another type of resource: %v", groupVersionKind.Kind)) + panic(fmt.Errorf("role binding definition in the operator configuration defines another type of resource: %v", groupVersionKind.Kind)) default: c.PodServiceAccountRoleBinding = obj.(*rbacv1.RoleBinding) c.PodServiceAccountRoleBinding.Namespace = "" diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index e4e429abb..ec4af6427 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -96,7 +96,7 @@ type Config struct { DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p16"` Sidecars map[string]string `name:"sidecar_docker_images"` // default name `operator` enables backward compatibility with the older ServiceAccountName field - PodServiceAccountName string `name:"pod_service_account_name" default:"operator"` + PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` // value of this string must be valid JSON or YAML; see initPodServiceAccount PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""` PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""` diff --git a/ui/manifests/ui-service-account-rbac.yaml b/ui/manifests/ui-service-account-rbac.yaml index f0a6e8bb7..2e09797a0 100644 --- a/ui/manifests/ui-service-account-rbac.yaml +++ b/ui/manifests/ui-service-account-rbac.yaml @@ -61,7 +61,5 @@ roleRef: name: postgres-operator-ui subjects: - kind: ServiceAccount -# note: the cluster role binding needs to be defined -# for every namespace the operator-ui service account lives in. name: postgres-operator-ui namespace: default From d5660f65bbef956c69bc7d6ea46c7f519432917e Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 19 Feb 2020 12:58:24 +0100 Subject: [PATCH 047/104] [UI] add tab for monthly costs per cluster (#796) * add tab for monthly costs per cluster * sync run_local and update version number * lowering resources * some Makefile polishing and updated admin docs on UI * extend admin docs on UI * add api-service manifest for operator * set min limits in UI to default min limits of operator * reflect new UI helm charts in docs * make cluster name label configurable --- charts/postgres-operator-ui/Chart.yaml | 4 +- .../postgres-operator/templates/service.yaml | 5 +- docs/administrator.md | 72 ++++++++++++----- docs/operator-ui.md | 10 ++- docs/quickstart.md | 57 ++++++++++++- manifests/api-service.yaml | 12 +++ manifests/kustomization.yaml | 1 + ui/Makefile | 20 ++--- ui/app/package.json | 2 +- ui/app/src/new.tag.pug | 16 ++-- ui/app/src/postgresql.tag.pug | 3 + ui/app/src/postgresqls.tag.pug | 79 +++++++++++++++---- ui/manifests/deployment.yaml | 23 ++++-- ui/operator_ui/main.py | 12 +++ ui/operator_ui/spiloutils.py | 6 +- ui/requirements.txt | 14 ++-- ui/run_local.sh | 7 +- 17 files changed, 257 insertions(+), 86 deletions(-) create mode 100644 manifests/api-service.yaml diff --git a/charts/postgres-operator-ui/Chart.yaml b/charts/postgres-operator-ui/Chart.yaml index 4be7d8af1..4418675b6 100644 --- a/charts/postgres-operator-ui/Chart.yaml +++ b/charts/postgres-operator-ui/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: postgres-operator-ui version: 0.1.0 -appVersion: 1.2.0 +appVersion: 1.3.0 home: https://github.com/zalando/postgres-operator description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience keywords: @@ -12,6 +12,8 @@ keywords: - patroni - spilo maintainers: +- name: Zalando + email: opensource@zalando.de - name: siku4 email: sk@sik-net.de sources: diff --git a/charts/postgres-operator/templates/service.yaml b/charts/postgres-operator/templates/service.yaml index 52990c5d4..38ea9a062 100644 --- a/charts/postgres-operator/templates/service.yaml +++ b/charts/postgres-operator/templates/service.yaml @@ -8,6 +8,7 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} name: {{ template "postgres-operator.fullname" . }} spec: + type: ClusterIP ports: - port: 8080 protocol: TCP @@ -15,7 +16,3 @@ spec: selector: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/name: {{ template "postgres-operator.name" . }} - sessionAffinity: None - type: ClusterIP -status: - loadBalancer: {} \ No newline at end of file diff --git a/docs/administrator.md b/docs/administrator.md index 2175ae595..3597b65ca 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -480,37 +480,71 @@ A secret can be pre-provisioned in different ways: ## Setting up the Postgres Operator UI -With the v1.2 release the Postgres Operator is shipped with a browser-based +Since the v1.2 release the Postgres Operator is shipped with a browser-based configuration user interface (UI) that simplifies managing Postgres clusters -with the operator. The UI runs with Node.js and comes with it's own Docker -image. +with the operator. -Run NPM to continuously compile `tags/js` code. Basically, it creates an -`app.js` file in: `static/build/app.js` +### Building the UI image -``` -(cd ui/app && npm start) -``` - -To build the Docker image open a shell and change to the `ui` folder. Then run: +The UI runs with Node.js and comes with it's own Docker +image. However, installing Node.js to build the operator UI is not required. It +is handled via Docker containers when running: ```bash -docker build -t registry.opensource.zalan.do/acid/postgres-operator-ui:v1.2.0 . +make docker ``` -Apply all manifests for the `ui/manifests` folder to deploy the Postgres -Operator UI on K8s. For local tests you don't need the Ingress resource. +### Configure endpoints and options + +The UI talks to the K8s API server as well as the Postgres Operator [REST API](developer.md#debugging-the-operator). +K8s API server URLs are loaded from the machine's kubeconfig environment by +default. Alternatively, a list can also be passed when starting the Python +application with the `--cluster` option. + +The Operator API endpoint can be configured via the `OPERATOR_API_URL` +environment variables in the [deployment manifest](../ui/manifests/deployment.yaml#L40). +You can also expose the operator API through a [service](../manifests/api-service.yaml). +Some displayed options can be disabled from UI using simple flags under the +`OPERATOR_UI_CONFIG` field in the deployment. + +### Deploy the UI on K8s + +Now, apply all manifests from the `ui/manifests` folder to deploy the Postgres +Operator UI on K8s. Replace the image tag in the deployment manifest if you +want to test the image you've built with `make docker`. Make sure the pods for +the operator and the UI are both running. ```bash -kubectl apply -f ui/manifests +sed -e "s/\(image\:.*\:\).*$/\1$TAG/" manifests/deployment.yaml | kubectl apply -f manifests/ +kubectl get all -l application=postgres-operator-ui ``` -Make sure the pods for the operator and the UI are both running. For local -testing you need to apply proxying and port forwarding so that the UI can talk -to the K8s and Postgres Operator REST API. You can use the provided -`run_local.sh` script for this. Make sure it uses the correct URL to your K8s -API server, e.g. for minikube it would be `https://192.168.99.100:8443`. +### Local testing + +For local testing you need to apply K8s proxying and operator pod port +forwarding so that the UI can talk to the K8s and Postgres Operator REST API. +The Ingress resource is not needed. You can use the provided `run_local.sh` +script for this. Make sure that: + +* Python dependencies are installed on your machine +* the K8s API server URL is set for kubectl commands, e.g. for minikube it would usually be `https://192.168.99.100:8443`. +* the pod label selectors for port forwarding are correct + +When testing with minikube you have to build the image in its docker environment +(running `make docker` doesn't do it for you). From the `ui` directory execute: ```bash +# compile and build operator UI +make docker + +# build in image in minikube docker env +eval $(minikube docker-env) +docker build -t registry.opensource.zalan.do/acid/postgres-operator-ui:v1.3.0 . + +# apply UI manifests next to a running Postgres Operator +kubectl apply -f manifests/ + +# install python dependencies to run UI locally +pip3 install -r requirements ./run_local.sh ``` diff --git a/docs/operator-ui.md b/docs/operator-ui.md index 99526bb5c..7912e8264 100644 --- a/docs/operator-ui.md +++ b/docs/operator-ui.md @@ -31,9 +31,13 @@ status page. ![pgui-waiting-for-master](diagrams/pgui-waiting-for-master.png "Waiting for master pod") Usually, the startup should only take up to 1 minute. If you feel the process -got stuck click on the "Logs" button to inspect the operator logs. From the -"Status" field in the top menu you can also retrieve the logs and queue of each -worker the operator is using. The number of concurrent workers can be +got stuck click on the "Logs" button to inspect the operator logs. If the logs +look fine, but the UI seems to got stuck, check if you are have configured the +same [cluster name label](../ui/manifests/deployment.yaml#L45) like for the +[operator](../manifests/configmap.yaml#L13). + +From the "Status" field in the top menu you can also retrieve the logs and queue +of each worker the operator is using. The number of concurrent workers can be [configured](reference/operator_parameters.md#general). ![pgui-operator-logs](diagrams/pgui-operator-logs.png "Checking operator logs") diff --git a/docs/quickstart.md b/docs/quickstart.md index 8cc5bc0c0..d2c88b9a4 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -52,6 +52,7 @@ cd postgres-operator kubectl create -f manifests/configmap.yaml # configuration kubectl create -f manifests/operator-service-account-rbac.yaml # identity and permissions kubectl create -f manifests/postgres-operator.yaml # deployment +kubectl create -f manifests/api-service.yaml # operator API to be used by UI ``` There is a [Kustomization](https://github.com/kubernetes-sigs/kustomize) @@ -104,7 +105,7 @@ kubectl create -f https://operatorhub.io/install/postgres-operator.yaml This installs the operator in the `operators` namespace. More information can be found on [operatorhub.io](https://operatorhub.io/operator/postgres-operator). -## Create a Postgres cluster +## Check if Postgres Operator is running Starting the operator may take a few seconds. Check if the operator pod is running before applying a Postgres cluster manifest. @@ -115,7 +116,61 @@ kubectl get pod -l name=postgres-operator # if you've created the operator using helm chart kubectl get pod -l app.kubernetes.io/name=postgres-operator +``` +If the operator doesn't get into `Running` state, either check the latest K8s +events of the deployment or pod with `kubectl describe` or inspect the operator +logs: + +```bash +kubectl logs "$(kubectl get pod -l name=postgres-operator --output='name')" +``` + +## Deploy the operator UI + +In the following paragraphs we describe how to access and manage PostgreSQL +clusters from the command line with kubectl. But it can also be done from the +browser-based [Postgres Operator UI](operator-ui.md). Before deploying the UI +make sure the operator is running and its REST API is reachable through a +[K8s service](../manifests/api-service.yaml). The URL to this API must be +configured in the [deployment manifest](../ui/manifests/deployment.yaml#L43) +of the UI. + +To deploy the UI simply apply all its manifests files or use the UI helm chart: + +```bash +# manual deployment +kubectl apply -f ui/manifests/ + +# or helm chart +helm install postgres-operator-ui ./charts/postgres-operator-ui +``` + +Like with the operator, check if the UI pod gets into `Running` state: + +```bash +# if you've created the operator using yaml manifests +kubectl get pod -l name=postgres-operator-ui + +# if you've created the operator using helm chart +kubectl get pod -l app.kubernetes.io/name=postgres-operator-ui +``` + +You can now access the web interface by port forwarding the UI pod (mind the +label selector) and enter `localhost:8081` in your browser: + +```bash +kubectl port-forward "$(kubectl get pod -l name=postgres-operator-ui --output='name')" 8081 +``` + +Available option are explained in detail in the [UI docs](operator-ui.md). + +## Create a Postgres cluster + +If the operator pod is running it listens to new events regarding `postgresql` +resources. Now, it's time to submit your first Postgres cluster manifest. + +```bash # create a Postgres cluster kubectl create -f manifests/minimal-postgres-manifest.yaml ``` diff --git a/manifests/api-service.yaml b/manifests/api-service.yaml new file mode 100644 index 000000000..616448177 --- /dev/null +++ b/manifests/api-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres-operator +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + targetPort: 8080 + selector: + name: postgres-operator diff --git a/manifests/kustomization.yaml b/manifests/kustomization.yaml index a39627956..32d81d920 100644 --- a/manifests/kustomization.yaml +++ b/manifests/kustomization.yaml @@ -4,3 +4,4 @@ resources: - configmap.yaml - operator-service-account-rbac.yaml - postgres-operator.yaml +- api-service.yaml diff --git a/ui/Makefile b/ui/Makefile index e4eed45e5..f1cf16840 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -1,17 +1,6 @@ .PHONY: clean test appjs docker push mock -BINARY ?= postgres-operator-ui -BUILD_FLAGS ?= -v -CGO_ENABLED ?= 0 -ifeq ($(RACE),1) - BUILD_FLAGS += -race -a - CGO_ENABLED=1 -endif - -LOCAL_BUILD_FLAGS ?= $(BUILD_FLAGS) -LDFLAGS ?= -X=main.version=$(VERSION) - -IMAGE ?= registry.opensource.zalan.do/acid/$(BINARY) +IMAGE ?= registry.opensource.zalan.do/acid/postgres-operator-ui VERSION ?= $(shell git describe --tags --always --dirty) TAG ?= $(VERSION) GITHEAD = $(shell git rev-parse --short HEAD) @@ -32,8 +21,11 @@ appjs: docker run $(TTYFLAGS) -u $$(id -u) -v $$(pwd):/workdir -w /workdir/app node:10.1.0-alpine npm run build docker: appjs - docker build --build-arg "VERSION=$(VERSION)" -t "$(IMAGE):$(TAG)" . - @echo 'Docker image $(IMAGE):$(TAG) can now be used.' + echo `(env)` + echo "Tag ${TAG}" + echo "Version ${VERSION}" + echo "git describe $(shell git describe --tags --always --dirty)" + docker build --rm -t "$(IMAGE):$(TAG)" -f Dockerfile . push: docker docker push "$(IMAGE):$(TAG)" diff --git a/ui/app/package.json b/ui/app/package.json index 3fa66f7d0..d0528e0bd 100644 --- a/ui/app/package.json +++ b/ui/app/package.json @@ -1,6 +1,6 @@ { "name": "postgres-operator-ui", - "version": "1.0.0", + "version": "1.3.0", "description": "PostgreSQL Operator UI", "main": "src/app.js", "config": { diff --git a/ui/app/src/new.tag.pug b/ui/app/src/new.tag.pug index bd0cc764e..fe9d78226 100644 --- a/ui/app/src/new.tag.pug +++ b/ui/app/src/new.tag.pug @@ -408,7 +408,7 @@ new ref='cpuLimit' type='number' placeholder='{ cpu.state.limit.initialValue }' - min='1' + min='250' required value='{ cpu.state.limit.state }' onchange='{ cpu.state.limit.edit }' @@ -434,7 +434,7 @@ new onkeyup='{ memory.state.request.edit }' ) .input-group-addon - .input-units Gi + .input-units Mi .input-group .input-group-addon.resource-type Limit @@ -442,14 +442,14 @@ new ref='memoryLimit' type='number' placeholder='{ memory.state.limit.initialValue }' - min='1' + min='250' required value='{ memory.state.limit.state }' onchange='{ memory.state.limit.edit }' onkeyup='{ memory.state.limit.edit }' ) .input-group-addon - .input-units Gi + .input-units Mi .col-lg-3 help-general(config='{ opts.config }') @@ -519,10 +519,10 @@ new resources: requests: cpu: {{ cpu.state.request.state }}m - memory: {{ memory.state.request.state }}Gi + memory: {{ memory.state.request.state }}Mi limits: cpu: {{ cpu.state.limit.state }}m - memory: {{ memory.state.limit.state }}Gi{{#if restoring}} + memory: {{ memory.state.limit.state }}Mi{{#if restoring}} clone: cluster: "{{ backup.state.name.state }}" @@ -786,8 +786,8 @@ new return instance } - this.cpu = DynamicResource({ request: 100, limit: 1000 }) - this.memory = DynamicResource({ request: 1, limit: 1 }) + this.cpu = DynamicResource({ request: 100, limit: 500 }) + this.memory = DynamicResource({ request: 100, limit: 500 }) this.backup = DynamicSet({ type: () => 'empty', diff --git a/ui/app/src/postgresql.tag.pug b/ui/app/src/postgresql.tag.pug index 88e5e130b..be7173dbe 100644 --- a/ui/app/src/postgresql.tag.pug +++ b/ui/app/src/postgresql.tag.pug @@ -76,6 +76,9 @@ postgresql .alert.alert-danger(if='{ progress.requestStatus !== "OK" }') Create request failed .alert.alert-success(if='{ progress.requestStatus === "OK" }') Create request successful ({ new Date(progress.createdTimestamp).toLocaleString() }) + .alert.alert-info(if='{ !progress.postgresql }') PostgreSQL cluster manifest pending + .alert.alert-success(if='{ progress.postgresql }') PostgreSQL cluster manifest created + .alert.alert-info(if='{ !progress.statefulSet }') StatefulSet pending .alert.alert-success(if='{ progress.statefulSet }') StatefulSet created diff --git a/ui/app/src/postgresqls.tag.pug b/ui/app/src/postgresqls.tag.pug index 41d648737..250c175ec 100644 --- a/ui/app/src/postgresqls.tag.pug +++ b/ui/app/src/postgresqls.tag.pug @@ -45,12 +45,14 @@ postgresqls thead tr th(style='width: 120px') Team + th(style='width: 130px') Namespace + th Name th(style='width: 50px') Pods th(style='width: 140px') CPU th(style='width: 130px') Memory th(style='width: 100px') Size - th(style='width: 130px') Namespace - th Name + th(style='width: 120px') Cost/Month + th(stlye='width: 120px') tbody tr( @@ -58,19 +60,21 @@ postgresqls hidden='{ !namespaced_name.toLowerCase().includes(filter.state.toLowerCase()) }' ) td { team } - td { nodes } - td { cpu } / { cpu_limit } - td { memory } / { memory_limit } - td { volume_size } - td(style='white-space: pre') | { namespace } - td a( href='/#/status/{ cluster_path(this) }' ) | { name } + td { nodes } + td { cpu } / { cpu_limit } + td { memory } / { memory_limit } + td { volume_size } + td { calcCosts(nodes, cpu, memory, volume_size) }$ + + td + .btn-group.pull-right( aria-label='Cluster { qname } actions' @@ -124,12 +128,14 @@ postgresqls thead tr th(style='width: 120px') Team + th(style='width: 130px') Namespace + th Name th(style='width: 50px') Pods th(style='width: 140px') CPU th(style='width: 130px') Memory th(style='width: 100px') Size - th(style='width: 130px') Namespace - th Name + th(style='width: 120px') Cost/Month + th(stlye='width: 120px') tbody tr( @@ -137,20 +143,20 @@ postgresqls hidden='{ !namespaced_name.toLowerCase().includes(filter.state.toLowerCase()) }' ) td { team } - td { nodes } - td { cpu } / { cpu_limit } - td { memory } / { memory_limit } - td { volume_size } - td(style='white-space: pre') | { namespace } - td - a( href='/#/status/{ cluster_path(this) }' ) | { name } + td { nodes } + td { cpu } / { cpu_limit } + td { memory } / { memory_limit } + td { volume_size } + td { calcCosts(nodes, cpu, memory, volume_size) }$ + + td .btn-group.pull-right( aria-label='Cluster { qname } actions' @@ -223,6 +229,45 @@ postgresqls + '/' + encodeURI(cluster.name) ) + const calcCosts = this.calcCosts = (nodes, cpu, memory, disk) => { + costs = nodes * (toCores(cpu) * opts.config.cost_core + toMemory(memory) * opts.config.cost_memory + toDisk(disk) * opts.config.cost_ebs) + return costs.toFixed(2) + } + + const toDisk = this.toDisk = value => { + if(value.endsWith("Gi")) { + value = value.substring(0, value.length-2) + value = Number(value) + return value + } + + return value + } + + const toMemory = this.toMemory = value => { + if (value.endsWith("Mi")) { + value = value.substring(0, value.length-2) + value = Number(value) / 1000. + return value + } + else if(value.endsWith("Gi")) { + value = value.substring(0, value.length-2) + value = Number(value) + return value + } + + return value + } + + const toCores = this.toCores = value => { + if (value.endsWith("m")) { + value = value.substring(0, value.length-1) + value = Number(value) / 1000. + return value + } + return value + } + this.on('mount', () => jQuery .get('/postgresqls') diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index c270cbe11..477e4d655 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -4,23 +4,23 @@ metadata: name: "postgres-operator-ui" namespace: "default" labels: - application: "postgres-operator-ui" + name: "postgres-operator-ui" team: "acid" spec: replicas: 1 selector: matchLabels: - application: "postgres-operator-ui" + name: "postgres-operator-ui" template: metadata: labels: - application: "postgres-operator-ui" + name: "postgres-operator-ui" team: "acid" spec: serviceAccountName: postgres-operator-ui containers: - name: "service" - image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.2.0 + image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.3.0 ports: - containerPort: 8081 protocol: "TCP" @@ -32,8 +32,8 @@ spec: timeoutSeconds: 1 resources: limits: - cpu: "300m" - memory: "3000Mi" + cpu: "200m" + memory: "200Mi" requests: cpu: "100m" memory: "100Mi" @@ -41,7 +41,9 @@ spec: - name: "APP_URL" value: "http://localhost:8081" - name: "OPERATOR_API_URL" - value: "http://localhost:8080" + value: "http://postgres-operator:8080" + - name: "OPERATOR_CLUSTER_NAME_LABEL" + value: "cluster-name" - name: "TARGET_NAMESPACE" value: "default" - name: "TEAMS" @@ -60,9 +62,14 @@ spec: "replica_load_balancer_visible": true, "resources_visible": true, "users_visible": true, + "cost_ebs": 0.119, + "cost_core": 0.0575, + "cost_memory": 0.014375, "postgresql_versions": [ + "12", "11", "10", - "9.6" + "9.6", + "9.5" ] } diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index f34d16492..5a3054f0e 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -76,6 +76,7 @@ ACCESS_TOKEN_URL = getenv('ACCESS_TOKEN_URL') TOKENINFO_URL = getenv('OAUTH2_TOKEN_INFO_URL') OPERATOR_API_URL = getenv('OPERATOR_API_URL', 'http://postgres-operator') +OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-name') OPERATOR_UI_CONFIG = getenv('OPERATOR_UI_CONFIG', '{}') OPERATOR_UI_MAINTENANCE_CHECK = getenv('OPERATOR_UI_MAINTENANCE_CHECK', '{}') READ_ONLY_MODE = getenv('READ_ONLY_MODE', False) in [True, 'true'] @@ -84,6 +85,13 @@ SUPERUSER_TEAM = getenv('SUPERUSER_TEAM', 'acid') TARGET_NAMESPACE = getenv('TARGET_NAMESPACE') GOOGLE_ANALYTICS = getenv('GOOGLE_ANALYTICS', False) +# storage pricing, i.e. https://aws.amazon.com/ebs/pricing/ +COST_EBS = float(getenv('COST_EBS', 0.119)) # GB per month + +# compute costs, i.e. https://www.ec2instances.info/?region=eu-central-1&selected=m5.2xlarge +COST_CORE = 30.5 * 24 * float(getenv('COST_CORE', 0.0575)) # Core per hour m5.2xlarge / 8. +COST_MEMORY = 30.5 * 24 * float(getenv('COST_MEMORY', 0.014375)) # Memory GB m5.2xlarge / 32. + WALE_S3_ENDPOINT = getenv( 'WALE_S3_ENDPOINT', 'https+path://s3-eu-central-1.amazonaws.com:443', @@ -293,6 +301,9 @@ DEFAULT_UI_CONFIG = { 'dns_format_string': '{0}.{1}.{2}', 'pgui_link': '', 'static_network_whitelist': {}, + 'cost_ebs': COST_EBS, + 'cost_core': COST_CORE, + 'cost_memory': COST_MEMORY } @@ -1003,6 +1014,7 @@ def main(port, secret_key, debug, clusters: list): logger.info(f'App URL: {APP_URL}') logger.info(f'Authorize URL: {AUTHORIZE_URL}') logger.info(f'Operator API URL: {OPERATOR_API_URL}') + logger.info(f'Operator cluster name label: {OPERATOR_CLUSTER_NAME_LABEL}') logger.info(f'Readonly mode: {"enabled" if READ_ONLY_MODE else "disabled"}') # noqa logger.info(f'Spilo S3 backup bucket: {SPILO_S3_BACKUP_BUCKET}') logger.info(f'Spilo S3 backup prefix: {SPILO_S3_BACKUP_PREFIX}') diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py index 7f080e3c9..33d07d88a 100644 --- a/ui/operator_ui/spiloutils.py +++ b/ui/operator_ui/spiloutils.py @@ -3,7 +3,7 @@ from datetime import datetime, timezone from furl import furl from json import dumps from logging import getLogger -from os import environ +from os import environ, getenv from requests import Session from urllib.parse import urljoin from uuid import UUID @@ -16,6 +16,8 @@ logger = getLogger(__name__) session = Session() +OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-name') + def request(cluster, path, **kwargs): if 'timeout' not in kwargs: @@ -137,7 +139,7 @@ def read_pods(cluster, namespace, spilo_cluster): cluster=cluster, resource_type='pods', namespace=namespace, - label_selector={'version': spilo_cluster}, + label_selector={OPERATOR_CLUSTER_NAME_LABEL: spilo_cluster}, ) diff --git a/ui/requirements.txt b/ui/requirements.txt index f9bfdcfa0..2f1bd661a 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -1,14 +1,14 @@ Flask-OAuthlib==0.9.5 -Flask==1.0.2 -backoff==1.5.0 -boto3==1.5.14 -boto==2.48.0 +Flask==1.1.1 +backoff==1.8.1 +boto3==1.10.4 +boto==2.49.0 click==6.7 -furl==1.0.1 +furl==1.0.2 gevent==1.2.2 jq==0.1.6 json_delta>=2.0 kubernetes==3.0.0 -requests==2.20.1 +requests==2.22.0 stups-tokens>=1.1.19 -wal_e==1.1.0 \ No newline at end of file +wal_e==1.1.0 diff --git a/ui/run_local.sh b/ui/run_local.sh index 2951fe049..e331b2414 100755 --- a/ui/run_local.sh +++ b/ui/run_local.sh @@ -19,10 +19,15 @@ default_operator_ui_config='{ "nat_gateways_visible": false, "resources_visible": true, "users_visible": true, + "cost_ebs": 0.119, + "cost_core": 0.0575, + "cost_memory": 0.014375, "postgresql_versions": [ + "12", "11", "10", - "9.6" + "9.6", + "9.5" ], "static_network_whitelist": { "localhost": ["172.0.0.1/32"] From 54796945f6c5676e2860704a1d95c0d8abfce756 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 19 Feb 2020 14:19:55 +0100 Subject: [PATCH 048/104] added pinned werkzeug dep to UI requirements (#835) --- ui/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/requirements.txt b/ui/requirements.txt index 2f1bd661a..5d987416c 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -12,3 +12,4 @@ kubernetes==3.0.0 requests==2.22.0 stups-tokens>=1.1.19 wal_e==1.1.0 +werkzeug==0.16.1 From 742d7334a1fe69609c66b14878bfece896dd8f15 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 19 Feb 2020 15:01:01 +0100 Subject: [PATCH 049/104] use cluster-name as default label everywhere (#782) * use cluster-name as default label everywhere * fix e2e test --- charts/postgres-operator/values.yaml | 2 +- docs/user.md | 2 +- e2e/tests/test_e2e.py | 22 +++++++++++----------- manifests/configmap.yaml | 2 +- pkg/cluster/k8sres.go | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 60a0a1f04..78624e0bd 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -55,7 +55,7 @@ configKubernetes: # additional labels assigned to the cluster objects cluster_labels: application:spilo # label assigned to Kubernetes objects created by the operator - cluster_name_label: version + cluster_name_label: cluster-name # annotations attached to each database pod # custom_pod_annotations: "keya:valuea,keyb:valueb" diff --git a/docs/user.md b/docs/user.md index f81e11ede..e1baf9ad1 100644 --- a/docs/user.md +++ b/docs/user.md @@ -65,7 +65,7 @@ our test cluster. ```bash # get name of master pod of acid-minimal-cluster -export PGMASTER=$(kubectl get pods -o jsonpath={.items..metadata.name} -l application=spilo,version=acid-minimal-cluster,spilo-role=master) +export PGMASTER=$(kubectl get pods -o jsonpath={.items..metadata.name} -l application=spilo,cluster-name=acid-minimal-cluster,spilo-role=master) # set up port forward kubectl port-forward $PGMASTER 6432:5432 diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 2d81a0647..12106601e 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -65,7 +65,7 @@ class EndToEndTestCase(unittest.TestCase): ''' k8s = self.k8s - cluster_label = 'version=acid-minimal-cluster' + cluster_label = 'cluster-name=acid-minimal-cluster' # enable load balancer services pg_patch_enable_lbs = { @@ -113,7 +113,7 @@ class EndToEndTestCase(unittest.TestCase): Lower resource limits below configured minimum and let operator fix it ''' k8s = self.k8s - cluster_label = 'version=acid-minimal-cluster' + cluster_label = 'cluster-name=acid-minimal-cluster' _, failover_targets = k8s.get_pg_nodes(cluster_label) # configure minimum boundaries for CPU and memory limits @@ -172,7 +172,7 @@ class EndToEndTestCase(unittest.TestCase): k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") k8s.wait_for_pod_start("spilo-role=master", self.namespace) - self.assert_master_is_unique(self.namespace, version="acid-test-cluster") + self.assert_master_is_unique(self.namespace, "acid-test-cluster") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_scaling(self): @@ -180,7 +180,7 @@ class EndToEndTestCase(unittest.TestCase): Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. ''' k8s = self.k8s - labels = "version=acid-minimal-cluster" + labels = "cluster-name=acid-minimal-cluster" k8s.wait_for_pg_to_scale(3) self.assertEqual(3, k8s.count_pods_with_label(labels)) @@ -196,7 +196,7 @@ class EndToEndTestCase(unittest.TestCase): Add taint "postgres=:NoExecute" to node with master. This must cause a failover. ''' k8s = self.k8s - cluster_label = 'version=acid-minimal-cluster' + cluster_label = 'cluster-name=acid-minimal-cluster' # get nodes of master and replica(s) (expected target of new master) current_master_node, failover_targets = k8s.get_pg_nodes(cluster_label) @@ -334,9 +334,9 @@ class EndToEndTestCase(unittest.TestCase): "foo": "bar", } self.assertTrue(k8s.check_service_annotations( - "version=acid-service-annotations,spilo-role=master", annotations)) + "cluster-name=acid-service-annotations,spilo-role=master", annotations)) self.assertTrue(k8s.check_service_annotations( - "version=acid-service-annotations,spilo-role=replica", annotations)) + "cluster-name=acid-service-annotations,spilo-role=replica", annotations)) # clean up unpatch_custom_service_annotations = { @@ -346,14 +346,14 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(unpatch_custom_service_annotations) - def assert_master_is_unique(self, namespace='default', version="acid-minimal-cluster"): + def assert_master_is_unique(self, namespace='default', clusterName="acid-minimal-cluster"): ''' Check that there is a single pod in the k8s cluster with the label "spilo-role=master" To be called manually after operations that affect pods ''' k8s = self.k8s - labels = 'spilo-role=master,version=' + version + labels = 'spilo-role=master,cluster-name=' + clusterName num_of_master_pods = k8s.count_pods_with_label(labels, namespace) self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) @@ -438,7 +438,7 @@ class K8s: _ = self.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body) - labels = 'version=acid-minimal-cluster' + labels = 'cluster-name=acid-minimal-cluster' while self.count_pods_with_label(labels) != number_of_instances: time.sleep(self.RETRY_TIMEOUT_SEC) @@ -448,7 +448,7 @@ class K8s: def wait_for_master_failover(self, expected_master_nodes, namespace='default'): pod_phase = 'Failing over' new_master_node = '' - labels = 'spilo-role=master,version=acid-minimal-cluster' + labels = 'spilo-role=master,cluster-name=acid-minimal-cluster' while (pod_phase != 'Running') or (new_master_node not in expected_master_nodes): pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 6b00c30f4..4289a134c 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -10,7 +10,7 @@ data: cluster_domain: cluster.local cluster_history_entries: "1000" cluster_labels: application:spilo - cluster_name_label: version + cluster_name_label: cluster-name # custom_service_annotations: "keyx:valuez,keya:valuea" # custom_pod_annotations: "keya:valuea,keyb:valueb" db_hosted_zone: db.example.com diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index e6561e0f3..4468c8428 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1498,8 +1498,8 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { ) labels := map[string]string{ - "version": c.Name, - "application": "spilo-logical-backup", + c.OpConfig.ClusterNameLabel: c.Name, + "application": "spilo-logical-backup", } podAffinityTerm := v1.PodAffinityTerm{ LabelSelector: &metav1.LabelSelector{ From e2a9b0391343e6c7251d72f5612306aa6af46a3e Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 20 Feb 2020 16:21:21 +0100 Subject: [PATCH 050/104] bump spilo version to latest release (#836) --- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- manifests/complete-postgres-manifest.yaml | 2 +- manifests/configmap.yaml | 2 +- manifests/postgresql-operator-default-configuration.yaml | 2 +- pkg/util/config/config.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 08c255a04..195a03380 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -24,7 +24,7 @@ configGeneral: # etcd connection string for Patroni. Empty uses K8s-native DCS. etcd_host: "" # Spilo docker image - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p16 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 # max number of instances in Postgres cluster. -1 = no limit min_instances: -1 # min number of instances in Postgres cluster. -1 = no limit diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 78624e0bd..8b52a7d67 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -24,7 +24,7 @@ configGeneral: # etcd connection string for Patroni. Empty uses K8s-native DCS. etcd_host: "" # Spilo docker image - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p16 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 # max number of instances in Postgres cluster. -1 = no limit min_instances: "-1" # min number of instances in Postgres cluster. -1 = no limit diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 9e3b891c3..5ae817ca3 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -5,7 +5,7 @@ metadata: # labels: # environment: demo spec: - dockerImage: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p16 + dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 teamId: "acid" volume: size: 1Gi diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 4289a134c..aa7bef034 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -19,7 +19,7 @@ data: # default_cpu_request: 100m # default_memory_limit: 500Mi # default_memory_request: 100Mi - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p16 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 # enable_admin_role_for_users: "true" # enable_crd_validation: "true" # enable_database_access: "true" diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 695a4e9c5..bdb131fc5 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -5,7 +5,7 @@ metadata: configuration: # enable_crd_validation: true etcd_host: "" - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p16 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 # enable_shm_volume: true max_instances: -1 min_instances: -1 diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index ec4af6427..0e88c60d7 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -93,7 +93,7 @@ type Config struct { WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p16"` + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` Sidecars map[string]string `name:"sidecar_docker_images"` // default name `operator` enables backward compatibility with the older ServiceAccountName field PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` From 7b94060d1754f12ba2466b8011bb84ea09ac051d Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 21 Feb 2020 16:36:23 +0100 Subject: [PATCH 051/104] fix validation for S3ForcePathStyle (#841) --- charts/postgres-operator/crds/postgresqls.yaml | 2 +- manifests/postgresql.crd.yaml | 2 +- pkg/apis/acid.zalan.do/v1/crds.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index b4b676236..af535e2c8 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -94,7 +94,7 @@ spec: s3_secret_access_key: type: string s3_force_path_style: - type: string + type: boolean s3_wal_path: type: string timestamp: diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 276bc94b8..453916b26 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -58,7 +58,7 @@ spec: s3_secret_access_key: type: string s3_force_path_style: - type: string + type: boolean s3_wal_path: type: string timestamp: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 4cfc9a9e6..28dfa1566 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -160,7 +160,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ Type: "string", }, "s3_force_path_style": { - Type: "string", + Type: "boolean", }, "s3_wal_path": { Type: "string", From b997e3682f2281188cacd0bf96ac4bca50d686f1 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 24 Feb 2020 15:14:14 +0100 Subject: [PATCH 052/104] be more permissive with standbys (#842) * be more permissive with standbys * reflect feedback and updated docs --- docs/administrator.md | 10 +- docs/reference/operator_parameters.md | 6 +- docs/user.md | 134 ++++++++++++++++++-------- pkg/cluster/k8sres.go | 10 +- 4 files changed, 109 insertions(+), 51 deletions(-) diff --git a/docs/administrator.md b/docs/administrator.md index 3597b65ca..9d877c783 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -11,11 +11,11 @@ switchover (planned failover) of the master to the Pod with new minor version. The switch should usually take less than 5 seconds, still clients have to reconnect. -Major version upgrades are supported via [cloning](user.md#clone-directly). The -new cluster manifest must have a higher `version` string than the source cluster -and will be created from a basebackup. Depending of the cluster size, downtime -in this case can be significant as writes to the database should be stopped and -all WAL files should be archived first before cloning is started. +Major version upgrades are supported via [cloning](user.md#how-to-clone-an-existing-postgresql-cluster). +The new cluster manifest must have a higher `version` string than the source +cluster and will be created from a basebackup. Depending of the cluster size, +downtime in this case can be significant as writes to the database should be +stopped and all WAL files should be archived first before cloning is started. Note, that simply changing the version string in the `postgresql` manifest does not work at present and leads to errors. Neither Patroni nor Postgres Operator diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index ca972c22b..ad519b657 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -110,8 +110,10 @@ Those are top-level keys, containing both leaf keys and groups. * **min_instances** operator will run at least the number of instances for any given Postgres - cluster equal to the value of this parameter. When `-1` is specified, no - limits are applied. The default is `-1`. + cluster equal to the value of this parameter. Standby clusters can still run + with `numberOfInstances: 1` as this is the [recommended setup](../user.md#setting-up-a-standby-cluster). + When `-1` is specified for `min_instances`, no limits are applied. The default + is `-1`. * **resync_period** period between consecutive sync requests. The default is `30m`. diff --git a/docs/user.md b/docs/user.md index e1baf9ad1..91a010b9c 100644 --- a/docs/user.md +++ b/docs/user.md @@ -254,29 +254,22 @@ spec: ## How to clone an existing PostgreSQL cluster -You can spin up a new cluster as a clone of the existing one, using a clone +You can spin up a new cluster as a clone of the existing one, using a `clone` section in the spec. There are two options here: -* Clone directly from a source cluster using `pg_basebackup` -* Clone from an S3 bucket +* Clone from an S3 bucket (recommended) +* Clone directly from a source cluster -### Clone directly - -```yaml -spec: - clone: - cluster: "acid-batman" -``` - -Here `cluster` is a name of a source cluster that is going to be cloned. The -cluster to clone is assumed to be running and the clone procedure invokes -`pg_basebackup` from it. The operator will setup the cluster to be cloned to -connect to the service of the source cluster by name (if the cluster is called -test, then the connection string will look like host=test port=5432), which -means that you can clone only from clusters within the same namespace. +Note, that cloning can also be used for [major version upgrades](administrator.md#minor-and-major-version-upgrade) +of PostgreSQL. ### Clone from S3 +Cloning from S3 has the advantage that there is no impact on your production +database. A new Postgres cluster is created by restoring the data of another +source cluster. If you create it in the same Kubernetes environment, use a +different name. + ```yaml spec: clone: @@ -287,7 +280,8 @@ spec: Here `cluster` is a name of a source cluster that is going to be cloned. A new cluster will be cloned from S3, using the latest backup before the `timestamp`. -In this case, `uid` field is also mandatory - operator will use it to find a +Note, that a time zone is required for `timestamp` in the format of +00:00 which +is UTC. The `uid` field is also mandatory. The operator will use it to find a correct key inside an S3 bucket. You can find this field in the metadata of the source cluster: @@ -299,9 +293,6 @@ metadata: uid: efd12e58-5786-11e8-b5a7-06148230260c ``` -Note that timezone is required for `timestamp`. Otherwise, offset is relative -to UTC, see [RFC 3339 section 5.6) 3339 section 5.6](https://www.ietf.org/rfc/rfc3339.txt). - For non AWS S3 following settings can be set to support cloning from other S3 implementations: @@ -317,14 +308,35 @@ spec: s3_force_path_style: true ``` +### Clone directly + +Another way to get a fresh copy of your source DB cluster is via basebackup. To +use this feature simply leave out the timestamp field from the clone section. +The operator will connect to the service of the source cluster by name. If the +cluster is called test, then the connection string will look like host=test +port=5432), which means that you can clone only from clusters within the same +namespace. + +```yaml +spec: + clone: + cluster: "acid-batman" +``` + +Be aware that on a busy source database this can result in an elevated load! + ## Setting up a standby cluster -Standby clusters are like normal cluster but they are streaming from a remote -cluster. As the first version of this feature, the only scenario covered by -operator is to stream from a WAL archive of the master. Following the more -popular infrastructure of using Amazon's S3 buckets, it is mentioned as -`s3_wal_path` here. To start a cluster as standby add the following `standby` -section in the YAML file: +Standby cluster is a [Patroni feature](https://github.com/zalando/patroni/blob/master/docs/replica_bootstrap.rst#standby-cluster) +that first clones a database, and keeps replicating changes afterwards. As the +replication is happening by the means of archived WAL files (stored on S3 or +the equivalent of other cloud providers), the standby cluster can exist in a +different location than its source database. Unlike cloning, the PostgreSQL +version between source and target cluster has to be the same. + +To start a cluster as standby, add the following `standby` section in the YAML +file and specify the S3 bucket path. An empty path will result in an error and +no statefulset will be created. ```yaml spec: @@ -332,20 +344,62 @@ spec: s3_wal_path: "s3 bucket path to the master" ``` -Things to note: +At the moment, the operator only allows to stream from the WAL archive of the +master. Thus, it is recommended to deploy standby clusters with only [one pod](../manifests/standby-manifest.yaml#L10). +You can raise the instance count when detaching. Note, that the same pod role +labels like for normal clusters are used: The standby leader is labeled as +`master`. -- An empty string in the `s3_wal_path` field of the standby cluster will result - in an error and no statefulset will be created. -- Only one pod can be deployed for stand-by cluster. -- To manually promote the standby_cluster, use `patronictl` and remove config - entry. -- There is no way to transform a non-standby cluster to a standby cluster - through the operator. Adding the standby section to the manifest of a running - Postgres cluster will have no effect. However, it can be done through Patroni - by adding the [standby_cluster](https://github.com/zalando/patroni/blob/bd2c54581abb42a7d3a3da551edf0b8732eefd27/docs/replica_bootstrap.rst#standby-cluster) - section using `patronictl edit-config`. Note that the transformed standby - cluster will not be doing any streaming. It will be in standby mode and allow - read-only transactions only. +### Providing credentials of source cluster + +A standby cluster is replicating the data (including users and passwords) from +the source database and is read-only. The system and application users (like +standby, postgres etc.) all have a password that does not match the credentials +stored in secrets which are created by the operator. One solution is to create +secrets beforehand and paste in the credentials of the source cluster. +Otherwise, you will see errors in the Postgres logs saying users cannot log in +and the operator logs will complain about not being able to sync resources. +This, however, can safely be ignored as it will be sorted out once the cluster +is detached from the source (and it’s still harmless if you don’t plan to). + +You can also edit the secrets afterwards. Find them by: + +```bash +kubectl get secrets --all-namespaces | grep +``` + +### Promote the standby + +One big advantage of standby clusters is that they can be promoted to a proper +database cluster. This means it will stop replicating changes from the source, +and start accept writes itself. This mechanism makes it possible to move +databases from one place to another with minimal downtime. Currently, the +operator does not support promoting a standby cluster. It has to be done +manually using `patronictl edit-config` inside the postgres container of the +standby leader pod. Remove the following lines from the YAML structure and the +leader promotion happens immediately. Before doing so, make sure that the +standby is not behind the source database. + +```yaml +standby_cluster: + create_replica_methods: + - bootstrap_standby_with_wale + - basebackup_fast_xlog + restore_command: envdir "/home/postgres/etc/wal-e.d/env-standby" /scripts/restore_command.sh + "%f" "%p" +``` + +Finally, remove the `standby` section from the postgres cluster manifest. + +### Turn a normal cluster into a standby + +There is no way to transform a non-standby cluster to a standby cluster through +the operator. Adding the `standby` section to the manifest of a running +Postgres cluster will have no effect. But, as explained in the previous +paragraph it can be done manually through `patronictl edit-config`. This time, +by adding the `standby_cluster` section to the Patroni configuration. However, +the transformed standby cluster will not be doing any streaming. It will be in +standby mode and allow read-only transactions only. ## Sidecar Support diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 4468c8428..e2251a67c 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1048,11 +1048,13 @@ func (c *Cluster) getNumberOfInstances(spec *acidv1.PostgresSpec) int32 { cur := spec.NumberOfInstances newcur := cur - /* Limit the max number of pods to one, if this is standby-cluster */ if spec.StandbyCluster != nil { - c.logger.Info("Standby cluster can have maximum of 1 pod") - min = 1 - max = 1 + if newcur == 1 { + min = newcur + max = newcur + } else { + c.logger.Warningf("operator only supports standby clusters with 1 pod") + } } if max >= 0 && newcur > max { newcur = max From fb9ef11e4e0199ca0bde1d87f0b87285e8077917 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 24 Feb 2020 17:48:14 +0100 Subject: [PATCH 053/104] align UI pipeline with operator (#844) * align UI pipeline with operator --- delivery.yaml | 21 +++++++-------------- ui/Makefile | 13 +++++++++---- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index be35d3e27..144448ea9 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -66,20 +66,13 @@ pipeline: - desc: 'Build and push Docker image' cmd: | cd ui - image_base='registry-write.opensource.zalan.do/acid/postgres-operator-ui' - if [[ "${CDP_TARGET_BRANCH}" == 'master' && -z "${CDP_PULL_REQUEST_NUMBER}" ]] + IS_PR_BUILD=${CDP_PULL_REQUEST_NUMBER+"true"} + if [[ ${CDP_TARGET_BRANCH} == "master" && ${IS_PR_BUILD} != "true" ]] then - image="${image_base}" + IMAGE=registry-write.opensource.zalan.do/acid/postgres-operator-ui else - image="${image_base}-test" + IMAGE=registry-write.opensource.zalan.do/acid/postgres-operator-ui-test fi - image_with_tag="${image}:c${CDP_BUILD_VERSION}" - - if docker pull "${image}" - then - docker build --cache-from="${image}" -t "${image_with_tag}" . - else - docker build -t "${image_with_tag}" . - fi - - docker push "${image_with_tag}" + export IMAGE + make docker + make push diff --git a/ui/Makefile b/ui/Makefile index f1cf16840..e7d5df674 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -5,9 +5,13 @@ VERSION ?= $(shell git describe --tags --always --dirty) TAG ?= $(VERSION) GITHEAD = $(shell git rev-parse --short HEAD) GITURL = $(shell git config --get remote.origin.url) -GITSTATU = $(shell git status --porcelain || echo 'no changes') +GITSTATUS = $(shell git status --porcelain || echo 'no changes') TTYFLAGS = $(shell test -t 0 && echo '-it') +ifdef CDP_PULL_REQUEST_NUMBER + CDP_TAG := -${CDP_BUILD_VERSION} +endif + default: docker clean: @@ -24,11 +28,12 @@ docker: appjs echo `(env)` echo "Tag ${TAG}" echo "Version ${VERSION}" + echo "CDP tag ${CDP_TAG}" echo "git describe $(shell git describe --tags --always --dirty)" - docker build --rm -t "$(IMAGE):$(TAG)" -f Dockerfile . + docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)" -f Dockerfile . -push: docker - docker push "$(IMAGE):$(TAG)" +push: + docker push "$(IMAGE):$(TAG)$(CDP_TAG)" mock: docker run -it -p 8080:8080 "$(IMAGE):$(TAG)" --mock From b24da3201ceeeb8e23ffbb4fd99076822f388898 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 25 Feb 2020 09:50:54 +0100 Subject: [PATCH 054/104] bump version to 1.4.0 + some polishing (#839) * bump version to 1.4.0 + some polishing * align version for UI chart * update user docs to warn for standby replicas * minor log message changes for RBAC resources --- charts/postgres-operator-ui/Chart.yaml | 4 +-- charts/postgres-operator-ui/index.yaml | 29 ++++++++++++++++++ .../postgres-operator-ui-1.4.0.tgz | Bin 0 -> 3517 bytes charts/postgres-operator-ui/values.yaml | 2 +- charts/postgres-operator/Chart.yaml | 4 +-- charts/postgres-operator/index.yaml | 28 +++++++++++++++-- .../postgres-operator-1.4.0.tgz | Bin 0 -> 42200 bytes .../templates/clusterrole.yaml | 6 ++-- charts/postgres-operator/values-crd.yaml | 8 ++++- charts/postgres-operator/values.yaml | 8 ++++- docs/user.md | 11 ++++--- manifests/configmap.yaml | 2 ++ manifests/operator-service-account-rbac.yaml | 6 ++-- manifests/postgres-operator.yaml | 2 +- pkg/controller/controller.go | 2 +- pkg/controller/postgresql.go | 14 ++++----- pkg/util/config/config.go | 11 +++---- 17 files changed, 102 insertions(+), 35 deletions(-) create mode 100644 charts/postgres-operator-ui/index.yaml create mode 100644 charts/postgres-operator-ui/postgres-operator-ui-1.4.0.tgz create mode 100644 charts/postgres-operator/postgres-operator-1.4.0.tgz diff --git a/charts/postgres-operator-ui/Chart.yaml b/charts/postgres-operator-ui/Chart.yaml index 4418675b6..a6e46ab3e 100644 --- a/charts/postgres-operator-ui/Chart.yaml +++ b/charts/postgres-operator-ui/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: postgres-operator-ui -version: 0.1.0 -appVersion: 1.3.0 +version: 1.4.0 +appVersion: 1.4.0 home: https://github.com/zalando/postgres-operator description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience keywords: diff --git a/charts/postgres-operator-ui/index.yaml b/charts/postgres-operator-ui/index.yaml new file mode 100644 index 000000000..0cd03d6e5 --- /dev/null +++ b/charts/postgres-operator-ui/index.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +entries: + postgres-operator-ui: + - apiVersion: v1 + appVersion: 1.4.0 + created: "2020-02-24T15:32:47.610967635+01:00" + description: Postgres Operator UI provides a graphical interface for a convenient + database-as-a-service user experience + digest: 00e0eff7056d56467cd5c975657fbb76c8d01accd25a4b7aca81bc42aeac961d + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - ui + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + - email: sk@sik-net.de + name: siku4 + name: postgres-operator-ui + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-ui-1.4.0.tgz + version: 1.4.0 +generated: "2020-02-24T15:32:47.610348278+01:00" diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.4.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.4.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..8d1276dd16ab28ad5d176c810c86ad617c682359 GIT binary patch literal 3517 zcmV;u4MOrCiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH;Na@#nP`OT;3rEjZdvLQuE@<&)Tck6Y0Dc4RMmF;9}Z)?g0 zku3>h2;cyqY{$_(`xU?sk(6x7iN~4T3O^)}Xf%LEqr1^)A{9!HB`Qy(D2W!*%V_6l zLZtR@$&|f#iWCGvu-EU~{~!q3|G|ED|3%o_>-P6|_q$>EA_#YT-S7njPty9DlqpT* zi{Pu9B0;v=;nxFy# zV@c8pjR^zFHOdi*5k^7+fmrYva*A9#0144#NECWRdBjsFXEa906iUFG6eUF-qcagx zj9{X5s-jM3O!Xui`mvaHJ`qNEB05XF>s;aOwUCL591qG;$AhZegN!;J#7ty~$BCvh zG>}9~!A*fmDHG0=P_7B(C@mtK63QZjge#HB82?&yz)#Qt;KX$EfAUugaCP-pMXx-L zx(=r3Rn~X%{HmqrbJjMV|CQ^1Mp%aG$ppZf_21p?_uA`!cXxNY{+}c5 zz#ANqjA_Ux-7eQvOyN5_a513@s9EU$Jbd@FHxhD6w8jKRl%WB5gE1o#VMZh+Lxu{p zfFVLkl)}Ua1p&7rlb~^$GNMuWjw3N;G$uzP<2r)SadwQf7}0Si5i$yNO2+8e-^c-w zIHpR=+o;O2w?72+qt0;WPyEet4$X|lX(%z^{ZWNc)gT{i$}#@IkG8sA3Xo{Ua~ z9!ONkwd*)?NMh549XL-hrXz|9t|zD`DBSihhqB4ddedn}P?OdKe35onpA z;}l}=FpllO??Fr8yaBf!vB7gOK`U@Tj6hGYjYA6~Y<&xQG6F6%DAYiBQlK)EY>>SI zlH;KY#>frTFo8@_KDKyA8DqA*oX=9wH7C(136U(Q<5W2_Y(}S4+Z;fgW)bv)U}~SH zIJL9a3xePswNv_AhN=M^wt+(n+iw5(^rLc+&s0w6lXrvnXBUU37ZLnpLK#*n0%BwP zSWc;8B~w%>i7^5<$=A5MM3$l#B{Lf0 z=-bp(Ibxa8C{G3kV=APMz|9+IV5kW0=5{i7;3^xUgFv2Ohm;oo_&ZQ zqdwk{Y0A(Sa%_vhm;nI(GN-Aj?P;d+k{bG*9F3P_pOO;$ zw-)garj`1?lFpse_ZP?KzP{04)5bdczq{9O>Hqy8+~4Z|=SUX_C7W9PZWz7#0 zqQH=s^|Hq4L|3^68|LQrCg@b^k)<%1tlWQnq-p|xKzF;=xT z3x62iTEe_JJvXq%R#qv-Kl;we2xKlHb0o2_eg;+8nYpE{;hHjL4E)UOnTXkLna&rz z-HgtqVZI;_9aW?)3oG)LF!us%M1Q6hnEu0#CHiUeDX#UiB1?{?p;Z=rZM=PS2UAJ8 z9)bI3<^5T?Eo^zak72s|@-1d8jeb&W#2z!H&HdtOZE`Cj|3G=nvSd9|_&-*Y&_rwJ z1Rfehja7{_DEv-_O)a|?yq`lcGDNoN_-qyT-U+JnPo6IvmGIf@^qDa>+`qWsUb>H( zxo<6izj0cl|KofYCi_3RcXxHoAcrLO3m4FQ?$i$l zY!S)yL`eF{?$-UQ18cy~Lg$L|B!a?aa3&Z!Q`96OnnacB+qCqccv>akjJ@wbE=YX<{lYb-;C3wVJe=TS9zo5pRU8V2Xz@E|Voq)e20SXdGosp?(y$RlTZqy$@l! zyR%B>kF8@im;Xb`6UxV5pa#~-|6b7Y|8%=yzrU6L&yjv7`Tw_^S+;86N!5U0c!nc0 z{!&IgFfR_kvReN7Sy0*VSB$k)M>+oH;nI^#G+&%u+~-#OKE&%EKCO}e38qZkPLY32 zIM5pT-`{Wh|GGiH*WJqh=SX!C?;69YGYj29p1W_V*>@X~{NG#-*CgDUF4jCQR(7Ts z=OGL>;z30aIsgh8##+dno>OBR|6C*R6O#B#hFZ2z^sdGQCb33GpR}uA(%C#|BvZr9 zF-=_9fdP?ZiW(*Tg#TrM%6YK)B`x%Is>=o@h(-KZ*(tAJ8n>wHRwa-}xJ2NZK^2ym zedU)CE-;^WRuq=;Eij))OU0UuVLo4KW>AZ7S*o)LeyP(lKmV_dB_B^U%7KjzI0z2H z1_VGVg%+`35x5segL+pal7w;o#%}oVBS(K2v@( z%B0!v8up9Bv$w|=m#2sCj?V{&N5>mtZ$+fskN)`Z-T6H@KYOkB0RLKg1I4mmD)?m$ zwLY9&9=$*P?~}Kiw|=*(^+d$#l2Lx;Ms69vxg5ltqB~q>~ z^I*G6r6uKK*9ePn?%f4*|E@c~r&bYVt1f3$(IG=OkH%REJSAq2aLEKoE{B8}AuBhj z$BDii6OGs8w%(47?O9o&(0z|@saj#10A~v2Mt}<+*=2E*q10BWfnVqIV8A5d zW4s@-j1{dczgivpzHY;KcHr~Pw4Zjxlp{qTT$C3 zH;kc5(&gLcCHKn0t#WZRSaKkKiP!9jCN%$aWW}&dfdO&0a`_LIHk1EFyy6$Cfi?0! z*z0#&=RaY0Z~Ohnv!v$ve=8VtbtFJrO_T?}+q3>B)Hly{y1jD!`-V<*%!S0KXk*>_ z-|g-#egECve*f_-X$J;GYn0r6s-4d{Tu+e0FrzG?d<-dxugDlx^#!=fQftkCnjm8^ zW@4BhTT(uL1rnLPSi$ZRYwrn9oE_jew)=bdA(c3yH<;wEfj|A|!+XwdL2z5sbO}W=-*yYKkJPqJfw%NLVz$NZF!YQ{SY zL}t&LrS{tw<-GLOb?UtIhvdq6>Fa6gy!>Bh2R;%>MW*27&9QR)REl3Q){ajTB%K^i zieDXnrecv`=R2};R<3`_6TEr0`(MKT{?h#qy}j-J?^#kyu_M=#=B}nyoKeQtHtH1W zt#xmVd0X625P{nbx`7vTy>4(3?nm8T6zuy!(0ko|y>}4)B?zM+$PYYzkLzYE-i@xD zv5<6(N=Fcm@F02J>-O*<814<@(QY_8IOz4eqc|S+Uz6ZSYlV=+D+ElJOd$&#gl;3XyHq3-J zV-(ijb7NBTuA4@+_I)=C@cho3#q&MaZ~!#U+8=rUjj57JwnVXdBcA0Dc zVQyr3R8em|NM&qo0PMX7ToXzAIF4OoZ>VP*!GcIaLa4DIO+}>$h+?_8$!?OB&2HS? z08tc0?AUwndiLJ+?ES=k-r0Na?d<$N+fpDwK+k)>|DXHb$2aWGJoC)VGf$rx13}SR z5~0L|0U;rpAU#8L5J^i+kREgTPLEV7mHK*nv;QxZN^}0d)K})`B$NAk$$jK+Fqr?8NxUUe5sXl3 z5;f55t7x8DKon0cKusbLjZgr>VW5X_NQ+QHsn|{(ff_SXG(u8P(XDX#eo}&%OSWAk2tR-;)94z5czteZ1v4*1wO; zTlRDP|A+_GLs~>30)RxcC`FSdh547nvLYlb0ZU+lEodMplqQ&>kQ#;U5H$caq-FRC zOVa=tj2IS8V5r)p050L0NP>&nM6t{T!Y~t{42T-lm;i(Wh$JBsAT)r* zFxrS=v51;PXi6e-1^iE-H&QfEAq?j+vH&=C1OpTYv<_j{E@gn!2vdnfG!{;iXl6kY zl<~z!fr*EEr&bNJx(`%$Kj-mMg~Lq}CAxg#c6oV8nnCCOv}F0yFnC z8eo>Ghs1Yy%d^AzY16 z456BZDlaT0Kr90-0Zwp3BLYB=;!1NB1#l6|IBfMO{tZB(6uUqT2#FFfP$6_Og5b>f zQi$5f4jYLWAe00o(#MEUG{rFk2ql$V9R={we+>j>!qv6_m^Wqsaw!Kv4dK8*B1tS& zA|$gMff{0FkFfU8Y#=t?o6?|3_JGL-75EOGhdYnpg2myDiPoc zT0t`_Y&_-`Rggm;Ml_HSqd{n7EPx3{Q)7my{!Qlu>du6=vec z2sMPU@F5sRIdKCpmcIgsqEIc4z)UBYnG@O}RJ{>3OQQl*jAzJ@!8}e1PNQTXvtMnX zS^6dbvmPRADzR#W878hJGx}oY9f!R|N&twaA+?Tc34zo)8}7<2A|pi;dZmGYm5{Y~ zaY;l>kb>3oAdf`Eq~d<7c#7tjF2&Aah*=ae&5%YMKx(de6`~I>!<08M5`Y;PGnE?%n6kj+xowdhJ+}ZG^%MMi2#ydP~mtLV{jQDiegP67L;va zC0kg@!BpBVzCM`Pwy>Bj%uPRrQZ$1RzdUFiM6>Kzg#Z-SAtcJsuZAS_HmgMs;iv|o zXdV;3oV_LEf#69fN#HEg=H?C6Ba9}%@F>ctW|*YLhygvSUG@>_P5RGtZQfM%6izZ+YYF-2&m|X--IWl9E zcUZ+SB@IF9A!c=$75;ww?DrQNU=>Htj8G<-#Xu7V0wc60kcgPrwsNg?26m41tQ2Y~ z6A@-{+iK~Irp92Bd*zt*VNP4MqA-vIktnx>nL#BCMz-?=4v-{-G?{rNgNao&B`2Nq zkijyCf^ot$$aIEtcd(5ZSmTCOC}@aA80w`M`-ITsQp6~}+m_l&gJ2pFBuiB+8JS!h zNR?dd5x6l_OCe+uszxYJ1cwa-iqlk6z9nXD3-rjTrpz0DEwGgUq*fC~oaP9Hm5%_T zK>OIJNY8|zh>n70&5=K=PpC{5WkBEoJg?>klLxKjK`SkwbFYe=XymUQd#!K_!9t-1 z0_Mh%=`v8EIL!2;fZc?1BIvM_>`?Qq*zK|H#6cvp9!Wilvr2)6Ak_@*U?u=*n2ypq zloGfj-(hocrIv)$NKVb$W_6nt0#;Gu%~jr7w6BC%u@j{LN&%Ym6ZE(K67!Oms7VAy zn6-;hP1v@zBw;jw@KA0ICNly+46)pt8BxxDqktQu{-PvD%Z&Wwq$WfGoWR8-LC}Jt z+>~E+C`KqX6boGeTx3!?KbFZlgx0a6B#2(Ka2EJB10Q892{Z}C5G?`&JtrtQsFwG( ziiBCYOonK+4xxDd#ZR4}%khF5& z)f@G|npzDcLl~HeQ~PppcHU688bmK2zTYfZ9^t}6wjBvC9|mcWS$$#*ON?FDjP6V5 zt!yM-XBWIke0hb9@|CPZaF7h4wBSO?EoV5s2qPH6Df+DT%>2dZ&P-K{wk6{`?hrb| zRau_Q$S#4gb2=Fhnf=27SQMubG6}-$x)>muVO=zpqdh@40wFP>wgxL}`a-aYYfONx z0CFmqddjN2gx6whuOt7{k-tAZ%TWS?wy; zw}-$=2BZSODhZMTF>@zT6Z%~CG{ck)2F%2nj-0&D*&=I`QM^HCV`Q5(ft-eR3TIsR z^{_I!c>yNat3qqE$$;eOAFIjnItepl7BfgteJ%av%}m|`V&EtRIQH%20<_D!hVpEf zFOyfx)`p0AS_Q4kT6MANt1V(y9FglMj8nGeNfV2T(vLBm`b+w8WWZ2C|0V^;L+HCB zM1+3lfr*$YV;>z6qi$Fz5QPDZsEk@Ig9<;=7GD=lA()0=J)ul7h0TN}A*70+5Do&s z2rXqX)UC8U_OlWr*hcL7Wy~#VjJTR}m!Y(YGnRR|1uUwajldO@P77TKZ)L-h7!;g#LG>IZ+{Z3#*I+UVWg@eIOWz=X8YquT$RmoRo6;)|2&=^Z~JYXw8 zoUy?$9-1I6O)@jc5Ex@kV1!o5R_8#w*z#JI?{i|FQ@Q;&#YM^msel`lOffaIY6GxR`sgD9?~-6mTfD4-hk`Ap04|IW z6LasxoGs7UMG&pCKx05bDk2FH5a7XUPz<3=6piT3Oh{1g+<0@rLCZ=5MC%m5#gl2@ z6GlnCU1lgqM9U1D?G%K?1`8$&Lovyek}IYFh*7L&W*Nf7G7FIKpa@Q%VIUY;Il?22 zLV5|Lhx!sYluWVN0KDEK2nri7f!T&mG^!9K3h7y!(qcfy%7MzLPUKvhMvCK7lnyl* z7;3Uwg(zkc=*N}$p`_$Wu4aCLbqVSSC`7AAaGH+-b_wbz4&!VuW_)b{T9YH|uO|r{ z^d?lipCq@rX&@wF6AQ%;dG1pNT0SGQ68rg?KF%uUQ7VG8X%hl%xkoN;4 zW`@lW9w&qC0OAWPfQwuTq*CC{{EsDH$Vd}fw9`arro~Z)}bdKq0&6*%1tz%>lAH#Grhd3Ekfpv&;Lcjp&l{U!` zmb?!X*AUibfG6us_wr(j9s zVSZW#<%4sNu>l6AEW@sJDu`7H%t*5n%lR<*lrg@sj(kvtqrPMvs@9q35l0xor9-Ty znx#1u=i-8Pya|>_u4QVuNwIRCT0T)Owpj2T<0C^1^H3qI=RcW5X&S+qiHC5Tpbs}N zj1HSSq{Sgb5;US_tTpo>Fvze3&o|HZN{Bae)LlTD&ao_Wz_)f9X;7`Tc4}=LvV)p) zo~^(eP_7F(DA0u!!Zbv)PJac2VSKX;k>^@E0&@|-va3aDBr%SZ6`r0NC=n5}pTv*> zmGHhT36-Rl*oM7DqzY0i`1BS;jcOR>R&bui1P6s!916k;wDFB_zZ8IBG9rL7XeyQ5z75wJG;?zzm!?J{L?F=NvFN79(MchHF@9NFg-mnqomQ zMn41013(-c)Y^~ENlsQlHDVY?pu7<&;JP^s$A**G0IVyBDL`7Bg`Pb*7-9nf zi*W)*JY9tto*2SmF@)2o*c|AygzNkDXAp`bFofjYF5eR;MrzP(`?R?QPm$#}r2yS~ zG6+NxFd{^N3xxo~6~$U33L~CO@5K~?aY<;NBIfT{_D_OjJN#d#c;@-!&i@rim-38^ ziVKUC&?)pcw(+(6e=lFJ9QnV#azCG+`M*Eni6cM~LULhkh|ZnZ4s?(?x5tCAvlNWb zVpfEj#(-FCj$yUz*AGZy5DbARL=q{y1^xR2EkbjKoEQTxkij6aT~7=Irn1@Z=8_8u z3F|2U641ZD%m4Or`ak#Nwf>bl1T!EcC7})9v=!hh>)%VBYyJCq`TflQ{~=GmexB|i z3DqlDjYorG2yHSTE%nT>sC9?}xO?{RFJelG!cq*Zos~nEQnUNTM0`0h?^}1gip7?4 zY^JA#+i=FP3?PZ;HlHvhql8SNR7Ul})xcNI{zUb$MvVqd0WM+-Fa(F$-yEhPoYDrF z+u)Ka`WPXM(I#O=muLGR5p_X0=xlkKX@ltx1yqO{V$4o1=w@-GvOU+J2*#&xuqvC7 zn1fQljkkIEr~;!HQU*lbR3Zx30Fq;m91z~hrMLm))k>PPCWZ79hyoCcJdbTY23b%+ zEn|af%;PG2G>@%N#$kbc4L(L$AT?w=qG*}_`L)8+JSH3z8+l^E)#h!15j2n}PHTWm zBTC$ea>)UWYqua2=jeA1O3az!l<)bpHH#vxX4X8yZYZwCjPO^;LDJMb{I)RJ*#LQ= zZ$+UD2jKUtG1Ms}Ntqb|r&Z^xXKZ;Fhx1}hHr#K1XCg>tLvDi6JmVq(^87hG%XPAd>~q_v)V~*4%AR0F@OwcTaqvu*dY^hAru?uk=Uv@U{eA3 z*W7%vKe^m3tDkBD;~f|5OD!8i60?DG_J@H%iT%ax31ok=cslL?8^X*N!hEpP3fF=m zKobB%XfqyaLD=dL2*wb~oZ*r`Y=jJIyQAkSq^nxxlG^Vml0qXm#pc`D?J%M?k|=Fr zlfF@#94^*s<_nrzvBkznA&lf_WZCx-W%=a)-?Sb4EBxQrTb5)0x0h7*^ZbV&^ZYpe z&$Q%NPqnA!^1S({q<}DtV*<=CB3a0E# zG(;P@*FsA+PWK-rY-mE@?*z$Is@kP0uY z;#UqkDRd$i?wr6tAUMaid^R#^-mjETL22HNWJg@G4dqlA_z*EfA^^fD!fdBSn~y^I zAJivk0))hL4mIyovDw4Ih2`@r$yosTL0iDkde*p^EqT1pZMfmnW@Su8^5Q`ce`IYlEl&84-eF$mS? zz|GFMaA;bvv%<+5B})_*CUAts5H{hE9#sPaNhG1nvO{rg;ZXhB2=bf&xB>#lO#D`N z3hm41Gnw=FTrD~(w_k|cgoVQZCICrb7{@%=voF4RxO1tt4hJ_M%Yy9=k~0Sq#jX3n z@<0@4$efKwbG-&k#r&h}iKHNE#|kjNJUKm?+fa#e@$;NXCRPUve7o}PVEF9wToy+* z$Zwf5o*k+QoF)m3vfbN~4~!v$!HzaB?DeDy$ZK*;y9H|@Ws+k%wh51Ruu}2;&nYZO z0%w4H14=o^REyA7yVdNd9PAEpZns@NIiqFon|+DK`wgQ*gDDB<%&l-8WH2B&#jzK* z6P|pxa<$3rskKYwmJR>he*b(Al(iCKn?XXElda7Cvd!G*Hxd?TE%2^l2i(tkS@Jkp z?{UqQm@l@sk2M!M4z>K;STd~%jeQ+OE?vuP*8cz0n*P7{6x9?}2nociB+^5p-5*Wnuv1N|ER_wkeE%>VKE>HqyP51)8}^fAi- z>!T*EPnygr5G-t8C}N-6=fzX4vJ?IvXY$;zG1w6akRHo18~o9oAQhlEjNmlz;u)gd z((Gw~`7CRd+LXTv!s!wDEN;EWKc7{E-KhC)!D<6xKI^c1r}GcrYO8U6M~UAJmaDk< ze@uqu$^S|CyPO04)%&inozjoQt8$PWUXc(WQN zVT^2#{I{=4<|t!UCH<{oS~X$2IKL+u6^4-PcbX$}hyCS0=k98*&I${Q{hiCP^bM{k zmQXCmR`YMfmep^anQk>g!qFm33BE(@D~WK{2OwzuN_1z3bXeeXGw77@IZU$hmjk z(u|#L^;S#5Va1=e2mj=y|2vj?1#9Cg`oC9h{ol*W=coPuL!Pfm1L2~#5NjF;i3!c+ z4tBB0G4)K6x$t&`T?jv2#f3LSY|oV`pwAd^XIVL}mcx1M%~eXYxuyj;zJs7(vse(X zG>?h-bRza>`@E#Hs1IXMs>>whTcNY64S$xR7Y}4b5s{(bmOmU5ABn%Y6In{nteM^Z9O1KKMVM zEL?ao@U{4_uUwWh{^RTY^Zc(L^4P|Ia~up}KNB>km=KZxf986AyRiA;IgYv4zl0eo zRErZN@}1iF+WPmA<<$RtWYVAhpC9tDr#R3E$=#B`O*^-hOl3qd_O>(wq)vpi2z&8% zoDO9#e`S>zKI?~(URlBp0HGUxiT8{`E?TBv!0ydB~1fP``$MUWp+pNmLEb}|uM z1QD+zh(s3!P?NBj6}A{=w#N8N1asnm3tPp6y%Y9*)@+0wrRWZh%XH-SQ37DVj3k6P z8U-$bjpAA(29XX5ab}H69dA)m12g#eAh`d$zA{xLSRNcZneWw0mAt^>;+4d zyX1&{!CfkD;7%v!?=EI<-mtp=lD#CB+sX{si!!6xdsHO0AtCF_HLTj^tzI!3)XxVF zY&o_`n_JGTFU__hfIRTAqM_W^YFi=5D{ErQHkPlU82Qy3OY@DHqw7np@QXlkak$k5 z?E-epWNQ&JH+m8L-GY`Ihs>LrSz&7DDu6tY)uXu5`hIu;tsonU;`giDz!A8CM7T;L z4%H(R4e1T6!^1Q!o6Vg7aOaM&2WSJ~#6i~fU?cn}%tt@iV1xm|gQCOZ z<*{lVqBrL%ap-Y3ABlu$K6mG$!Ya*xS@BkXS{S0$%iIeS@OL)SC}Ndw!*UO+T!dW2 z#61XZ)LR#0nsCf=IL>Hh(}e?{GdgF%xay%4%WAM+%X(c}Rpb_>0*GY*xHA8? z1TaboR|wh^{FhrG5z2(Cg-e6;+j9~zKqy(zS`?D=qE#hL@Ryh7dz$z=^KGtgaxgTx zD3y7vMWlVY&9e{51es`+1ZG?QD{~>U`M^hA^NqNpWd?vbmz2GBRk#8tcd5Mk#XRio zE`_Y0GaUTQT@u7JOz0sr7YF!iUxey8^B?nt7sxlA)uH zAtXv8@K>oBB1y<(=PODh`oc^#f1@fhvU(y3F<r9fj3~|LWqoZO{M!WVC{?AP zy)wWmo9kT*Tt5fpD`~<&V1(Ah-icQjXb7H!k_3BYjnbSJUnoeNH=6~-9h_5ODE72Y z;SmEDf`kzgfmy0Bxi~6ifJg|#5G;2A21y~AQ>n%viYnwJ68!yQg=5b4Y&Od&f!_sO zp}Un65Zg`u+?H=el14~9ibFJ_)RK@Iai9u*6_nNTQ`+t`DF_e)iV;c;#rbj^*0!&m z3Dqqp0u`L&TT++*JEq2+Cj~1l=VlaS7#-!%<+~?&sdD_S1WL8RXkXj^YUh9u08oG? zzq<8+o9R<~z|GuxNP58iWFGSVO#_=YY?5cYyutB5GqzBxrX`mT_W=#Y6 zSKY&+Fm3+#vth|9wbyci7Z$TvS+S)|hETfD+8L`j zBN)P}&EEux73LfW*8Q9W6pU@*FgfUki?RihL_H9w!*5zDd~^u=y4^)>+~9suY-# z9No%a(9Y#~$9{ozhFGVHecUfwE|CVUbo#BVqo&T z(*E1w6>`k;xRw6fK^2k)^K;+*cYsNK`!os{-+!|zl-k3+shcOK0`;Jp<#^S!pLhQp zdJw<2#9o_)ur5?at&-n7QfOfzoGQ=HQo-O@?RxnxUCQff|H5`K$F2R}0>*AvzYZby z9sC7WRBbNAFSwfP;DURHfWEgo)$Wb_vIPvH(1~Wnyh*9u4N1~j?axI0gvojm` z|87-Z%u(i|q0&o7_*P7$6bKK;RUr!Vo@pmrwW%>1a3M zlNlokqg92&RxMg5K<3>j4?nh3Me*j!-gcrq>L~Sryrxmk;x=^@qGX7i9;h=oKjAC;>w{TbC*E$qpBuNCP z0cFC~ywqnV*FvV|U_h=ev4tf<=lK+ObHo1!bDNn-;{W{4Da|=pAoy4Fn61wPbTC5p zM~Q=Lk4ad6H}lA^bkMPh#9!SR%MwBiggZ=RwdV=IjTi>0Fhl`p(um~v=)gtr73Yez zuASOlCJyM?15TCofP28HUfrc)Z{|-gpYBqzPtT@3BtN#0Ctt{mc`NTO6?^r}sp+dR zJCUb6ELZyz&&{K?^!WNj0*m!taJ%greAeIWi2p9mX-clt5TqJmqdiKBHWk9RM9G!O z5LUQ&5iRijZ!|EwOT_^_n>7`?Nm*h{m35cOyn0&x?k@A{$(CX0Pu8<(_c*$z8}o~M z$NcdI)-Q2o&>)W$ws$2JFVtZh$px85M0dNIzIs}l2 z7D+L1n`#m68bd}{ikU$HT!<04#k2_la=APJ#Bw*W0w0NQQ$Xt=nq`ux21O_Uu>;wc zz!3mZ0Kx&G(NG9&*}Y2>pmSV^T};AKAiaU%MY4bZU$M+fEDMN}`6}cA3LihQ)L$W$ z+N~Bz&wWrMU00SLD zvNK@5=axujG%27rQZ##=Dx$_95`h7Vv(wFn1wbNV5|mK}6!W*lwx4VOFuV|(acqtW zvtQToR-6bndTxEzSwSJhaDnr00THXy#}{xnP|oajX3k*5Kq6|0MsgC^zaga^IL_^o zUT)wF`XLT;^Z)no#jI&8)Qm2!R-Oua|~ZD zUD^KVePw@?t&lKq7n6UdPz19eVAX#Yr!r`jIu-Qef_)T5e{*%QCPB$o8cFAE%o* z+E$Sd`DY(mbgtvL^Iuu8rN5I=mz2ZP!8en~FeT;oOnen>?#XlSOMpb~aK}r4#5<2~ zK6!leb@)vtb^J-@_>-5NT{jR3=@B~5M3jG%m6CRp!F-UWJq@ogyBzYOF2R>+=Mx)^ z&G{kE6P$6o_NOAYN zTb)lG65ZKMTtEj=K#yV=N-;Jat;=oC3Q&2m0T@C?8LlG%tc4G$X(NPjmsn5$A`$M= z8qNSz6R=~5*7%VK8-)T$ih)EZ20Tb6j2O&Vlscmp0T2!Y9YK<8pa{b4+0YzRoS+%L zXaYI}GiZz$gNDUN4G@wfNT5e33eqA}bI!Uaa31U2t3v=w;FyWUly_S2#;g`iLU3Nc zEci_6Xxcz2JUwlri7=sNud-Dm23ok&mQQ5jPw^A03BAEcBVsd?5Swj5Pggwx;V7;Z z6B@Cj^;Eb75CLt$FM@Ehp3^zNdIFefP$8f%LK4gxQ6Ye#_1xNs0F%~1oWDR#kX%fV zJ<5&eTK0#Q;|brix)TT^P(!#tcS5)bq5uR87=+@2_QQ07eajvJCuSX@4xtKlj1U|tppf1Ls{nZc{0M~&t8=+6`S#S=-0WwsPJzQF4kj!wHBAf>5qqmvYCAXM5J;^91g%5J+zw=i@vK7n)5Sn#GC&kXV)kfT41yD) zgJNT2I>y@liW7O)j*QB?K7S6e%I?RIn6RL@u+U%!E$3M`GAhs7TwRi|OVdKjF;N}E z@@(5-eW7)S5cm`6?0yT6Y!?<29_OItylO^9tg_VcbC8D+nIUT5YO zTxH>s-L0QIbrrPlci4(vc!j*c%6M+$kD>_@(jsiAv*0O^Mpblv^t69_|4;7yUnmYE zDc`>7%j)^c{V(29dCvP^y#2i8KlgwAh$qLzPFA~+b5RDjCn-k-!0q5NzmiG3C0u3? zFYy$>#Y^fX6-&LuUeY+3pTf&a;VqZ=`gwZ?$bIDg&7?Agl#f{3+ge1KPe$h zv+^nH4M==WHU&r{j$76oid(rIxn@u;LeUB!ll%L~Ah}wN$o&wNzs4U@!*V|#janv! z<*>iXTdkJM{Cw0>4FV$p-rh1;6#xbJdBMJrm&(_h>!yy-BMP&dLW|Nmqe`MC^q%Y{ z1(@)(w~vjrDUt~iw(JJuOYwi&lobOtMi^l+&SvHc6}W^c{x8;OWq-4|ZPtl^Vu0u& zl)H)o=X@cpdE6vsw%i4#7E`ht?`k{eGqR% z1xcmeGQ?jB`T0s=Umsr|xu4w2=O+pNOC%__PlEFNlAu>UNl@nFFY^!hFG&!VN&RF} zwVzre3($BYUcSDFpP$rQ;{(f(0DoAmQTxhN2qF)V1^9S*$-R8lav!yqmm2bey)<4b z&Ht}S&|EE1Ya$4LFS%E1E%Soun@y!eRG&g<`OSy_CEqN9{hhTcA$1PE+_Ld3nTkKvi zweK2LzY`sXJwZD6p6g-idG+0?_K`6kH*a3Dr`EVo?_wRD$|2oOe|ftK?UJN6o~-}ib~x7^dRgC5Pf+;z$B+LyPjx|8*JVhfL| zfs{UDg6p)3#ik&4=I+0W6O|5J4($d{`E_;*KB(fH?%>R<1bpDpnlX+4o-}j8xxdqP z4RJd<`p(6HS&OTYqTrHFfp^ub8*RBj8@zju){c0=-z0;?QSF^iy%F0$ut`=0IW8*5- z7jbx2r3!<*GJ%k=uY+MO~l z8nvF%ozeYl*`z($8Qy-*hw3znigcRgKQgG@T6tvEQ?SR)uHmo38+Fg>Q2AWd<@Hw_ zdF|TBbxy*E8zqA0{^Iw!?$~ML&PsOet5<7SXqTgsM;iAEN!_-(K*{qSl?dAI|7OwW z@t!?K{Nr&aXxy@Q(e061RrY0Ge<@P#Lez7%?oMxabi}j)aNlE3N-NsFzSn$8K0a$!;?+_H7w>&X{eDH4_hc)#@|2PF(NiYYz53Rd-)SFxfT2X`HCm zl+JBVJ*ZH3^RjWtPRClUDn6r+IIf%;_w=9IPPu>KtRqWIhcZf@NISl5-JugPjp_Rj zGp~bPWik?u{8q;4+9ap9B`X9UXb8_Zxou;mKBHe1cNx0yx5jf?wmz|KOxz67OcJfT zqfhPaYvvQK8{HSx8#_-9jkqy;&)Bv@=VcBz9oRT?Z?BmlM4gUZ8m^=UH5fYoWcAOR zj;$J3T63twey5IyBn<~;Z5+8(hCCZD>9pEss&9~Hcl3uktBT!!d^UB3#@YXoF7tBF z3K+gT;Gc;NHl*v6*<-Hkt1pWXo!Af>ieGeA$C3kU|8cWl*K_r5RGac5<#ciUWoDvB z$fW&ON1K{fKU}s_%wH*W1~%(DIR0W}y;sNDQ1=g~?t6J)-;0m(;njbMXm_!`a#Ciw z8uHbNud6RUU88lU#}a<1}d~RP|Sn z%&hR)LvOW6+P(gO4LPyn!LUR94`nt8nc3Q> z?65e0TDre+;O8my%pp7C{yK3wwC1Rq?<;SSYr^+s29yrC)$RJYjvL$8E;r7rY2(XX z$ByvqLIyQ(TG=k(p==h^c&uS$)XU;AvNyj}Y2@rW+}~^U>>e|8hiMUF=c7OK9A(~U*0upq39&5Gxbwx&k^zW>v z`mlWFLG|J7zr}1E-f14<-sxv?tj^EFWtPf z+nriF1Cl)Y3|g@Cc)P8j`fM3?z4FB>!OhP+?9un*H2%^pnKs<$32X^xVGW4hc!3O7S%7IkJm2jqN%a@ zTKu%tZC*SLJ-BpdxoN!;+cpUdIJ>QAbkc&Cw@-C9ICrdxm{8NxfSAyPU*{F6mc6xv zo02HLf6$ikyK1F2TeP#!;}^gD?rdzQOz(E~V&lhG%hV|nc=pug2wnO-)BH0t?i@Oj z*vu{a^wj!3lO7(-?69@hKIbt#yASYR<@_q6d%ce%BGTrY(mOX3D_&3PUbb!t*O4{O zEE-bxI=yOG$F!A2nk~B1H*3P=F%>ts6(C>EWpmeUi@v?C(a3|+ zHrHA%PhGZUy^~9It-h$igDJ?-eF^s#HS8{)kf7f==xMohH^2DB^Vd}V6Kt0hp|jrK zJUe{JhkHx!rS82<#CP3vRpxeawfwI<%k1vmxl2a- z;dLvd1r%F+=txc_SnGX*S@3WjlM@g!$ zj;&tF{caRr-j+*=5+#lQ zy8HQ?9jgZ{jYvD3T(9sEITx*@*)F=m!(*|>wt#vMpEG?-9agJtb{J|k<^dBypT-RhH>aqZhQX+M49 z1!V0>+59f0s<(@qR_);G2N~_lc3!(;tG|fM(uor@uQYYpy?DyUze%DKdC&3_<&!4!|Dl9wcV_faPFFRs4O6t8Ub~`Q>jQ|-O`0O_x(Qj^oh93 zlilxro~>Fu?>KxJIWrR9HYO1FE@#FdnDdO9C!W@^yX6L^|UWF zDyOVb{kl@#zj9Hx*r1vha3iP^kb(To3xvY`Z z3k;j9$+-B-koIuZ`$@;o&#o`=oS%HGYF4>tyF9P%J72xyHe+P0-=GS|*Z&&ZutD*a zJq~3l#0~r2z7tY$^=d`;5%ijq##e!}UNpRQ@imsTbk!ceKeDgv`twuT?;k(?o%AK} z0A>0+ttd5q%#oq9hG$#~JCxRLyvOOxvDd@1AG+^}*}EdDR=~6lf1;uH!ip+aKe~16 zk+#nk_qMf<^?kah%*b;0KR@=Jt3F!s;FPh~PEVU$xy7Z0b6r{{D$fr%#Z+%j7_Z^R*k1g6X8T#Q=*#S2yC?;?DtK{3md#1H5_gTMXWPFLBW%sJ? zt-5>NpvTu#rG`kKA8Gz^0OAy(pZMZ_AI}tdktYWh7Q1<+->uU3%NrkEzQ68Tog?L> z!B^Egii|J2@b?9yr%j^8t)^@gA6qG2Tlq!R*q{$(jqvOJcW-syzhTjc!Dmi2slIyj zmMv|hXM-=6p}+Lrp5`;Vk#6;sONPtauXUK!b>&Rivjh4Tb0XTl>aT}8hyts4wSW1p zWYYbXlB3r$!WR~~Fkg8qdXm$Ce_Y8XeU@oWT}IZKIJ$3}JIxahFV6CPx+QRW@!e|Jzv+ebk}8FQeWf!52}`?ZMfLabCPk+)`f#Ss;=2|Ozsjqb6zhucjULfAQ&Mn=Mb0YYa`7o)4(JW%i$YN<>XK@!{5%{i~mh zZt^tduy5n3aWZAl!#3H=F3cPhqMf+8d)F;ky+$1ub$CAYl*h~u3rgbqu8bcxWgEQy zxz?EZE>rXQhNt$eW(CiU^*L-nl+~ueXKIQIQ(iQ8mqH~vx8A*S2J<|2a zgA3Ra%^R`L;L#V3#g04K|9$YLtHJc+=>uvtT6KF(Z*2LSlnoo-+?4#$b>iTuQ*Sp; z4f%E7(6_tq?eL_E;cK&pef)TG>f`Jid%yhoC2icjUms_0evp0U#qmFjZLL_bb)!#{ zqT1jW9wa6Q>z){jJL%g#9&^BN+oca9@ZrU-UY=0t@xZu^L*{S3FnRN`lx;!IgwX|d zX*hOp-StZ^o3_PV@SJ`6*O4*sRO7bj8;AFEy^7tLr&UHA7Li(73)Kc{8C z-_Yn|8u1QmmR`stXTua}qIxlu! ztJsG-<>~DX_bfW0;h>}b`h>R^ih4}!SaITx_u_?FtK zEgSdP_4~RzhfCc&d~?RBg^Kr6yHdz>=c%iE{L@b1u??7dH4e#q{Xvi5v-#TVXTgl4(!YM+1)<+pt(KkArgAGGX<2Ngfe ztlaZ&%}7`0jPPTacfalnANB9_wu2(!*WtILSB!44xBH*YPFqHSfob=BD@|P5>c}|* zQU7$j^j(K%3lA^;XF{#W!#kRH2wIk>pAGp8M?@ zOP8y<@VAHU4gW0Ze&WlAtOLIc8L7V9bzk!EU*C+Rt{9Bxl`qz9ji|MHb(>Wur$jtB z6|!*r#EZsbtA07(^SJT$)>+r|zdL^p{P6rs;MJWKuGO8B@vP#tUGBRF@0j)QZROoY zYLo8y?frirzOZyn*1n2O)~>MiEpUD0u{Y0}6**T? zvidjW$DM7~p1*VGC3LWdonYoUYN_rOq=KPfS zzHfUjesxHHeeIX^*DtNO@}}9ZcZM|Gl7PM}J#|a+sm_}&G=1^rpHZ6!UVQ2P$BC)y zQ_tTmk-VkHrI*sbhLo9HVQHt$^;-8^ydreofKsQ7HCOid_%d3R3|$^Ns#l4>k0s*x zx~-k(x2#T%4AHGAwGQueqL^3Z4$c>%zhwUI+4g3xg zZ?dody*EBP@W|QA*;{C-;Z5Woui&j4%V0Y$JbSwR^WNlHo9?Am3~#g~1|1NkzwDh| zacHSkpOk~2yw5IWx_`0b@l88!oGZU$$KjM&qcYy&8V`jwHgd$4R$0aGOglPhbICt4 zo0RETb^E;2ZR8D>MqSx-Q~ofO~iBwXk5z}vBDi9MIR4^6!(xmNAd@muv$#~m#? zY0-?~RbV=M`R!fvl2e`MBrG8^>u+AAC9mn9eRA)s8%f#OgIiqsl$M?O zKKsu$$ioLttB!3iGxt@XLe|T)=Ejy8&TE_R{Aba?dbj8LHUd|M_VX-Lq`T<1&0<4csD==CqY$n`+{*q+`qL#O*!`n~tv*4bm8p4f5k^&mX!a^lN9F?3q( zWyN-FUfn0;jb>)7LHO@^d_aljes`)p?s>7}+_JTlr#&xr>3`m|HFWU)y;m=^lXqRV zKtExx^?zxjd!4(aURjhb*>t@A+?eMFuOGa=qG7c<6-pi-_^iq7>jvpxjjLW3 z{`)ksVO4tj^?I>oR);tIYs|%~4Mz1U@wnODN*;|Leb#*GKJ}kx-P?WK;`Lzejv>`e zuYO(sWfS?O^So!%od31lnPnU}9Ue={%>)NJnV+VIzT*YsATJp-f z&a<1|8meqpZe+{-i(;d$PI>`}CIp-c>okLUJ+e}g3_9t5c+=!#CF}+vmC6?Rr2qb=Al2n`<@Q!u@yh?cc2~-K#e@yP}g*cJ^KQ(_u~v z{_9}a|ASBd;=l5LG5+iABlFE2|MmVE|NSA)x5R%-rB)03$88=7N|zqKvBCU|aXW64 z;h?R&VUePX#%28S{I4CecXVIBrQhSHjRLBa?x|?^40K-9q11Ru#Eju?yZo9KO)nk% zW%ukWyDlX*D0AeW{<{NL1nzpAwbLYjb@xO+;_6QA;)JXNt>eSHHyGT%gET89oR7V%8?hfcXb&SrrS4a(6|djS`QdI`i^u-*CWlg z+_}0fdwi)4Rc3?zRlFCMsMWiKW|QJe+zRjIl}c1dC>GK+;I(`BBIj{#F=eIFO1D5+ zsi>55=Dv>OCbwTq>nfI8H2MzMU-VF$PQ{U;#TPrxn;lWHmWUkJc^X>wTv7Sqgao}| z?TG7t`0hH^x_kP^UJcvapx2ffj|`y^g!nJZZiC_`YUC zo=K2O_9y~t+9a!yfuSGFp=h1`u&K%)6 z>{TG$Q14ZFQh3=<{)b9_uIAKQbarC&Fm;_Jhr9l^zg)>9>&Fx=I_b}G>E+tE_-|}b z*GsW4efm7-=i&`@)LjBDED?ztRgWyCE?KlLO%2}H7Rea%w;Gy&Sh)Zvom%MVem;Y5X_}$d$-V?Jn)W2S%@#yr4OZ#b67_?peI<9!#Kcebaj9=LV>*(CBkD^&bm6AK0PnD}s zYxtv@!*3+5s~IwJOrI*HN2R$AxDC_|+%g9%st&F-?@^r%r}ngtykDl2IKIlQdCuof zZeBO;hVpIafb9#tq?LxHwr#h$m2xuCPKyjLm!8ph`0-(WRd1Ek40qkSvby}awAiCj zug}am^nA)Em9%w9aov^+x($Ey;Z(JAMSHACPndSFeei&?WLA2Ks-=e=GR*#U`Llqf zn)LQxMac|#JvJT45n+Mhy8-MM?KKGz`ua+&peLQ82=bL)#X23_t zPVq(8xi_1?;Y_)xAf_1=?z z7l-y7nKG@xm7&ubG~IdX!RgvPA=BipJ)6e;lX19r=-VT)8^`Pk`*JIxJt1H8>d|pk z7g6A3-=VL&Rf*X;xv^VT6Swh(*-t)5js}Z|kBlf&e(dv?vzx@$E@OIIGB~(3$R6vT zIpL3Kp|_5nYS8#a_o0;36+E9u%Td<0VT!+h<>w-xEf4by`<#jk5ATm4B~Pt<>~#ndgd@fAryCr?ypHQY)A2 zq?Lw*Y}dA&x<>M(%>86|{N+bohn6;u=!6#8z4%7oIgQr)HiKbho%Nds?LF;0Bs5KY z-Lx&ZYl+lh`i9dRdj0LME_$MoY}}A)!+xnYrQMfNVS6$)Wdr4qq*7H zxuI{YcZFC`rhJ_fApB7KTb-(fQNJaxe7$_v)`XX7vf?Ky&1yO0Kv&-)Z%&?!+IwF% zxb20SrOv;so^S-NRI(-_J>@&{m7B+dvXe$PtX?PL<$aOcqBkk8&U`*zKkB{m;_kn$ zXB;^9SGDR>5AIUc_B@*%;u?JAc6RB}ZC=HHNvj&=eqvNc#_0~l) zNfa zd2Jc>L8a&n1?Z?e$G5@vhXs0Qz-%l8ENy`@d!^8GBLPXA1sDSXl0YF2I2v$ff&~G3 zI~tXkgcQCkL74z+g@y(yMVt{Z0(%sIh&neb4509i;6Nrpl0XlEjpgxV0fvwlu`R$L z^QBz}nlL)XRr=@{Td9oC0tFg;l669=gF2#ZR`B_i;prF?0gB6h;>6YEZJ`mgG6cmL zfnQx#S(o*5gC?|F7>e`t+l>-Mu@PW`+*t7Wbd@&~WHaj+;Q)d%kT(1!_#PBy}C@nizzAk~GJF8mt$0Y9Bz zp(gA<&2j&`p8WQI(R{+tBz&dgfS20;H5%9bKQu(GiFmvJzlkS1TfRT=`majRXL*Jo zKO97#FdJv48KMu;X;tZskpQ2e)YUjcw<=T-m!fOj1N0WX*mfuNWnX;HD}V8n)AT3KXPj@q6$2u5aHskb9wj(sIX zO2-JU)e=Oe%-BfbSP`=ltr^;8MT4M3b`e*biOja%Vx#-j~*EK`FUv%b;E*L-5TJb76^c3F!hZfvi8Y;8fcP)22MCsadv5O%m|^z3>Pt>!kq>O9;O&Nd5-cwvTIT=70=(&rrd+?c(z#EWuzg2D;P^lvsJ z6CB-|;TxBgM|MKtOA#dl4@C9ix^YK|p{ZW9p5@oFaKOV#hgd*3#CjECl(Ew6Y@})Y zuP7P?l>bGdDViGa68T>p7Ut&v7pe|x6#7>Fe-qDZk^enK|CR#1yww0+hXz2CFc}&} z9|y$i@|g|=0G^82D-J$u)Axb~4yhpm@D!J#3V%S+=m^vdCFYOz8X)owB=NFf|KzsquAk^}oi;i}K{l|0Qke6`es} z#{V0s!`#n*!Zl%U`Tv`EM8iAm?brk4jxxZ{;V9$hz9Bf-zQdQsE4R;&r^>9;|IeAc zG;CHH*NI$9CvrXAD{(r$!*o~7Lb^?`=yVHm072o9;y`GArIFK60n5rx%L-OOa;Pd` zak=AT0jo$Z%>`~j=J{8^I>;5e;P7S6iv=Ae(M8RpxbjstseJiAgTI!0059SHA)z4= z?*2dF>bK|rZ{%^6{|yW-6aRVAVlc`CK^-f`nS($by`5jx+Gar2iXb7YYE9C5;K$QR za3G=-Kaqm#NChA#R7SeGQYk0bSESsx&+B;dTmLw;lC<3dsaL!X__FiAun-UbpGIMC z&;Q=YBbVqftCe+X^D&OucaiJ<-%4mj6KPp1G{~H-SuIeb`y!&RGQz}yaRzgH*|@&0 zku6Fm9oJU`)MPV{;K`h-#aD&XAb|KF_jJ}~5tU3A@iqXtf`B@zR1p_dON@m{{4V0> zrj+_M04k6JSThzH9>Lly&L$jwKd+-vug9bXK|t8vWWywlk*kBcce&54j6?2gEQcv; z!gX=JPBJPAA$jk#K;}OCD}MIusz+y~g@t~5C_k@Y7(#^NybmY7Q94(@+%riwLgGCj zKfZ+6A@oWmU3LI{`v!aYC|z{`efw%XK697}IX$~Hk|eug;ZehR|3Z!&q%vM@y-NL$ zPE2SO*IE}Jm846EijQ@C2LRn0F1*I?YpqHRCMBk%woZ;s(IqCuCP$?vChG(gIwnV@ z#w8}`5?i*7jZS@yc*xh2%Q2EC!8$%V3058AllmQVmm(a>I}5sh@z$0{?Eo@%Cny7P}`w zN0AKj-SU0kY4!lhIqnG;-0&?{b-i?7NK3c~1IuJ}G@~;atiAzw+Ql6R$nSUOhk;j1 zxd;Up!h;mhWG)SxAey#VXbNes9+}`LB>IMpIJqqip{ht0#$k}T@k3_ zELKHeKZQT&gfRqdV?kU@EUQpi8M+rVa0(THLo8S*$I!k0cXoZ}$JR!S255%a~T2TgO7}cJYyir5EO+qr6j?=9K{4N@P)f$ zvUA7^jF~UHhYY`Xf>OpwK_+F`@sn+Wj3;{W1VvbE7C_l7ddTpII$~6!&D0UUR~(l7 zk#&i2DMKsDg`&g@gf4I-ZXjq71Wlz97RYj##fo%mny=^Zw8l}aF}=%$9RP$&swm_Z z02wmf1D)_lAw30Sloe7@NpYPtDF!pNIL@jC^l0R|u8>Z1dfNyF;#%1;Q@U^^N&3^t z5M?ht7bXB!h9^djfU@&e#64tZQ69u3JdmVjLX7fOT^7c*;8h(_UJHt=YaCdwwdCtkR#8}# zTA)+|;LrcJ8NdmaM;%wV75tZ5aE(P<(4M5D4^o)kqRfTb<%5m&3CAnqzLpsPJ_0v-l*+GP{Nonhu~7Bhj{{!0 zOK#zgy&#isU)-W0`>gI2FHm4QDKH+gh|7iU3X-tJsLLMCR&TybKEg!%#K+qzmcj9e zCjBaaJY!B56`CV7b)f+el##($jHr%^1UAb}wHE z4qA#b)nY_v6fv95fR5m?h3x@F;}8YbCMd|VFIxxyHUaxOsw`};+(VTYx_-U^%!>jI zQW*q8Qz)*v!!5E%kURsV7Z!>)sZ)BCz|SSzx}UF7R*b<&5|W+;cqAof9-V>2Shk3h z=o;p|DAnC{HfL0u*8;A{-Es$nx+&hjeN`lV1ivpVD6e7i3k?JX%r~-vEyrhV@6wr< zc>9Is_)gcg{~S~E%nVSNVf5whP~e{AC(3!obuTgp#J9p-SLK&obGtTfbOd9 zc(yu7)3-s326gJ^+pZM*{mr)Q5|sXxn+|~8@rDgrI1p&qpoRD`3RAi>0VKjQ5N!g@)!MuS?-+$$OHQ!oZQcuz9OeXi9QUzN89R zq%01F>YNWaIxEJR3lGehW2>-L(`|aFBQOgR_6l8_q)oDPGKH<3$#=d;wcVWyy;Upp z)aI9*rw9X+Z(^QThgZZg&nLsrtAp~)1@szVvade+Qf{GFmSA$a2Q~-@79^MMY(uwK z4YI@fx`{8il-|y>%_4}!@(g)J=2`n%x+LO+;TD1t)J-ETSd@!yALMb= zA=5Mu=as;C?dr=RSTT#v=b@M@2w(E0$dHSw%ZtjHV-$qmyg56dm7zIk;AI;}FWzA5 zhyt86Rp{{(>RSl``2x=5zb!60=uu~53MZF)y0jM<|9mAODO;XTTznZ&d?nQ@G$6O! zdxfem`RBiuu@Hcv*(u7dCsu#c_6}LIZ`Rx)He58(O04|fVJdM&=09pFk)gxuxsWLo zqY#~C6z;`5mcq-~nG5{?w0Gj$=ih#O`2Wh%1-yC$u>AdhHR`Z1kNA)3xA%YF%=7R0 z|H?_|f5Pe4i72-tV>Bws>s0(i@#4HN0tl!1=lP?As$XaQN zCG;c|eRoBN&*=7E9X|OnQt`rR94>27?f{v8qays24((-`#XVba4u;BE%n{*3@-Kdg zb8CXbsg?Pb!w^BF zf#6^DW0uE*@-af*M@fL|jY-JAJ9y+vI_O)|#7L6Pgm?-fgk+5J>B#EMphDS564R5; zEYtGGho$8uHY&Nwl9Q%ZM)v5A=cv2m-SM2zv=C(&|7U1;T8J{dM}zLFH(MxJEELMW z)ue?eLwmR@Oa{`+xm!L40Phno%%gnwc=<#ErxB#^>hCL9>|gm9;AiU- zNf-&p0t^jgADV(F$t(qEqmhN2^Y$)BgN~`uUNMQy!Yo#v7pWs7Bb4e;r8+WI9ii1k zYQq~TLmF#CLcCT>5K}}DtdKxgT9R^PJSy6})46z<3~V+6FAN<>=O=IQ4uP}llCM4M zmLJZKEyd(uIHkK2TWf(||fjA&2G~MWQ1*Ai}B#iQ)_`g-I{Y1x~aYeX{ z;}X%udtKjGapIBhfIKWsVId?7ZUe6ZBE@X!RLI*v?%wVEoZ-;QKm*1>cLMu2c4;5y z4_-;n6F854v`^ps|2}+4IgO?GIN!}@LA~m^t_9qyL`qbZ#cBS<!Lce|Fnq#BA+HD)=%E3# zu|k4ESt!xyq>c{a0%nW_7J?)Rl%|_Ad+M_iRB_t?JRuW2*HHna;bR8QhLJ)La~5C> z6p|JQ3ZQ|;eM6Mv$3S!n1z?tyfh-RmWYRVg=QT>R%>)5P;lNBY3_2*nC~rEn0F|OS zfHE|iA!#+*NFEKRjT%6Pp&4L-EQ^^S+eFarX-dS}b9G2yQ95lNQ_*N4Dzhdc15%=V zS@@YUbDWjc1_!&ICgQXKr7|=?D<|bN6#c=(D1J%+;s8Ssp%lCn& zE`eNF2w0nuhrntz+ zCS)c|=E8F!B%;6FHs5ZW|AB4uwNb3EAUFSg2LvccL#ve5-aiJ3f{ z1suBszn?J}P8c`-- ziq5}wLSp{)#aO_4uOFk6W1~`IV_N!XIp4YoiTTzR>N2!fnwDBlPHZ2WZ`(fWORf8a zAV!7r`YkS@O>A;ps*jfQtC^6PUtN+m5#HoLC1{`6IxfLSd%kNWB>Ju=i)rcgdva`2 z`?zS7^3g}TFN2Vf_%axd3%6dsrgTh-P41Wyo9v_g{A(v9das{f+}gs6EP}(i{K;2V zA^NCiSqT|NGyCK^2Nd=M9{?m7K{>{hp{ue>vFzeMXeaW7e zrFgviJfGZxjiUDxRc~o0SGs^9Iw=>P?g}>Bks0Z;1?RA zj#P#;R;o3r>PA|PMym-`HPVEKgsQ_rH4Q^TwISk(wYRH9tRphI2$4lbI6=#20c9ff8H6J$9pz||p*)FFmO2BSLC7zRTlB4DFNAz{XFTn!@|;|8N4 zLam2T6RD024-M6XMi?~VhR{$0)(8(ZhU$$77^F-D1+~CLb5=nYFw+*Obr>#81ZTGC zRR-D;jIJ%z-QNQFkfk++-N zHBbql1tWyy6qH~c`Ao!3ulCh;PU z@5_7uxE#qV3qNEwhV&RNM+qe&MHOf2gGyw$Qm6@K$IOpD!!(+3O=M$_hFoxC1-( zniFL2aqh{{wh774)~{c*z3RA_u;T5Dm4V%ky?DHw>ZvN*pu^;4-&J9gE2U^Yn}5vw z=aBgtcsrFn-m&V*9|O;}n69>Dm!4ej{MqB0Ps+S^q+Hn>=Xdm6_0PQve_QWuxp;2J z-4{D{ytwf9g)KX*d4KnOQlre>(gS{>>qvgRR6;Z zKV}py-z(PQnqfuV4QEPpPZ)B=e`?^2x;xr0%qgKs*Jy72wRq>@2fc%ni9AiasiQh> z+p}Uvq^^5FjTs$|4!M+mbjXA3D)VBl_U;_o@LKiK>ZOxDj;hkWUWG~v+P674_{#L2 ze@q;X4}P?;!=^D$&gY!oI`#cx<7*Gi&0P7-rR&FsBnM~wm5~>*r^+z4>yz#a=KLJi zXuz{IXG@+eyRP5T6-Q?HKZ?D4es&%^`O29yi;HdS_u}5&vX9?y_b{*L#tqwM<(0cX zY-0T10mY8y1dci}vcum?J0BWQQ;h|8N)1VA)n!O**E*LfjVjgiw~w!ux?lIR$@TXS zZyU4rlc?*5u5A6b{`6`pOZ^WrE)*|SWpbU;Ayvoj#@}iG?(yl$&*J-sG~GYp%Ba@U z(~g(U*q%2ytWo73s@F|SC^ob4h^RKJGzk@t;z5_X#666wn|8Qe`JXF(Q)}tIKmF_a z&+hv4V#$_sK5g{8#@MOjj;prrtXcJon9loEzZ(O~W^~)=hh@(Fy=2tp#*Y>}|1!Ay z@V|ntMvYtYB&jW|T4Cp?3l9{!t zScfq2J%M5R!s@d;mrGyXqp4nN)}xQU8-2;TsLbON1DZy6{c!b&!<#bjvfJ~jYHNN!nSQo= zNu58o?7C?FwaE=*UaS5ok4xdR$84%+obW?v)!u`ur}k>Ha!;S5HFlN*lladwJ*~dH_>J4gbCwz_H~!r` z>P(NaB=t??UlVGt8EDq!jXArsmO5T>XiZEEb*i!U1ih=EDQ6*c*Fuc0Sm>^Zqlecy zH*r*%Dw-APe^y#}tjcE{>T%(%>prX>s4um{)H`Tq&HjT+)Cw`|PJQr6lL7vn*UtZ8 z+k;_QYx_^nirfF~_ym8`ncDc%Q~f(VT3)PE{SLF*^%@PPb#LeY$K)ylHuTcQZ>)W6 z|AI=}>s4Iv;N2nhb_Q1UPg%VLH!ZsK(ZMQvmfDlP=)?R|K^<}CQj2?oo6gh>=(4@o z#GzQl&x1xCj+-^~a?^~>^C}$NUA|Y#Hj!6LetshIV9va0tF>dv6_Ms4hqm1O;)lLJ zjH(?yeWu^~PF9Bt9sStbX)qg}RmfhcymeF{}G@^ADGs zMok}>ivM}<2@`WU^^pArHnwNxxc7z(PP<#G;ntdUz{oa-{n~_RCe>H}v~TW(`)g9; zJ|?E`of|pw?Cu{f)qXMj-5)>xr}Kf0>8(FL->>nHcgQNswAaVuHP|~B8y)Q0>%t#b zKHQNUv|-IS#flpHnQkBZef9Cu$&0)049aZ#;z`}6+79aKQ~voNczCT^O~$mG5mIwY zgYXVoxbN$QHS9OvfBMagJ%+uTzfN8|tixOw*x`7|nGZ{@J25@ugKVPLkl5va%)Hz* z{nK2t3Dmn2mpr{IUh$nN>{pXw7QV+WD*5fO6cBiD$-}nq&SAHHR_43zV|I00cXauBh~FS9?KzO~m+q5h-pWpr%vqhe$laVbzgKh`w5 zJsB66HMo2F#@Qo>&1gQQ_oBgTudSv_E$+Qz?BP{A8I(udH%k#$A!8mlxJO*& z?e!Y}eY^cs{kMI7h4-!BblpAW(A4+VOj_TNXhu)Wo{_aKW@YcpF8x2<@z<0gq2Jw_ zRQ=bHVPhk|P1b+Ec>UsTSF3&(nGw``z`Vr=+iV1tW~s>w;+2p5N-TOS7RCQe2 zjCINDS7~cz*ZXL=qW;jiW9MqB-?l&AIJI{Z|G?hm7I29Q^}s($PKSC`h2~e)3lgZv zuTMN%G@?QHqmpZmjjGmS);~upjX0K_oT9$|?SsWf1Lt+?-R1jgQP<-ZkGqv{YE_*j zw?0`rOHr$&rITrXXJeK1=Q~YZ(dz!)m_3WXD>Jocdh7ZvB9Cu+Hz{M@gDXeVtd-k; z0_}wTZe(&y*U#s^^KstBk^wrp#I6AwzT8?hr{RL{df&eP>GzdwZFB>>9Y0m?_PJ8k z-)V9D=$Ux)z`6FXewu#uhkfY{1M-edsTDr))}B%AHul_Ec}$PAevOw`emFR-=Ck4P zxnJ1_c5JBB{y8zNbd8e!BdYwgU`UM%-10Bl=PrAv;exAu4v(KSrrer<;>)`BT6xrN z<@%fno4+5J?(Yomw7uqz*~GLO_`0vdqAR3c9`aGC$(q{f+uQ#5*L#bGZ|XSs{G(>8 z6!WT%Jv?jwAj62qcJ;CEI=9{y9k`+9-;;LD=y0^2-=lqP@2xynqUoirM7hJu>?2xj zznn30^lDIZx~w~=4mh=Tpf-&v7YVfm${vsaUWSJr0U%zC%$rCZNVUrssNvqsCy8Bw?NfzH&X@TSwAG_Z}@(`xiPv)N{yhBoDCz!4*_y4-g;-M`Yw%zATJAyRarDFegHQUGd69mj;e@J<%>xpKJ!^X5Q&1(LC4o;`F@1OW zm{t|X4{f;MR#|RwgZ^<<8=t*5DlM#I=fQ1<)hL@AS$yHeN5_UQzIJiQ-*a`Xr_H~N zRXWsr0F|{{Rpytet*ZV!>ek+2y}x0Ik*Z27QYw8Ec&0;O#K5w>*00DuX&W=+p?2!5 zjz>TGxZ2Xl;E(>eP~Y#O5erIe_f_iD#IfJ0zh87U#E=m&Frn7zdwW|x%lPu+>4{&G zhiYfp9)D~7V4OkK{>F&W{jZ<>p!Kx6XR(r34qa?wx&j8zoH(*|w^eKRCO_^Pmw}l( z9<3HVI;Nb%yg z9ksstt#pq=@dKVSPb!z0Gp|Lj>M75eC+nXbut&5xUvIm%NwT>^!&2M6S)DoJ^x^K2 z6JmFdpAw`!(yaKHn~k3IEE`AG^Bq zyKHv7oqb)1?EsoFKn_08kq4?QJ zGr3K3!sjg>x#HQ`L$;Rw+nJ@wKVw6NkC-*9dA z{X96RV*JRxTONI`dS}p_g!8|csvNviX3t`sIxAHbFn9g@VzVb0sn_B8uwjJXgm_tMH`S1Y_fd;imKcBi!d zBy88IV;NtJ>omFbjUf|bKEsDy?ECQNrFz@m-hV0WeTZ(V9V_=%s;BSPXw|KRa^-FI znduia7dNlkJ)p)<{*${ZlE-uz|HSy^{ZoVShOLZzwYU9|O&J&f|Z?Yw%G#XI$;J2Tr)Tv!!S@$0fYn65S zT)7%uZp8|1+Hh{=5!=HSGw;{AeCkg!WAXCsjeg8KyY-iUa=(A}&ub)Vb zb+rS1?rtwNqRfrwww3T(7vJ;-46Yk&W9h{|k=g#i=#V?)BzWl+B4{g7nxv~0u^?hYRTAnj( zdFRX0^S_@rdg??@*?jU^<$-0&RpswjOo@70+J^tR>)PeCU27H$AN14F`ju9U-msxn z$nln^N^viGZO#p!RoA@Y>}l(n&F9-U#GQ@{5TlfCnZ>JvuyX?3+p`rd_yBkpc!F|EY575qx%AC{j@<2;nf>A(Hv8bg z&HauVDnH8k=;p1!uWOZEkQlTUXv4s9I^C9NjVZ>f;f2*0uW({5-nY#4X)V`rUZE=0v|%f%h-2 zY4v^uY;tzXCiMvQoQG5QHopJQgH{{vWL6nEFyusJ`3ae zqCej`^zpXqTY}l*)T+EMo;^D<<#yi19WQ=)kvs1C=eP6L-^}~z{=r|0Z!A~tv%3FG zOl(D+yqTWa(tO8SqL`)i?J>I>Z94sQI5n*JxijNGy4^o@?U1k5pPaOQN!F&Q%CyZ7 z_p38@P>pXFpRsRBJ{dgg*ykgr*VQLv>#{RVi%+dyXlvN{{6Ou-PA{sQ`?*H&piOZv zTGYB5yfUw8wcD1fdC^Uq|8Y3Cq3yHh-+cKr-v6NDs|wFzzdZ5N3+=RSBNK8jxBMQY zExL5eI<8{c#aVS~WORAnsNIBx?>yeHWBWaT|G03-+27k9Z&tb55 ze;T>!!=)d8Q89x0YW>u#q^{J{VpEhdQxEOV42VHIU zNpcvqp;}I}dfm5vzxwLl_b=_eH2vs&?cY{f*}KftKm^Mw^>Fd~xij zXLpa-p0>EwbzbELQ!>9R`{aj5c{}btc$(KUYt@Ooa`(exY2!EMbG;*?)5a1e_fci|M)kO%CkO{@4o-H2?hF1KYau8&Vhj`Rwzbv0tW}6AnMa=Lcg8PRz<6zhA%Y1-oVG z^23!Yj_kK#V6Ds#%2b^H^{uwnzZRt(dhzt|?oWq|Fr4YKGjrJIk4CU(t+o@o`>Qv` zS6#89)$${g<8K~~p8w^9Q?>)kKRwaopzX@WnddFvSAO2&=^rmzocpfq`5Lna-z#^1 zYv8s)TV~#RTmeTUxZ8f4ja_H{nVo1n-8rzG2qyE?2AF`&TPwx5gPSBEs%(3N=b!ITY| zM?0=N+2H=8zecX_f9gTtkB6pwn{(n?$;=JiPd^Cxbx5g6Wfyl?U-Prr`A8y&QH*4m| z!H+3pkk*uvFnmMv!zHdx-9K^tdq0k?z+&vs(9w@wS>SM=t+W@@Cm>jkoqR+4}6Lr_Igjwxe?AoU;oCs@5H> zH7EIxJs0*|SX$@f>SfH}#Kuo4MS(%X0lM z*JAeXs`Jn3ceX9*{K;q4?A^u=>b9^#qhYz3%dSFl2)KFUyIM;h;4H|fBKx2pJ^IjM?HZ1?4~%EfeBYuZ1}X!}v0j@)(aCwImj z-_mU#5&hXx|Ac!hMm9cM?3_5)V)OpfhhhH@AJ6k&;R=<~z~HaT z2`K;b-_WoyO{m-XZ+Jvl=-cz(H}N>4_%qN%upDF8I)15~?jiEH$qcm8EWy!yQOrQ# zUVSq-%%lYwYLx~#16ge(nIz(-9@jf>AaEKy*5iqNO>~WAc?4Xw(+;K9T4z6D%a05B3)k zBPf#+z2bW$#8PCHG^NCpRvK6Md*I3stiK~Xt;L=!4sAPV7fK{x7N}6Tg?|Crefd}I zg&^7dUkc%hQ!2)oAg2X>l1o&y{N_`1ats^xb1(^hA28mL9+D?wkjS5E=9s`5xWC3j@ zzjuGYuQ?82lpRno(;55DO5+GBZR3RdOTvHvngXoZ;=Ft-CG9p_bfW*P6Ex%uINxDP z41Os1a0(HN1qUe=h_zTbJFu~k1??KMq8Am2iB3`Rz&Qhsa{Fn4pC1C5VkJmg45=qf zu|kTlCuYIOcg+N&Rf7Dnlm^_YREKKSstBc39ib3!w2C)35w%M(wRjSC4{xmkN;PN( zC|VdD1&XGnb5zj+rCI<Cz;f0T}~ru@D?jxdOF8 z_iU&-juvx|03$}SP$ABQl?NnVlM|Q+;9O^(6(9;lT(ZmwUjUSXWBQ4|1PT%V;4qV) z(3o?67}~}Wlo-xa;-P@GVwn_b-pbGh$g+;{;YZFO1h6Lg)sRUu=_qJn7{B_Rt6OxZ z)br%;n09pSMO^nCr7JN04(~Zfj7t6oj#iD#uii0tge*$3?7L2?t(ETLxdL~%$fL34 z`=*BumaVsoVcSs1Y%64B&>X&yBIZtqP62%}=PigyNC7x);pu`l@@rcnbC8J~${nLi z$h}0Rj7CCDtz&7Bv;kf3@|Ruy5mOfcjAaQE1##iNJan`p`0TSXsBXCqq_NUEJT(0U zm@Xh?7v)8W@^O+AimWA+W>ZLPF1=Io@7%i}%c8)yh2f}4h_0*4d9kqOgygG2_Ij5P zW&VQGDy9{j_c7-yz4%Hmkh;LGS+_8ie4C7o0+>O-L=TNL14U*_u)-dJE_8|P-Xln- zgXeR!$;3Cn?;jY&5kShZ&+i=kU^&R&D&!gvA-bpJm$-o(W>QQ`9#a-_{Qu$!Dk=jb z_(29CfM$T&6&g;kj19>%I=u}?o3(Ri5{E(JF9y${AZg0Pf0jl;z@Ft7i!dM3&L`dE zbwWogN{R`KWDBVQe*F`ak-=Dwu^BiU0|7(xhzJZ0qi|rwSQa@SD};;V^1>*O8mp7# zZkH#wP`VaYqQb(olLU(b6o@MZ1t8`*xE@NknRpJv&=!{kWWgxHD5NFucb1c_B7fwB zF2C!;zkyT+!O#@q`NI4O0sRmQV{Itjwb4cb3t7GXED(f-SQOG0`0z&Bm&;b87G0Mi7LjgypLkjIJn1BHP4dtiJK_cTfd4^vn9BrjZ+GIx&k@>bfSb#!^awuQsI;j>4fDZgzbpc-V<1z8` z7*avnj>oKm=qn>RkbyCTu)6s{rL8v9d72Wrs>DorOk~x>?1UD~ieycJl0+3qh^`=A zbU=+*kqU)G6b|DcPtGi_1)+>6u3{Z4cH3E7XO$E%nn7fS=*sTrV*~QwnjnBDV=)Ie z8!Sb!XB&tup=^ud<|D|u;$=sFH#iu>lLuoVD72lSVDd&Aue@PKel?;{DB=neVLOWJ zOblj#?t-NYu{ky{Bxi^^8gRJ#d)Yo^)UA_1X@Ot1B!2W&IVezNU?2{8QX*MZHfLqh z2@3CvexWQweTCsfzd>A#Kqr|z2U7|m_xYg}R2CKlNLTqJ_AfAfj0F@;DH)pPB+;*d zNErl4>x`_J&qy7j5hy0p3^_9zIBDN->?{(i;h__}>`Y`3Bs4)BSOnfCz;wQs?|=?r zDN_tLFRU+dE+w%<+@r(@G?Ko}i>0U*u5-N80>3~%X=4!B4@Q#CIxH`~N~$V!1_yG; zy?pzgySY$9dY44w$?{N-{wUCp5Mj{)!0I)1jVMIQ5;+1tjWD4Gt4%^fys21lP*Pe! zpbin$7*SIx6jCk;6M?!lA%|2H)ZCID^4P@_{0xXoj}=k@2-L~a7MbuYN^Fu*LM*R! z*>*F{kOG+)!AZ)uCqohTJsgry5c81y#Q()hPkdG9KC2c-&L?!9Q}X-LiTYI^Z(5j(_9C^+*{#s6uW0*5a-p!>JB49M-f{L_R69gD zn1`d&JK}G6>P&@bRkR-D+pJi|ce`@Q(=olG;m<`lkoD6QEoYlzA90FxlSiFGKsGNa z_f=&J z809oO2`3&P0Lx_g8YY7k$i(ReJiZp%MsZSyB&Eq|xsnu7R7?x}_$^E+yi*D)o1kxC zoY@JD2ZibB3@9NWf;SQ*WbG^mEe^&b$w>jC#ON|AJ6b;bZ4C#Vt%f<@4i8T{eD^E+Z)klzaiF*9` zA39c}6KWO|Sm&trfQ2Sr6BSR6I!Cov#tI4yKfbO2<>ZLWxC~8!UbJ3RhdtFe#~OA` zUL+pLNirfUO|_VT2Ix^q1UW$5&3R>s1y+*|w^^(hN%Ev17S{>Ib*^VWelI`#5$h|3 z`hL=QMG4%l9kheE7|&)r623VOq~t>_keJvJ2C_C1nV&sjImXvwR|>zCHimRSlr(x^ zV@N?#p=eGF9vSRidz4hgnFk^&Ey>|!Ac{#j1M8r4&jUuG*$vEtmpmM1P@=54RNq_O zUCzC?E>(4BXxWLPqOQjkf_rq0vZ5O`@nl`r$FApSVjz0Z;2Mo)qq1gQAJLphqVBGb zBWK<7Rn>j;bPs}?v-yYS54vwx{l2fh`rg%N>KF5$IBVI`+=8zZUprHL@Gt$Ve%3A9 zCUhPzou;wdcXW1cy5=YE&zW%Yt&>jq>gg{`UD`SKjgAw3`R1%;FLwTL)DAvj^ukpq zo;YswmO~4#{Zi|xOV@uiw0_5vl}T*}W>1^>>(@FTe`oj3<@ayBH zf82gXuKk4Ln(vKYHR@i(Y?2=ld+UAj#N37DQ-A%? z%5Oh*-9T@v+}g6{;NtZUFFNCv;@%Ad*UumO@elWoC(mE;y(@QJ^|1fV9n%;5?1kGV zpML$F&gmZ>%3bv4%$1M-?O!gbv_3g$=jdBjUj6F4W$!((bq!y!d3#ISqd$JT@||6a z?znqi`PGwW>}$HV@`H&(cmC^|ec%1b!Y$KIT6~3n*0Qs{kUeK`+PEA4e8OMx8>jtXy=_e_i`oTjsLug&!|l(7U?g?U`}TI5Brs=EkNamrQ*0 zl%DH1ytjSX8{K0jtZ7GQL`GI-B>YY=%a$oO^X3!AKR*4Q_a{$1XWEz#*WGdF#w9au-@SdZ`>hkFJho)h zhK^B_U!B08e+=zMFa7(H9b=|WdWdhG_Pu3y?7r05-TA_ApZ?km=imo7lY{R~ef^UK z-`_v+q}SM~1fxW+nb^`ZZH=8g|`?!i}{{QiV} z=>b^Yf0hnAiA?kBr`?lcC|p8oKgPj&VNlTW(f-OKLTwU8bkezfr5EgyeS z`Q*`|Uw!@4-~H!hqeqSU^was?1Di%=M*p$@>eaDV^s4>)Z|XK=k+r$=hq?s#@PGfU zt-XEj@W21oeopJ$V}Jkc|KmkHTv8Wtb5f#0@=88_mrJfn%&9vGbLvt=P68&VWH1!l z)Tw(Z_Z*ee4ktV4J`sPKwDDmHqK$ zm^sRt9cGJWoB1{gqE5v#FBGMRum9lp<`x%+GWNBL9~b*61c455no1k zjy8vZqwXH<>g^V2kjgSTFCczL&+-fAT!3BbL^t3Q9~WF=+1#-P;^%oJc}NryusMma zUqL<@L=Kj?fCWJTa}+8IC32}pjX)6dktln}BOc?G1YVRq%iLj~Koe;@uwb9qYPI?v zbyQ>=_2|n&b7-*_$O;Mt5opCcVQv&f-Ccb-0GGC7m6-rJHz>%yfVvDh%g;7JTHr$r;!*hc{;14 zD;klO{?;-yEXgdj-aIN|SCEX^i2#5Eb%>3*_8Ul6Z@SQkdWesbeCdKA3^LwNxX7cq zVEBn)n>&OHp!xB(haxfg^M$z?w=up%R(%f(hc}Kg7)}D0+S+SGdl&16-t6dmE1I1L z0pT!4$rjdDAhEYDR1o#4bP5@y2d2>CD}fE?$t}j3eTNir=t|VRx(_+b z1HE8HFzZOo(O{YIlyqtOYGgCU(lhh1X}HYBuENKTLv?&Yu5!o-3qeahabOX!ZxJ0< zxP;hBha*%yAs<~29my&mUsT+vF{>bx1xAQzIi(dPMG0i3$;E5P!n5H^z)6?^TG&w} z$pGP%DmSccE)&m zq{uiiewGRy9*C28dYEn{VL9dQB`cMH0E7BINx;ueWK_vXQiuxS`n1YZ|3t}9#R--vo@KB{PRv~m$oKAw05kur(>{?N{%rC*$ zkrXud0*poPM-=olL|O*QqF7N+P-#U(0PLUoMUI8!p)EsB5NE)Mc5v8tP=JL{cW^7H z>cN&8Y-voZ+8#H$+3LYrHCS)EON9iEjK#<@mdb+{2%>(Oa0=R7@1 z)X}qCVKE%avZV%&xc9LSikxs9#w76q7fYg3uI^6M)WaOq9a{(LRMs&|>mXDIPC@&x zBI6#0v@5PCHXGF#$_7qBCu0!gIoOG00drZY0$xYas&N5;J1gKRK&6>llT7sk1Yt4y zc7H`AP}P%4Q6!cG)LB3Q#ELa(oiNKzW?Mm2doixmOzBmnVT1rw<26wesIee?fZ9Yf z6W<9K^`)3)D1qTe!AGaL@WdF0nuM@zv(T4XQB*KOSfF$J&RgAlVaKYKQI@XlLFFHX zN|HQ84#K!_EldM#8E7kkP9z#_;czyNj7~i&+7J>A4Va_l2unr<>N~KGqgtm>HW{ADS6O$QF{s4O z)WfF!te>x>+=&%In~ zi7YF@MS(gF<&kd(bPB}^9iT2L5eIoXm}t!8c{=5Yv(rH=?J^O@p_#pm@<5Xml;{BQ zlimi@0cw_G#n1An;mn385^H~tT{oE{pk9dD0#L{4nl&jjfPQWIq-R9WdYIE|CYtcv z)pNcsCKG_fxZ<1#s%M?GTXqme)mTGoLyTeA(5d86K$&^4VWw_0rQuOAWeP^aS9?p# zDw@XnJj_+spo%{OTE^T_Yxby5z0gBxO$|6GhFj4oVvVaeU4WDj+uF^#6|7uztL`o6Xb$~bQ1q)DJAk}Ks z@c?D2wc;}DpaOQWZzpY!l?*$b=9yN;?$v$hQs!$rjo(0xk zP>n<4SOA$vP9bNSKRKIeJsMts2Z3ABMW#*whEr4(oG`^-RaqXV26apGO{NaNv|hDd zRpmyMWhEfP)~eDxld!<ebdqLDh7b4olXI8JTXnLW#FG zO2^VUO=_LhY^6M-UZdlSVRcKAyKIJvYACl^s{?3*Nmq`Fk7(s1F%qr%Fq~vzOnr7T zkai@RNFC&mLRcyRD@M*q(@YBD7L65k!?1kSb`B6;V1j4_7fdp>OM!6_piTRGa99lH{%`>_)k0q7%^*OV2Rms{5q4SR}SmHp7&P4!w)})oIXbE;C?MZzohMMr8fEgt_K&dKV7Vzt- z0?sTbDz1|bXVkDz!}6#BLReNm%j(izmq}Qb6EuLr1vWq;1`K?WxZ4P6igCnXSD^#&~N>x_yO(3L@9>0x<9!mJHDjNNQ&0k%F!3ScwEFdBm6IaB*G7%IsoYcY%DX0n_TJ z+WnmQ=n_^iZIjySGqz!BW49CA{h?7nbxafGh$7l*R}Vx5HBuornP?74vp_+~azZb_ zt_!MQhFf~LRee0z@Y&~sW`0)BZ;0g8p}pM-QbEhRYsrW?#=BYlq+(GgIkQZQTC12@ z2qz+R)iFgaRxDqylGX3CA$Ma$qq(eTuf+BUl~U7ho~b8CIMYtWEWt4K=lj7@}U<7z!{|W1%rZokmj%QCK*jX)x2MF}@sO zbbjy3v`50BEJMgKC!bC6!6@Vgsx7X4e}Fx!>ZcWnr%U=duo2jCRQn|*5VHqSj-b0( z>a+mHf^R7ou!`PLoEo=mx{u0nI81NUN^{hr^<=P2?Q$%0pFoHylhn2-?Vvs)*e=(r z`jaEuWFxc`qXU@Q@6Dl4#EDs%Ir9NiFA&wXALLX@;=^uXKYfrhTT2G=%-X{$SU+cy z*icu+Qc&z}bpDtw;gm>+Z*hzqP|bX`!l!7$XEK^jk_K};^!vphbJQ9V*kAex}8Pd!p> zN6uMS;;Eln9Krl3vYOd(=W$qDt_HwNAF1(JueEnLjM|Rls#M4E2!TdwIIcQe!|^k~ zV#ajSju)_+`n-fVlCpn`y5@4g< z9s0CmBpMj?RgL>>V8B^xeh(Pg)G!R~|8r{S@}kO61d+N~sGz_~0BknG8i%HjI&?Op ziiXg2wI4F&d2k3UtMy4zLng!1O?2I<){emrLZuF#`lL5D(T}KC zpBmdbd~DpZ?TCIB5ucQ*6F+rm-iUgS;-`^`_iEM{*?>7&pE;ztF*;_=V&BPPU(#%R z7MuLgV)GC%w3fMqn?~3lk|zl1Qqh+&h2~81UgXjGi-CdF%puXK{pVr;LSMENQin9d w>a&8l`ZtHo8Tj0%J~FNr{Kx*8@3Ggh*Rj{JSIz5x0{{U3|GrFW?*Nbi0N<|8z5oCK literal 0 HcmV?d00001 diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 9a4165797..7b3dd462d 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -63,9 +63,9 @@ rules: - secrets verbs: - create - - update - delete - get + - update # to check nodes for node readiness label - apiGroups: - "" @@ -102,9 +102,9 @@ rules: - delete - get - list - - watch - - update - patch + - update + - watch # to resize the filesystem in Spilo pods when increasing volume size - apiGroups: - "" diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 195a03380..b5d561807 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.3.1 + tag: v1.4.0 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. @@ -100,8 +100,14 @@ configKubernetes: pod_management_policy: "ordered_ready" # label assigned to the Postgres pods (and services/endpoints) pod_role_label: spilo-role + # service account definition as JSON/YAML string to be used by postgres cluster pods + # pod_service_account_definition: "" + # name of service account to be used by postgres cluster pods pod_service_account_name: "postgres-pod" + # role binding definition as JSON/YAML string to be used by pod service account + # pod_service_account_role_binding_definition: "" + # Postgres pods are terminated forcefully after this timeout pod_terminate_grace_period: 5m # template for database user secrets generated by the operator diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 8b52a7d67..07ba76285 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.3.1 + tag: v1.4.0 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. @@ -93,8 +93,14 @@ configKubernetes: pod_management_policy: "ordered_ready" # label assigned to the Postgres pods (and services/endpoints) pod_role_label: spilo-role + # service account definition as JSON/YAML string to be used by postgres cluster pods + # pod_service_account_definition: "" + # name of service account to be used by postgres cluster pods pod_service_account_name: "postgres-pod" + # role binding definition as JSON/YAML string to be used by pod service account + # pod_service_account_role_binding_definition: "" + # Postgres pods are terminated forcefully after this timeout pod_terminate_grace_period: 5m # template for database user secrets generated by the operator diff --git a/docs/user.md b/docs/user.md index 91a010b9c..295c149bd 100644 --- a/docs/user.md +++ b/docs/user.md @@ -359,13 +359,16 @@ stored in secrets which are created by the operator. One solution is to create secrets beforehand and paste in the credentials of the source cluster. Otherwise, you will see errors in the Postgres logs saying users cannot log in and the operator logs will complain about not being able to sync resources. -This, however, can safely be ignored as it will be sorted out once the cluster -is detached from the source (and it’s still harmless if you don’t plan to). -You can also edit the secrets afterwards. Find them by: +When you only run a standby leader, you can safely ignore this, as it will be +sorted out once the cluster is detached from the source. It is also harmless if +you don’t plan it. But, when you created a standby replica, too, fix the +credentials right away. WAL files will pile up on the standby leader if no +connection can be established between standby replica(s). You can also edit the +secrets after their creation. Find them by: ```bash -kubectl get secrets --all-namespaces | grep +kubectl get secrets --all-namespaces | grep ``` ### Promote the standby diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index aa7bef034..0300b5495 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -63,7 +63,9 @@ data: pod_label_wait_timeout: 10m pod_management_policy: "ordered_ready" pod_role_label: spilo-role + # pod_service_account_definition: "" pod_service_account_name: "postgres-pod" + # pod_service_account_role_binding_definition: "" pod_terminate_grace_period: 5m # postgres_superuser_teams: "postgres_superusers" # protected_role_names: "admin" diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 80fcd89ef..e5bc49f83 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -64,9 +64,9 @@ rules: - secrets verbs: - create - - update - delete - get + - update # to check nodes for node readiness label - apiGroups: - "" @@ -103,9 +103,9 @@ rules: - delete - get - list - - watch - - update - patch + - update + - watch # to resize the filesystem in Spilo pods when increasing volume size - apiGroups: - "" diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index e3bc3e3e4..63f17d9fa 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -15,7 +15,7 @@ spec: serviceAccountName: postgres-operator containers: - name: postgres-operator - image: registry.opensource.zalan.do/acid/postgres-operator:v1.3.1 + image: registry.opensource.zalan.do/acid/postgres-operator:v1.4.0 imagePullPolicy: IfNotPresent resources: requests: diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 3c49b9a13..140d2bc4e 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -224,7 +224,7 @@ func (c *Controller) initRoleBinding() { switch { case err != nil: - panic(fmt.Errorf("unable to parse the definition of the role binding for the pod service account definition from the operator configuration: %v", err)) + panic(fmt.Errorf("unable to parse the role binding definition from the operator configuration: %v", err)) case groupVersionKind.Kind != "RoleBinding": panic(fmt.Errorf("role binding definition in the operator configuration defines another type of resource: %v", groupVersionKind.Kind)) default: diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 8e8f9ae85..96d12bb9f 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -505,11 +505,11 @@ func (c *Controller) submitRBACCredentials(event ClusterEvent) error { namespace := event.NewSpec.GetNamespace() if err := c.createPodServiceAccount(namespace); err != nil { - return fmt.Errorf("could not create pod service account %v : %v", c.opConfig.PodServiceAccountName, err) + return fmt.Errorf("could not create pod service account %q : %v", c.opConfig.PodServiceAccountName, err) } if err := c.createRoleBindings(namespace); err != nil { - return fmt.Errorf("could not create role binding %v : %v", c.PodServiceAccountRoleBinding.Name, err) + return fmt.Errorf("could not create role binding %q : %v", c.PodServiceAccountRoleBinding.Name, err) } return nil } @@ -520,16 +520,16 @@ func (c *Controller) createPodServiceAccount(namespace string) error { _, err := c.KubeClient.ServiceAccounts(namespace).Get(podServiceAccountName, metav1.GetOptions{}) if k8sutil.ResourceNotFound(err) { - c.logger.Infof(fmt.Sprintf("creating pod service account in the namespace %v", namespace)) + c.logger.Infof(fmt.Sprintf("creating pod service account %q in the %q namespace", podServiceAccountName, namespace)) // get a separate copy of service account // to prevent a race condition when setting a namespace for many clusters sa := *c.PodServiceAccount if _, err = c.KubeClient.ServiceAccounts(namespace).Create(&sa); err != nil { - return fmt.Errorf("cannot deploy the pod service account %v defined in the config map to the %v namespace: %v", podServiceAccountName, namespace, err) + return fmt.Errorf("cannot deploy the pod service account %q defined in the configuration to the %q namespace: %v", podServiceAccountName, namespace, err) } - c.logger.Infof("successfully deployed the pod service account %v to the %v namespace", podServiceAccountName, namespace) + c.logger.Infof("successfully deployed the pod service account %q to the %q namespace", podServiceAccountName, namespace) } else if k8sutil.ResourceAlreadyExists(err) { return nil } @@ -545,14 +545,14 @@ func (c *Controller) createRoleBindings(namespace string) error { _, err := c.KubeClient.RoleBindings(namespace).Get(podServiceAccountRoleBindingName, metav1.GetOptions{}) if k8sutil.ResourceNotFound(err) { - c.logger.Infof("Creating the role binding %v in the namespace %v", podServiceAccountRoleBindingName, namespace) + c.logger.Infof("Creating the role binding %q in the %q namespace", podServiceAccountRoleBindingName, namespace) // get a separate copy of role binding // to prevent a race condition when setting a namespace for many clusters rb := *c.PodServiceAccountRoleBinding _, err = c.KubeClient.RoleBindings(namespace).Create(&rb) if err != nil { - return fmt.Errorf("cannot bind the pod service account %q defined in the config map to the cluster role in the %q namespace: %v", podServiceAccountName, namespace, err) + return fmt.Errorf("cannot bind the pod service account %q defined in the configuration to the cluster role in the %q namespace: %v", podServiceAccountName, namespace, err) } c.logger.Infof("successfully deployed the role binding for the pod service account %q to the %q namespace", podServiceAccountName, namespace) diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 0e88c60d7..fee65be81 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -91,12 +91,11 @@ type Config struct { Scalyr LogicalBackup - WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' - EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` - Sidecars map[string]string `name:"sidecar_docker_images"` - // default name `operator` enables backward compatibility with the older ServiceAccountName field - PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` + WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' + EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` + Sidecars map[string]string `name:"sidecar_docker_images"` + PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` // value of this string must be valid JSON or YAML; see initPodServiceAccount PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""` PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""` From 51909204fd7085f5b890cc4689221fece54ccecc Mon Sep 17 00:00:00 2001 From: Hengchu Zhang Date: Fri, 28 Feb 2020 08:13:58 -0500 Subject: [PATCH 055/104] Change `logging_rest_api.api_port` to `8080` instead of `8008` (#848) The documentation states that the default operator REST service is at port `8080`, but the current default CRD based configuration is `8008`. Changing the default config to match documentation. --- manifests/postgresql-operator-default-configuration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index bdb131fc5..33838b2a9 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -110,7 +110,7 @@ configuration: log_statement: all # teams_api_url: "" logging_rest_api: - api_port: 8008 + api_port: 8080 cluster_history_entries: 1000 ring_log_lines: 100 scalyr: From ae2a38d62a7202f9bda9070a8f9dd0406557fb0d Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 6 Mar 2020 12:55:34 +0100 Subject: [PATCH 056/104] add e2e test for node readiness label (#846) * add e2e test for node readiness label * refactoring and order tests alphabetically * always wait for replica after failover --- e2e/tests/test_e2e.py | 372 +++++++++++++++++++++++++---------------- pkg/controller/node.go | 8 +- 2 files changed, 235 insertions(+), 145 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 12106601e..6760e815d 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -57,6 +57,7 @@ class EndToEndTestCase(unittest.TestCase): k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") k8s.wait_for_pod_start('spilo-role=master') + k8s.wait_for_pod_start('spilo-role=replica') @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_enable_load_balancer(self): @@ -107,141 +108,6 @@ class EndToEndTestCase(unittest.TestCase): self.assertEqual(repl_svc_type, 'ClusterIP', "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_min_resource_limits(self): - ''' - Lower resource limits below configured minimum and let operator fix it - ''' - k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' - _, failover_targets = k8s.get_pg_nodes(cluster_label) - - # configure minimum boundaries for CPU and memory limits - minCPULimit = '500m' - minMemoryLimit = '500Mi' - patch_min_resource_limits = { - "data": { - "min_cpu_limit": minCPULimit, - "min_memory_limit": minMemoryLimit - } - } - k8s.update_config(patch_min_resource_limits) - - # lower resource limits below minimum - pg_patch_resources = { - "spec": { - "resources": { - "requests": { - "cpu": "10m", - "memory": "50Mi" - }, - "limits": { - "cpu": "200m", - "memory": "200Mi" - } - } - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) - k8s.wait_for_master_failover(failover_targets) - - pods = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector='spilo-role=master,' + cluster_label).items - self.assert_master_is_unique() - masterPod = pods[0] - - self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, - "Expected CPU limit {}, found {}" - .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) - self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, - "Expected memory limit {}, found {}" - .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_multi_namespace_support(self): - ''' - Create a customized Postgres cluster in a non-default namespace. - ''' - k8s = self.k8s - - with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: - pg_manifest = yaml.safe_load(f) - pg_manifest["metadata"]["namespace"] = self.namespace - yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) - - k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") - k8s.wait_for_pod_start("spilo-role=master", self.namespace) - self.assert_master_is_unique(self.namespace, "acid-test-cluster") - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_scaling(self): - ''' - Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. - ''' - k8s = self.k8s - labels = "cluster-name=acid-minimal-cluster" - - k8s.wait_for_pg_to_scale(3) - self.assertEqual(3, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() - - k8s.wait_for_pg_to_scale(2) - self.assertEqual(2, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_taint_based_eviction(self): - ''' - Add taint "postgres=:NoExecute" to node with master. This must cause a failover. - ''' - k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' - - # get nodes of master and replica(s) (expected target of new master) - current_master_node, failover_targets = k8s.get_pg_nodes(cluster_label) - num_replicas = len(failover_targets) - - # if all pods live on the same node, failover will happen to other worker(s) - failover_targets = [x for x in failover_targets if x != current_master_node] - if len(failover_targets) == 0: - nodes = k8s.api.core_v1.list_node() - for n in nodes.items: - if "node-role.kubernetes.io/master" not in n.metadata.labels and n.metadata.name != current_master_node: - failover_targets.append(n.metadata.name) - - # taint node with postgres=:NoExecute to force failover - body = { - "spec": { - "taints": [ - { - "effect": "NoExecute", - "key": "postgres" - } - ] - } - } - - # patch node and test if master is failing over to one of the expected nodes - k8s.api.core_v1.patch_node(current_master_node, body) - k8s.wait_for_master_failover(failover_targets) - k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - - new_master_node, new_replica_nodes = k8s.get_pg_nodes(cluster_label) - self.assertNotEqual(current_master_node, new_master_node, - "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) - self.assertEqual(num_replicas, len(new_replica_nodes), - "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) - self.assert_master_is_unique() - - # undo the tainting - body = { - "spec": { - "taints": [] - } - } - k8s.api.core_v1.patch_node(new_master_node, body) - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_logical_backup_cron_job(self): ''' @@ -306,6 +172,133 @@ class EndToEndTestCase(unittest.TestCase): self.assertEqual(0, len(jobs), "Expected 0 logical backup jobs, found {}".format(len(jobs))) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_min_resource_limits(self): + ''' + Lower resource limits below configured minimum and let operator fix it + ''' + k8s = self.k8s + cluster_label = 'cluster-name=acid-minimal-cluster' + labels = 'spilo-role=master,' + cluster_label + _, failover_targets = k8s.get_pg_nodes(cluster_label) + + # configure minimum boundaries for CPU and memory limits + minCPULimit = '500m' + minMemoryLimit = '500Mi' + patch_min_resource_limits = { + "data": { + "min_cpu_limit": minCPULimit, + "min_memory_limit": minMemoryLimit + } + } + k8s.update_config(patch_min_resource_limits) + + # lower resource limits below minimum + pg_patch_resources = { + "spec": { + "resources": { + "requests": { + "cpu": "10m", + "memory": "50Mi" + }, + "limits": { + "cpu": "200m", + "memory": "200Mi" + } + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) + k8s.wait_for_pod_failover(failover_targets, labels) + k8s.wait_for_pod_start('spilo-role=replica') + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=labels).items + self.assert_master_is_unique() + masterPod = pods[0] + + self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, + "Expected CPU limit {}, found {}" + .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) + self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, + "Expected memory limit {}, found {}" + .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_multi_namespace_support(self): + ''' + Create a customized Postgres cluster in a non-default namespace. + ''' + k8s = self.k8s + + with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: + pg_manifest = yaml.safe_load(f) + pg_manifest["metadata"]["namespace"] = self.namespace + yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) + + k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") + k8s.wait_for_pod_start("spilo-role=master", self.namespace) + self.assert_master_is_unique(self.namespace, "acid-test-cluster") + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_node_readiness_label(self): + ''' + Remove node readiness label from master node. This must cause a failover. + ''' + k8s = self.k8s + cluster_label = 'cluster-name=acid-minimal-cluster' + labels = 'spilo-role=master,' + cluster_label + readiness_label = 'lifecycle-status' + readiness_value = 'ready' + + # get nodes of master and replica(s) (expected target of new master) + current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + num_replicas = len(current_replica_nodes) + failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # add node_readiness_label to potential failover nodes + patch_readiness_label = { + "metadata": { + "labels": { + readiness_label: readiness_value + } + } + } + for failover_target in failover_targets: + k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) + + # define node_readiness_label in config map which should trigger a failover of the master + patch_readiness_label_config = { + "data": { + "node_readiness_label": readiness_label + ':' + readiness_value, + } + } + k8s.update_config(patch_readiness_label_config) + new_master_node, new_replica_nodes = self.assert_failover( + current_master_node, num_replicas, failover_targets, cluster_label) + + # patch also node where master ran before + k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_scaling(self): + ''' + Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. + ''' + k8s = self.k8s + labels = "cluster-name=acid-minimal-cluster" + + k8s.wait_for_pg_to_scale(3) + self.assertEqual(3, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() + + k8s.wait_for_pg_to_scale(2) + self.assertEqual(2, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_service_annotations(self): ''' @@ -346,18 +339,116 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(unpatch_custom_service_annotations) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_taint_based_eviction(self): + ''' + Add taint "postgres=:NoExecute" to node with master. This must cause a failover. + ''' + k8s = self.k8s + cluster_label = 'cluster-name=acid-minimal-cluster' + + # get nodes of master and replica(s) (expected target of new master) + current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + num_replicas = len(current_replica_nodes) + failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # taint node with postgres=:NoExecute to force failover + body = { + "spec": { + "taints": [ + { + "effect": "NoExecute", + "key": "postgres" + } + ] + } + } + + # patch node and test if master is failing over to one of the expected nodes + k8s.api.core_v1.patch_node(current_master_node, body) + new_master_node, new_replica_nodes = self.assert_failover( + current_master_node, num_replicas, failover_targets, cluster_label) + + # add toleration to pods + patch_toleration_config = { + "data": { + "toleration": "key:postgres,operator:Exists,effect:NoExecute" + } + } + k8s.update_config(patch_toleration_config) + + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + + def get_failover_targets(self, master_node, replica_nodes): + ''' + If all pods live on the same node, failover will happen to other worker(s) + ''' + k8s = self.k8s + + failover_targets = [x for x in replica_nodes if x != master_node] + if len(failover_targets) == 0: + nodes = k8s.api.core_v1.list_node() + for n in nodes.items: + if "node-role.kubernetes.io/master" not in n.metadata.labels and n.metadata.name != master_node: + failover_targets.append(n.metadata.name) + + return failover_targets + + def assert_failover(self, current_master_node, num_replicas, failover_targets, cluster_label): + ''' + Check if master is failing over. The replica should move first to be the switchover target + ''' + k8s = self.k8s + k8s.wait_for_pod_failover(failover_targets, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + + new_master_node, new_replica_nodes = k8s.get_pg_nodes(cluster_label) + self.assertNotEqual(current_master_node, new_master_node, + "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) + self.assertEqual(num_replicas, len(new_replica_nodes), + "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) + self.assert_master_is_unique() + + return new_master_node, new_replica_nodes + def assert_master_is_unique(self, namespace='default', clusterName="acid-minimal-cluster"): ''' Check that there is a single pod in the k8s cluster with the label "spilo-role=master" To be called manually after operations that affect pods ''' - k8s = self.k8s labels = 'spilo-role=master,cluster-name=' + clusterName num_of_master_pods = k8s.count_pods_with_label(labels, namespace) self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) + def assert_distributed_pods(self, master_node, replica_nodes, cluster_label): + ''' + Other tests can lead to the situation that master and replica are on the same node. + Toggle pod anti affinty to distribute pods accross nodes (replica in particular). + ''' + k8s = self.k8s + failover_targets = self.get_failover_targets(master_node, replica_nodes) + + # enable pod anti affintiy in config map which should trigger movement of replica + patch_enable_antiaffinity = { + "data": { + "enable_pod_antiaffinity": "true" + } + } + k8s.update_config(patch_enable_antiaffinity) + self.assert_failover( + master_node, len(replica_nodes), failover_targets, cluster_label) + + # disable pod anti affintiy again + patch_disable_antiaffinity = { + "data": { + "enable_pod_antiaffinity": "false" + } + } + k8s.update_config(patch_disable_antiaffinity) + class K8sApi: @@ -445,15 +536,14 @@ class K8s: def count_pods_with_label(self, labels, namespace='default'): return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) - def wait_for_master_failover(self, expected_master_nodes, namespace='default'): + def wait_for_pod_failover(self, failover_targets, labels, namespace='default'): pod_phase = 'Failing over' - new_master_node = '' - labels = 'spilo-role=master,cluster-name=acid-minimal-cluster' + new_pod_node = '' - while (pod_phase != 'Running') or (new_master_node not in expected_master_nodes): + while (pod_phase != 'Running') or (new_pod_node not in failover_targets): pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items if pods: - new_master_node = pods[0].spec.node_name + new_pod_node = pods[0].spec.node_name pod_phase = pods[0].status.phase time.sleep(self.RETRY_TIMEOUT_SEC) diff --git a/pkg/controller/node.go b/pkg/controller/node.go index 6f7befa27..8052458c3 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -5,7 +5,7 @@ import ( "time" "github.com/zalando/postgres-operator/pkg/util/retryutil" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -172,19 +172,19 @@ func (c *Controller) nodeDelete(obj interface{}) { } func (c *Controller) moveMasterPodsOffNode(node *v1.Node) { - + // retry to move master until configured timeout is reached err := retryutil.Retry(1*time.Minute, c.opConfig.MasterPodMoveTimeout, func() (bool, error) { err := c.attemptToMoveMasterPodsOffNode(node) if err != nil { - return false, fmt.Errorf("unable to move master pods off the unschedulable node; will retry after delay of 1 minute") + return false, err } return true, nil }, ) if err != nil { - c.logger.Warningf("failed to move master pods from the node %q: timeout of %v minutes expired", node.Name, c.opConfig.MasterPodMoveTimeout) + c.logger.Warningf("failed to move master pods from the node %q: %v", node.Name, err) } } From 35b2213e058aebfde274844844ea66ee9556fd3f Mon Sep 17 00:00:00 2001 From: Jonathan Herlin Date: Wed, 11 Mar 2020 11:32:13 +0100 Subject: [PATCH 057/104] Fix typo in values file (#861) * Fix typo Co-authored-by: Jonathan Herlin --- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- docs/reference/operator_parameters.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index b5d561807..d170e0b77 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -218,7 +218,7 @@ configLogicalBackup: logical_backup_s3_endpoint: "" # S3 Secret Access Key logical_backup_s3_secret_access_key: "" - # S3 server side encription + # S3 server side encryption logical_backup_s3_sse: "AES256" # backup schedule in the cron format logical_backup_schedule: "30 00 * * *" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 07ba76285..b6f18f305 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -209,7 +209,7 @@ configLogicalBackup: logical_backup_s3_endpoint: "" # S3 Secret Access Key logical_backup_s3_secret_access_key: "" - # S3 server side encription + # S3 server side encryption logical_backup_s3_sse: "AES256" # backup schedule in the cron format logical_backup_schedule: "30 00 * * *" diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index ad519b657..ba8e73cf8 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -284,7 +284,7 @@ configuration they are grouped under the `kubernetes` key. used for AWS volume resizing and not required if you don't need that capability. The default is `false`. - * **master_pod_move_timeout** +* **master_pod_move_timeout** The period of time to wait for the success of migration of master pods from an unschedulable node. The migration includes Patroni switchovers to respective replicas on healthy nodes. The situation where master pods still @@ -472,7 +472,7 @@ grouped under the `logical_backup` key. When using non-AWS S3 storage, endpoint can be set as a ENV variable. The default is empty. * **logical_backup_s3_sse** - Specify server side encription that S3 storage is using. If empty string + Specify server side encryption that S3 storage is using. If empty string is specified, no argument will be passed to `aws s3` command. Default: "AES256". * **logical_backup_s3_access_key_id** From cde61f3f0b9d91863ac16cab89463c647d297517 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 11 Mar 2020 14:08:54 +0100 Subject: [PATCH 058/104] e2e: wait for pods after disabling anti affinity (#862) --- e2e/tests/test_e2e.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 6760e815d..f6be8a600 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -441,13 +441,15 @@ class EndToEndTestCase(unittest.TestCase): self.assert_failover( master_node, len(replica_nodes), failover_targets, cluster_label) - # disable pod anti affintiy again + # now disable pod anti affintiy again which will cause yet another failover patch_disable_antiaffinity = { "data": { "enable_pod_antiaffinity": "false" } } k8s.update_config(patch_disable_antiaffinity) + k8s.wait_for_pod_start('spilo-role=master') + k8s.wait_for_pod_start('spilo-role=replica') class K8sApi: From 650b8daf77f35d03b0fbd989908c3dea4a78108f Mon Sep 17 00:00:00 2001 From: grantlanglois Date: Thu, 12 Mar 2020 04:12:53 -0700 Subject: [PATCH 059/104] add json:omitempty option to ClusterDomain (#851) Co-authored-by: mlu42 --- pkg/apis/acid.zalan.do/v1/operator_configuration_type.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index ded5261fb..8463be6bd 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -53,7 +53,7 @@ type KubernetesMetaConfiguration struct { EnableInitContainers *bool `json:"enable_init_containers,omitempty"` EnableSidecars *bool `json:"enable_sidecars,omitempty"` SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"` - ClusterDomain string `json:"cluster_domain"` + ClusterDomain string `json:"cluster_domain,omitempty"` OAuthTokenSecretName spec.NamespacedName `json:"oauth_token_secret_name,omitempty"` InfrastructureRolesSecretName spec.NamespacedName `json:"infrastructure_roles_secret_name,omitempty"` PodRoleLabel string `json:"pod_role_label,omitempty"` From 65fb2ce1a622109d4ec0f9cc40bf8590d8647659 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Fri, 13 Mar 2020 11:44:38 +0100 Subject: [PATCH 060/104] add support for custom TLS certificates (#798) * add support for custom TLS certificates --- docs/reference/cluster_manifest.md | 21 ++++ docs/user.md | 47 ++++++++ go.mod | 1 + go.sum | 1 + manifests/complete-postgres-manifest.yaml | 7 ++ manifests/postgresql.crd.yaml | 13 +++ pkg/apis/acid.zalan.do/v1/crds.go | 18 ++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 8 ++ .../acid.zalan.do/v1/zz_generated.deepcopy.go | 21 ++++ pkg/cluster/k8sres.go | 100 +++++++++++++++--- pkg/cluster/k8sres_test.go | 67 +++++++++++- 11 files changed, 285 insertions(+), 19 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 7b049b6fa..92e457d7e 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -359,3 +359,24 @@ CPU and memory limits for the sidecar container. * **memory** memory limits for the sidecar container. Optional, overrides the `default_memory_limits` operator configuration parameter. Optional. + +## Custom TLS certificates + +Those parameters are grouped under the `tls` top-level key. + +* **secretName** + By setting the `secretName` value, the cluster will switch to load the given + Kubernetes Secret into the container as a volume and uses that as the + certificate instead. It is up to the user to create and manage the + Kubernetes Secret either by hand or using a tool like the CertManager + operator. + +* **certificateFile** + Filename of the certificate. Defaults to "tls.crt". + +* **privateKeyFile** + Filename of the private key. Defaults to "tls.key". + +* **caFile** + Optional filename to the CA certificate. Useful when the client connects + with `sslmode=verify-ca` or `sslmode=verify-full`. diff --git a/docs/user.md b/docs/user.md index 295c149bd..6e71d0404 100644 --- a/docs/user.md +++ b/docs/user.md @@ -511,3 +511,50 @@ monitoring is outside the scope of operator responsibilities. See [configuration reference](reference/cluster_manifest.md) and [administrator documentation](administrator.md) for details on how backups are executed. + +## Custom TLS certificates + +By default, the spilo image generates its own TLS certificate during startup. +This certificate is not secure since it cannot be verified and thus doesn't +protect from active MITM attacks. In this section we show how a Kubernete +Secret resources can be loaded with a custom TLS certificate. + +Before applying these changes, the operator must also be configured with the +`spilo_fsgroup` set to the GID matching the postgres user group. If the value +is not provided, the cluster will default to `103` which is the GID from the +default spilo image. + +Upload the cert as a kubernetes secret: +```sh +kubectl create secret tls pg-tls \ + --key pg-tls.key \ + --cert pg-tls.crt +``` + +Or with a CA: +```sh +kubectl create secret generic pg-tls \ + --from-file=tls.crt=server.crt \ + --from-file=tls.key=server.key \ + --from-file=ca.crt=ca.crt +``` + +Alternatively it is also possible to use +[cert-manager](https://cert-manager.io/docs/) to generate these secrets. + +Then configure the postgres resource with the TLS secret: + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql + +metadata: + name: acid-test-cluster +spec: + tls: + secretName: "pg-tls" + caFile: "ca.crt" # add this if the secret is configured with a CA +``` + +Certificate rotation is handled in the spilo image which checks every 5 +minutes if the certificates have changed and reloads postgres accordingly. diff --git a/go.mod b/go.mod index 36686dcf6..be1ec32d4 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/lib/pq v1.2.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/sirupsen/logrus v1.4.2 + github.com/stretchr/testify v1.4.0 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect diff --git a/go.sum b/go.sum index f85dd060f..0737d3a5d 100644 --- a/go.sum +++ b/go.sum @@ -275,6 +275,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 5ae817ca3..5ba1fd1bc 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -100,3 +100,10 @@ spec: # env: # - name: "USEFUL_VAR" # value: "perhaps-true" + +# Custom TLS certificate. Disabled unless tls.secretName has a value. + tls: + secretName: "" # should correspond to a Kubernetes Secret resource to load + certificateFile: "tls.crt" + privateKeyFile: "tls.key" + caFile: "" # optionally configure Postgres with a CA certificate diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 453916b26..04c789fb9 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -251,6 +251,19 @@ spec: type: string teamId: type: string + tls: + type: object + required: + - secretName + properties: + secretName: + type: string + certificateFile: + type: string + privateKeyFile: + type: string + caFile: + type: string tolerations: type: array items: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 28dfa1566..eff47c255 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -417,6 +417,24 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "teamId": { Type: "string", }, + "tls": { + Type: "object", + Required: []string{"secretName"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "secretName": { + Type: "string", + }, + "certificateFile": { + Type: "string", + }, + "privateKeyFile": { + Type: "string", + }, + "caFile": { + Type: "string", + }, + }, + }, "tolerations": { Type: "array", Items: &apiextv1beta1.JSONSchemaPropsOrArray{ diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 07b42d4d4..862db6a4e 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -61,6 +61,7 @@ type PostgresSpec struct { StandbyCluster *StandbyDescription `json:"standby"` PodAnnotations map[string]string `json:"podAnnotations"` ServiceAnnotations map[string]string `json:"serviceAnnotations"` + TLS *TLSDescription `json:"tls"` // deprecated json tags InitContainersOld []v1.Container `json:"init_containers,omitempty"` @@ -126,6 +127,13 @@ type StandbyDescription struct { S3WalPath string `json:"s3_wal_path,omitempty"` } +type TLSDescription struct { + SecretName string `json:"secretName,omitempty"` + CertificateFile string `json:"certificateFile,omitempty"` + PrivateKeyFile string `json:"privateKeyFile,omitempty"` + CAFile string `json:"caFile,omitempty"` +} + // CloneDescription describes which cluster the new should clone and up to which point in time type CloneDescription struct { ClusterName string `json:"cluster,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index aaae1f04b..753b0490c 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -521,6 +521,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*out)[key] = val } } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(TLSDescription) + **out = **in + } if in.InitContainersOld != nil { in, out := &in.InitContainersOld, &out.InitContainersOld *out = make([]corev1.Container, len(*in)) @@ -752,6 +757,22 @@ func (in *StandbyDescription) DeepCopy() *StandbyDescription { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSDescription) DeepCopyInto(out *TLSDescription) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSDescription. +func (in *TLSDescription) DeepCopy() *TLSDescription { + if in == nil { + return nil + } + out := new(TLSDescription) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TeamsAPIConfiguration) DeepCopyInto(out *TeamsAPIConfiguration) { *out = *in diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index e2251a67c..84c407700 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -3,6 +3,7 @@ package cluster import ( "encoding/json" "fmt" + "path" "sort" "github.com/sirupsen/logrus" @@ -30,7 +31,10 @@ const ( patroniPGBinariesParameterName = "bin_dir" patroniPGParametersParameterName = "parameters" patroniPGHBAConfParameterName = "pg_hba" - localHost = "127.0.0.1/32" + + // the gid of the postgres user in the default spilo image + spiloPostgresGID = 103 + localHost = "127.0.0.1/32" ) type pgUser struct { @@ -446,6 +450,7 @@ func generatePodTemplate( podAntiAffinityTopologyKey string, additionalSecretMount string, additionalSecretMountPath string, + volumes []v1.Volume, ) (*v1.PodTemplateSpec, error) { terminateGracePeriodSeconds := terminateGracePeriod @@ -464,6 +469,7 @@ func generatePodTemplate( InitContainers: initContainers, Tolerations: *tolerationsSpec, SecurityContext: &securityContext, + Volumes: volumes, } if shmVolume != nil && *shmVolume { @@ -724,6 +730,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef sidecarContainers []v1.Container podTemplate *v1.PodTemplateSpec volumeClaimTemplate *v1.PersistentVolumeClaim + volumes []v1.Volume ) // Improve me. Please. @@ -840,21 +847,76 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef } // generate environment variables for the spilo container - spiloEnvVars := deduplicateEnvVars( - c.generateSpiloPodEnvVars(c.Postgresql.GetUID(), spiloConfiguration, &spec.Clone, - spec.StandbyCluster, customPodEnvVarsList), c.containerName(), c.logger) + spiloEnvVars := c.generateSpiloPodEnvVars( + c.Postgresql.GetUID(), + spiloConfiguration, + &spec.Clone, + spec.StandbyCluster, + customPodEnvVarsList, + ) // pickup the docker image for the spilo container effectiveDockerImage := util.Coalesce(spec.DockerImage, c.OpConfig.DockerImage) + // determine the FSGroup for the spilo pod + effectiveFSGroup := c.OpConfig.Resources.SpiloFSGroup + if spec.SpiloFSGroup != nil { + effectiveFSGroup = spec.SpiloFSGroup + } + volumeMounts := generateVolumeMounts(spec.Volume) + // configure TLS with a custom secret volume + if spec.TLS != nil && spec.TLS.SecretName != "" { + if effectiveFSGroup == nil { + c.logger.Warnf("Setting the default FSGroup to satisfy the TLS configuration") + fsGroup := int64(spiloPostgresGID) + effectiveFSGroup = &fsGroup + } + // this is combined with the FSGroup above to give read access to the + // postgres user + defaultMode := int32(0640) + volumes = append(volumes, v1.Volume{ + Name: "tls-secret", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: spec.TLS.SecretName, + DefaultMode: &defaultMode, + }, + }, + }) + + mountPath := "/tls" + volumeMounts = append(volumeMounts, v1.VolumeMount{ + MountPath: mountPath, + Name: "tls-secret", + ReadOnly: true, + }) + + // use the same filenames as Secret resources by default + certFile := ensurePath(spec.TLS.CertificateFile, mountPath, "tls.crt") + privateKeyFile := ensurePath(spec.TLS.PrivateKeyFile, mountPath, "tls.key") + spiloEnvVars = append( + spiloEnvVars, + v1.EnvVar{Name: "SSL_CERTIFICATE_FILE", Value: certFile}, + v1.EnvVar{Name: "SSL_PRIVATE_KEY_FILE", Value: privateKeyFile}, + ) + + if spec.TLS.CAFile != "" { + caFile := ensurePath(spec.TLS.CAFile, mountPath, "") + spiloEnvVars = append( + spiloEnvVars, + v1.EnvVar{Name: "SSL_CA_FILE", Value: caFile}, + ) + } + } + // generate the spilo container c.logger.Debugf("Generating Spilo container, environment variables: %v", spiloEnvVars) spiloContainer := generateContainer(c.containerName(), &effectiveDockerImage, resourceRequirements, - spiloEnvVars, + deduplicateEnvVars(spiloEnvVars, c.containerName(), c.logger), volumeMounts, c.OpConfig.Resources.SpiloPrivileged, ) @@ -893,16 +955,10 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef tolerationSpec := tolerations(&spec.Tolerations, c.OpConfig.PodToleration) effectivePodPriorityClassName := util.Coalesce(spec.PodPriorityClassName, c.OpConfig.PodPriorityClassName) - // determine the FSGroup for the spilo pod - effectiveFSGroup := c.OpConfig.Resources.SpiloFSGroup - if spec.SpiloFSGroup != nil { - effectiveFSGroup = spec.SpiloFSGroup - } - annotations := c.generatePodAnnotations(spec) // generate pod template for the statefulset, based on the spilo container and sidecars - if podTemplate, err = generatePodTemplate( + podTemplate, err = generatePodTemplate( c.Namespace, c.labelsSet(true), annotations, @@ -920,10 +976,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.OpConfig.EnablePodAntiAffinity, c.OpConfig.PodAntiAffinityTopologyKey, c.OpConfig.AdditionalSecretMount, - c.OpConfig.AdditionalSecretMountPath); err != nil { - return nil, fmt.Errorf("could not generate pod template: %v", err) - } - + c.OpConfig.AdditionalSecretMountPath, + volumes, + ) if err != nil { return nil, fmt.Errorf("could not generate pod template: %v", err) } @@ -1539,7 +1594,8 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { false, "", c.OpConfig.AdditionalSecretMount, - c.OpConfig.AdditionalSecretMountPath); err != nil { + c.OpConfig.AdditionalSecretMountPath, + nil); err != nil { return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) } @@ -1671,3 +1727,13 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { func (c *Cluster) getLogicalBackupJobName() (jobName string) { return "logical-backup-" + c.clusterName().Name } + +func ensurePath(file string, defaultDir string, defaultFile string) string { + if file == "" { + return path.Join(defaultDir, defaultFile) + } + if !path.IsAbs(file) { + return path.Join(defaultDir, file) + } + return file +} diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index e8fe05456..7fd4cd3e6 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -3,16 +3,17 @@ package cluster import ( "reflect" - v1 "k8s.io/api/core/v1" - "testing" + "github.com/stretchr/testify/assert" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + v1 "k8s.io/api/core/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -451,3 +452,65 @@ func TestSecretVolume(t *testing.T) { } } } + +func TestTLS(t *testing.T) { + var err error + var spec acidv1.PostgresSpec + var cluster *Cluster + + makeSpec := func(tls acidv1.TLSDescription) acidv1.PostgresSpec { + return acidv1.PostgresSpec{ + TeamID: "myapp", NumberOfInstances: 1, + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + TLS: &tls, + } + } + + cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + spec = makeSpec(acidv1.TLSDescription{SecretName: "my-secret", CAFile: "ca.crt"}) + s, err := cluster.generateStatefulSet(&spec) + if err != nil { + assert.NoError(t, err) + } + + fsGroup := int64(103) + assert.Equal(t, &fsGroup, s.Spec.Template.Spec.SecurityContext.FSGroup, "has a default FSGroup assigned") + + defaultMode := int32(0640) + volume := v1.Volume{ + Name: "tls-secret", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "my-secret", + DefaultMode: &defaultMode, + }, + }, + } + assert.Contains(t, s.Spec.Template.Spec.Volumes, volume, "the pod gets a secret volume") + + assert.Contains(t, s.Spec.Template.Spec.Containers[0].VolumeMounts, v1.VolumeMount{ + MountPath: "/tls", + Name: "tls-secret", + ReadOnly: true, + }, "the volume gets mounted in /tls") + + assert.Contains(t, s.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "SSL_CERTIFICATE_FILE", Value: "/tls/tls.crt"}) + assert.Contains(t, s.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "SSL_PRIVATE_KEY_FILE", Value: "/tls/tls.key"}) + assert.Contains(t, s.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "SSL_CA_FILE", Value: "/tls/ca.crt"}) +} From b66734a0a9a94147bc99bffa6844e146b036a7f7 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 13 Mar 2020 11:48:19 +0100 Subject: [PATCH 061/104] omit PgVersion diff on sync (#860) * use PostgresParam.PgVersion everywhere * on sync compare pgVersion with SpiloConfiguration * update getNewPgVersion and added tests --- kubectl-pg/cmd/list.go | 25 ++++--- pkg/cluster/cluster.go | 7 +- pkg/cluster/k8sres.go | 48 +++++++++++++- pkg/cluster/k8sres_test.go | 129 +++++++++++++++++++++++++++++++++++++ pkg/cluster/sync.go | 12 ++++ 5 files changed, 208 insertions(+), 13 deletions(-) diff --git a/kubectl-pg/cmd/list.go b/kubectl-pg/cmd/list.go index df827ffaf..f4dea882d 100644 --- a/kubectl-pg/cmd/list.go +++ b/kubectl-pg/cmd/list.go @@ -24,13 +24,14 @@ package cmd import ( "fmt" - "github.com/spf13/cobra" - "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "log" "strconv" "time" + + "github.com/spf13/cobra" + v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -95,8 +96,12 @@ func listAll(listPostgres *v1.PostgresqlList) { template := "%-32s%-16s%-12s%-12s%-12s%-12s%-12s\n" fmt.Printf(template, "NAME", "STATUS", "INSTANCES", "VERSION", "AGE", "VOLUME", "NAMESPACE") for _, pgObjs := range listPostgres.Items { - fmt.Printf(template, pgObjs.Name, pgObjs.Status.PostgresClusterStatus, strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)), - pgObjs.Spec.PgVersion, time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp), pgObjs.Spec.Size, pgObjs.Namespace) + fmt.Printf(template, pgObjs.Name, + pgObjs.Status.PostgresClusterStatus, + strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)), + pgObjs.Spec.PostgresqlParam.PgVersion, + time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp), + pgObjs.Spec.Size, pgObjs.Namespace) } } @@ -104,8 +109,12 @@ func listWithNamespace(listPostgres *v1.PostgresqlList) { template := "%-32s%-16s%-12s%-12s%-12s%-12s\n" fmt.Printf(template, "NAME", "STATUS", "INSTANCES", "VERSION", "AGE", "VOLUME") for _, pgObjs := range listPostgres.Items { - fmt.Printf(template, pgObjs.Name, pgObjs.Status.PostgresClusterStatus, strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)), - pgObjs.Spec.PgVersion, time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp), pgObjs.Spec.Size) + fmt.Printf(template, pgObjs.Name, + pgObjs.Status.PostgresClusterStatus, + strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)), + pgObjs.Spec.PostgresqlParam.PgVersion, + time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp), + pgObjs.Spec.Size) } } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 91e7a5195..d740260d2 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -554,10 +554,11 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } }() - if oldSpec.Spec.PgVersion != newSpec.Spec.PgVersion { // PG versions comparison - c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PgVersion, newSpec.Spec.PgVersion) + if oldSpec.Spec.PostgresqlParam.PgVersion != newSpec.Spec.PostgresqlParam.PgVersion { // PG versions comparison + c.logger.Warningf("postgresql version change(%q -> %q) has no effect", + oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) //we need that hack to generate statefulset with the old version - newSpec.Spec.PgVersion = oldSpec.Spec.PgVersion + newSpec.Spec.PostgresqlParam.PgVersion = oldSpec.Spec.PostgresqlParam.PgVersion } // Service diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 84c407700..aaa27384a 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -27,7 +27,7 @@ import ( ) const ( - pgBinariesLocationTemplate = "/usr/lib/postgresql/%s/bin" + pgBinariesLocationTemplate = "/usr/lib/postgresql/%v/bin" patroniPGBinariesParameterName = "bin_dir" patroniPGParametersParameterName = "parameters" patroniPGHBAConfParameterName = "pg_hba" @@ -722,6 +722,50 @@ func makeResources(cpuRequest, memoryRequest, cpuLimit, memoryLimit string) acid } } +func extractPgVersionFromBinPath(binPath string, template string) (string, error) { + var pgVersion float32 + _, err := fmt.Sscanf(binPath, template, &pgVersion) + if err != nil { + return "", err + } + return fmt.Sprintf("%v", pgVersion), nil +} + +func (c *Cluster) getNewPgVersion(container v1.Container, newPgVersion string) (string, error) { + var ( + spiloConfiguration spiloConfiguration + runningPgVersion string + err error + ) + + for _, env := range container.Env { + if env.Name != "SPILO_CONFIGURATION" { + continue + } + err = json.Unmarshal([]byte(env.Value), &spiloConfiguration) + if err != nil { + return newPgVersion, err + } + } + + if len(spiloConfiguration.PgLocalConfiguration) > 0 { + currentBinPath := fmt.Sprintf("%v", spiloConfiguration.PgLocalConfiguration[patroniPGBinariesParameterName]) + runningPgVersion, err = extractPgVersionFromBinPath(currentBinPath, pgBinariesLocationTemplate) + if err != nil { + return "", fmt.Errorf("could not extract Postgres version from %v in SPILO_CONFIGURATION", currentBinPath) + } + } else { + return "", fmt.Errorf("could not find %q setting in SPILO_CONFIGURATION", patroniPGBinariesParameterName) + } + + if runningPgVersion != newPgVersion { + c.logger.Warningf("postgresql version change(%q -> %q) has no effect", runningPgVersion, newPgVersion) + newPgVersion = runningPgVersion + } + + return newPgVersion, nil +} + func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.StatefulSet, error) { var ( @@ -1680,7 +1724,7 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { // Postgres env vars { Name: "PG_VERSION", - Value: c.Spec.PgVersion, + Value: c.Spec.PostgresqlParam.PgVersion, }, { Name: "PGPORT", diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 7fd4cd3e6..25e0f7af4 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -382,6 +382,135 @@ func TestCloneEnv(t *testing.T) { } } +func TestExtractPgVersionFromBinPath(t *testing.T) { + testName := "TestExtractPgVersionFromBinPath" + tests := []struct { + subTest string + binPath string + template string + expected string + }{ + { + subTest: "test current bin path with decimal against hard coded template", + binPath: "/usr/lib/postgresql/9.6/bin", + template: pgBinariesLocationTemplate, + expected: "9.6", + }, + { + subTest: "test current bin path against hard coded template", + binPath: "/usr/lib/postgresql/12/bin", + template: pgBinariesLocationTemplate, + expected: "12", + }, + { + subTest: "test alternative bin path against a matching template", + binPath: "/usr/pgsql-12/bin", + template: "/usr/pgsql-%v/bin", + expected: "12", + }, + } + + for _, tt := range tests { + pgVersion, err := extractPgVersionFromBinPath(tt.binPath, tt.template) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if pgVersion != tt.expected { + t.Errorf("%s %s: Expected version %s, have %s instead", + testName, tt.subTest, tt.expected, pgVersion) + } + } +} + +func TestGetPgVersion(t *testing.T) { + testName := "TestGetPgVersion" + tests := []struct { + subTest string + pgContainer v1.Container + currentPgVersion string + newPgVersion string + }{ + { + subTest: "new version with decimal point differs from current SPILO_CONFIGURATION", + pgContainer: v1.Container{ + Name: "postgres", + Env: []v1.EnvVar{ + { + Name: "SPILO_CONFIGURATION", + Value: "{\"postgresql\": {\"bin_dir\": \"/usr/lib/postgresql/9.6/bin\"}}", + }, + }, + }, + currentPgVersion: "9.6", + newPgVersion: "12", + }, + { + subTest: "new version differs from current SPILO_CONFIGURATION", + pgContainer: v1.Container{ + Name: "postgres", + Env: []v1.EnvVar{ + { + Name: "SPILO_CONFIGURATION", + Value: "{\"postgresql\": {\"bin_dir\": \"/usr/lib/postgresql/11/bin\"}}", + }, + }, + }, + currentPgVersion: "11", + newPgVersion: "12", + }, + { + subTest: "new version is lower than the one found in current SPILO_CONFIGURATION", + pgContainer: v1.Container{ + Name: "postgres", + Env: []v1.EnvVar{ + { + Name: "SPILO_CONFIGURATION", + Value: "{\"postgresql\": {\"bin_dir\": \"/usr/lib/postgresql/12/bin\"}}", + }, + }, + }, + currentPgVersion: "12", + newPgVersion: "11", + }, + { + subTest: "new version is the same like in the current SPILO_CONFIGURATION", + pgContainer: v1.Container{ + Name: "postgres", + Env: []v1.EnvVar{ + { + Name: "SPILO_CONFIGURATION", + Value: "{\"postgresql\": {\"bin_dir\": \"/usr/lib/postgresql/12/bin\"}}", + }, + }, + }, + currentPgVersion: "12", + newPgVersion: "12", + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + for _, tt := range tests { + pgVersion, err := cluster.getNewPgVersion(tt.pgContainer, tt.newPgVersion) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if pgVersion != tt.currentPgVersion { + t.Errorf("%s %s: Expected version %s, have %s instead", + testName, tt.subTest, tt.currentPgVersion, pgVersion) + } + } +} + func TestSecretVolume(t *testing.T) { testName := "TestSecretVolume" tests := []struct { diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 053db9ff7..b04ff863b 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -285,6 +285,18 @@ func (c *Cluster) syncStatefulSet() error { // statefulset is already there, make sure we use its definition in order to compare with the spec. c.Statefulset = sset + // check if there is no Postgres version mismatch + for _, container := range c.Statefulset.Spec.Template.Spec.Containers { + if container.Name != "postgres" { + continue + } + pgVersion, err := c.getNewPgVersion(container, c.Spec.PostgresqlParam.PgVersion) + if err != nil { + return fmt.Errorf("could not parse current Postgres version: %v", err) + } + c.Spec.PostgresqlParam.PgVersion = pgVersion + } + desiredSS, err := c.generateStatefulSet(&c.Spec) if err != nil { return fmt.Errorf("could not generate statefulset: %v", err) From d666c521726eaf3ceaba8ac24171f290d4277742 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Fri, 13 Mar 2020 11:51:39 +0100 Subject: [PATCH 062/104] ClusterDomain default (#863) * add json:omitempty option to ClusterDomain * Add default value for ClusterDomain Unfortunately, omitempty in operator configuration CRD doesn't mean that defauls from operator config object will be picked up automatically. Make sure that ClusterDomain default is specified, so that even when someone will set cluster_domain = "", it will be overwritted with a default value. Co-authored-by: mlu42 --- pkg/controller/operator_config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index c6f10faa0..03602c3bd 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -6,6 +6,7 @@ import ( "time" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -50,7 +51,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodTerminateGracePeriod = time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod) result.SpiloPrivileged = fromCRD.Kubernetes.SpiloPrivileged result.SpiloFSGroup = fromCRD.Kubernetes.SpiloFSGroup - result.ClusterDomain = fromCRD.Kubernetes.ClusterDomain + result.ClusterDomain = util.Coalesce(fromCRD.Kubernetes.ClusterDomain, "cluster.local") result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat result.EnablePodDisruptionBudget = fromCRD.Kubernetes.EnablePodDisruptionBudget From cf829df1a49ebe9a72e2bbc9c9289b1077ff4a48 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 17 Mar 2020 16:34:31 +0100 Subject: [PATCH 063/104] define ownership between operator and clusters via annotation (#802) * define ownership between operator and postgres clusters * add documentation * add unit test --- cmd/main.go | 2 +- docs/administrator.md | 28 ++++++++++ manifests/complete-postgres-manifest.yaml | 2 + manifests/postgres-operator.yaml | 3 ++ pkg/controller/controller.go | 19 ++++++- pkg/controller/node_test.go | 14 ++--- pkg/controller/postgresql.go | 62 +++++++++++++++-------- pkg/controller/postgresql_test.go | 58 +++++++++++++++++++-- pkg/controller/util_test.go | 14 ++--- pkg/util/constants/annotations.go | 1 + 10 files changed, 160 insertions(+), 43 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 7fadd611a..a178c187e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -77,7 +77,7 @@ func main() { log.Fatalf("couldn't get REST config: %v", err) } - c := controller.NewController(&config) + c := controller.NewController(&config, "") c.Run(stop, wg) diff --git a/docs/administrator.md b/docs/administrator.md index 9d877c783..a3a0f70cc 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -95,6 +95,34 @@ lacks access rights to any of them (except K8s system namespaces like 'list pods' execute at the cluster scope and fail at the first violation of access rights. +## Operators with defined ownership of certain Postgres clusters + +By default, multiple operators can only run together in one K8s cluster when +isolated into their [own namespaces](administrator.md#specify-the-namespace-to-watch). +But, it is also possible to define ownership between operator instances and +Postgres clusters running all in the same namespace or K8s cluster without +interfering. + +First, define the [`CONTROLLER_ID`](../../manifests/postgres-operator.yaml#L38) +environment variable in the operator deployment manifest. Then specify the ID +in every Postgres cluster manifest you want this operator to watch using the +`"acid.zalan.do/controller"` annotation: + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: demo-cluster + annotations: + "acid.zalan.do/controller": "second-operator" +spec: + ... +``` + +Every other Postgres cluster which lacks the annotation will be ignored by this +operator. Conversely, operators without a defined `CONTROLLER_ID` will ignore +clusters with defined ownership of another operator. + ## Role-based access control for the operator The manifest [`operator-service-account-rbac.yaml`](../manifests/operator-service-account-rbac.yaml) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 5ba1fd1bc..ceb27a5c3 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -4,6 +4,8 @@ metadata: name: acid-test-cluster # labels: # environment: demo +# annotations: +# "acid.zalan.do/controller": "second-operator" spec: dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 teamId: "acid" diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index 63f17d9fa..4b254822c 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -35,3 +35,6 @@ spec: # In order to use the CRD OperatorConfiguration instead, uncomment these lines and comment out the two lines above # - name: POSTGRES_OPERATOR_CONFIGURATION_OBJECT # value: postgresql-operator-default-configuration + # Define an ID to isolate controllers from each other + # - name: CONTROLLER_ID + # value: "second-operator" diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 140d2bc4e..0ce0d026e 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -13,6 +13,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/cache" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/apiserver" "github.com/zalando/postgres-operator/pkg/cluster" "github.com/zalando/postgres-operator/pkg/spec" @@ -36,6 +37,7 @@ type Controller struct { stopCh chan struct{} + controllerID string curWorkerID uint32 //initialized with 0 curWorkerCluster sync.Map clusterWorkers map[spec.NamespacedName]uint32 @@ -61,13 +63,14 @@ type Controller struct { } // NewController creates a new controller -func NewController(controllerConfig *spec.ControllerConfig) *Controller { +func NewController(controllerConfig *spec.ControllerConfig, controllerId string) *Controller { logger := logrus.New() c := &Controller{ config: *controllerConfig, opConfig: &config.Config{}, logger: logger.WithField("pkg", "controller"), + controllerID: controllerId, curWorkerCluster: sync.Map{}, clusterWorkers: make(map[spec.NamespacedName]uint32), clusters: make(map[spec.NamespacedName]*cluster.Cluster), @@ -239,6 +242,7 @@ func (c *Controller) initRoleBinding() { func (c *Controller) initController() { c.initClients() + c.controllerID = os.Getenv("CONTROLLER_ID") if configObjectName := os.Getenv("POSTGRES_OPERATOR_CONFIGURATION_OBJECT"); configObjectName != "" { if err := c.createConfigurationCRD(c.opConfig.EnableCRDValidation); err != nil { @@ -412,3 +416,16 @@ func (c *Controller) getEffectiveNamespace(namespaceFromEnvironment, namespaceFr return namespace } + +// hasOwnership returns true if the controller is the "owner" of the postgresql. +// Whether it's owner is determined by the value of 'acid.zalan.do/controller' +// annotation. If the value matches the controllerID then it owns it, or if the +// controllerID is "" and there's no annotation set. +func (c *Controller) hasOwnership(postgresql *acidv1.Postgresql) bool { + if postgresql.Annotations != nil { + if owner, ok := postgresql.Annotations[constants.PostgresqlControllerAnnotationKey]; ok { + return owner == c.controllerID + } + } + return c.controllerID == "" +} diff --git a/pkg/controller/node_test.go b/pkg/controller/node_test.go index c0ec78aa8..28e178bfb 100644 --- a/pkg/controller/node_test.go +++ b/pkg/controller/node_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/zalando/postgres-operator/pkg/spec" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -13,10 +13,10 @@ const ( readyValue = "ready" ) -func initializeController() *Controller { - var c = NewController(&spec.ControllerConfig{}) - c.opConfig.NodeReadinessLabel = map[string]string{readyLabel: readyValue} - return c +func newNodeTestController() *Controller { + var controller = NewController(&spec.ControllerConfig{}, "node-test") + controller.opConfig.NodeReadinessLabel = map[string]string{readyLabel: readyValue} + return controller } func makeNode(labels map[string]string, isSchedulable bool) *v1.Node { @@ -31,7 +31,7 @@ func makeNode(labels map[string]string, isSchedulable bool) *v1.Node { } } -var c = initializeController() +var nodeTestController = newNodeTestController() func TestNodeIsReady(t *testing.T) { testName := "TestNodeIsReady" @@ -57,7 +57,7 @@ func TestNodeIsReady(t *testing.T) { }, } for _, tt := range testTable { - if isReady := c.nodeIsReady(tt.in); isReady != tt.out { + if isReady := nodeTestController.nodeIsReady(tt.in); isReady != tt.out { t.Errorf("%s: expected response %t doesn't match the actual %t for the node %#v", testName, tt.out, isReady, tt.in) } diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 96d12bb9f..5d48bac39 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -40,12 +40,23 @@ func (c *Controller) clusterResync(stopCh <-chan struct{}, wg *sync.WaitGroup) { // clusterListFunc obtains a list of all PostgreSQL clusters func (c *Controller) listClusters(options metav1.ListOptions) (*acidv1.PostgresqlList, error) { + var pgList acidv1.PostgresqlList + // TODO: use the SharedInformer cache instead of quering Kubernetes API directly. list, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.opConfig.WatchedNamespace).List(options) if err != nil { c.logger.Errorf("could not list postgresql objects: %v", err) } - return list, err + if c.controllerID != "" { + c.logger.Debugf("watch only clusters with controllerID %q", c.controllerID) + } + for _, pg := range list.Items { + if pg.Error == "" && c.hasOwnership(&pg) { + pgList.Items = append(pgList.Items, pg) + } + } + + return &pgList, err } // clusterListAndSync lists all manifests and decides whether to run the sync or repair. @@ -455,41 +466,48 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. } func (c *Controller) postgresqlAdd(obj interface{}) { - pg, ok := obj.(*acidv1.Postgresql) - if !ok { - c.logger.Errorf("could not cast to postgresql spec") - return + pg := c.postgresqlCheck(obj) + if pg != nil { + // We will not get multiple Add events for the same cluster + c.queueClusterEvent(nil, pg, EventAdd) } - // We will not get multiple Add events for the same cluster - c.queueClusterEvent(nil, pg, EventAdd) + return } func (c *Controller) postgresqlUpdate(prev, cur interface{}) { - pgOld, ok := prev.(*acidv1.Postgresql) - if !ok { - c.logger.Errorf("could not cast to postgresql spec") - } - pgNew, ok := cur.(*acidv1.Postgresql) - if !ok { - c.logger.Errorf("could not cast to postgresql spec") - } - // Avoid the inifinite recursion for status updates - if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) { - return + pgOld := c.postgresqlCheck(prev) + pgNew := c.postgresqlCheck(cur) + if pgOld != nil && pgNew != nil { + // Avoid the inifinite recursion for status updates + if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) { + return + } + c.queueClusterEvent(pgOld, pgNew, EventUpdate) } - c.queueClusterEvent(pgOld, pgNew, EventUpdate) + return } func (c *Controller) postgresqlDelete(obj interface{}) { + pg := c.postgresqlCheck(obj) + if pg != nil { + c.queueClusterEvent(pg, nil, EventDelete) + } + + return +} + +func (c *Controller) postgresqlCheck(obj interface{}) *acidv1.Postgresql { pg, ok := obj.(*acidv1.Postgresql) if !ok { c.logger.Errorf("could not cast to postgresql spec") - return + return nil } - - c.queueClusterEvent(pg, nil, EventDelete) + if !c.hasOwnership(pg) { + return nil + } + return pg } /* diff --git a/pkg/controller/postgresql_test.go b/pkg/controller/postgresql_test.go index 3d7785f92..b36519c5a 100644 --- a/pkg/controller/postgresql_test.go +++ b/pkg/controller/postgresql_test.go @@ -1,10 +1,12 @@ package controller import ( - acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - "github.com/zalando/postgres-operator/pkg/spec" "reflect" "testing" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/spec" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var ( @@ -12,9 +14,55 @@ var ( False = false ) -func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) { - c := NewController(&spec.ControllerConfig{}) +func newPostgresqlTestController() *Controller { + controller := NewController(&spec.ControllerConfig{}, "postgresql-test") + return controller +} +var postgresqlTestController = newPostgresqlTestController() + +func TestControllerOwnershipOnPostgresql(t *testing.T) { + tests := []struct { + name string + pg *acidv1.Postgresql + owned bool + error string + }{ + { + "Postgres cluster with defined ownership of mocked controller", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"acid.zalan.do/controller": "postgresql-test"}, + }, + }, + True, + "Postgres cluster should be owned by operator, but controller says no", + }, + { + "Postgres cluster with defined ownership of another controller", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"acid.zalan.do/controller": "stups-test"}, + }, + }, + False, + "Postgres cluster should be owned by another operator, but controller say yes", + }, + { + "Test Postgres cluster without defined ownership", + &acidv1.Postgresql{}, + False, + "Postgres cluster should be owned by operator with empty controller ID, but controller says yes", + }, + } + for _, tt := range tests { + if postgresqlTestController.hasOwnership(tt.pg) != tt.owned { + t.Errorf("%s: %v", tt.name, tt.error) + } + } +} + +func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) { tests := []struct { name string in *acidv1.PostgresSpec @@ -36,7 +84,7 @@ func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) { }, } for _, tt := range tests { - result := c.mergeDeprecatedPostgreSQLSpecParameters(tt.in) + result := postgresqlTestController.mergeDeprecatedPostgreSQLSpecParameters(tt.in) if !reflect.DeepEqual(result, tt.out) { t.Errorf("%s: %v", tt.name, tt.error) } diff --git a/pkg/controller/util_test.go b/pkg/controller/util_test.go index a5d3c7ac5..ef182248e 100644 --- a/pkg/controller/util_test.go +++ b/pkg/controller/util_test.go @@ -17,8 +17,8 @@ const ( testInfrastructureRolesSecretName = "infrastructureroles-test" ) -func newMockController() *Controller { - controller := NewController(&spec.ControllerConfig{}) +func newUtilTestController() *Controller { + controller := NewController(&spec.ControllerConfig{}, "util-test") controller.opConfig.ClusterNameLabel = "cluster-name" controller.opConfig.InfrastructureRolesSecretName = spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesSecretName} @@ -27,7 +27,7 @@ func newMockController() *Controller { return controller } -var mockController = newMockController() +var utilTestController = newUtilTestController() func TestPodClusterName(t *testing.T) { var testTable = []struct { @@ -43,7 +43,7 @@ func TestPodClusterName(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Namespace: v1.NamespaceDefault, Labels: map[string]string{ - mockController.opConfig.ClusterNameLabel: "testcluster", + utilTestController.opConfig.ClusterNameLabel: "testcluster", }, }, }, @@ -51,7 +51,7 @@ func TestPodClusterName(t *testing.T) { }, } for _, test := range testTable { - resp := mockController.podClusterName(test.in) + resp := utilTestController.podClusterName(test.in) if resp != test.expected { t.Errorf("expected response %v does not match the actual %v", test.expected, resp) } @@ -73,7 +73,7 @@ func TestClusterWorkerID(t *testing.T) { }, } for _, test := range testTable { - resp := mockController.clusterWorkerID(test.in) + resp := utilTestController.clusterWorkerID(test.in) if resp != test.expected { t.Errorf("expected response %v does not match the actual %v", test.expected, resp) } @@ -116,7 +116,7 @@ func TestGetInfrastructureRoles(t *testing.T) { }, } for _, test := range testTable { - roles, err := mockController.getInfrastructureRoles(&test.secretName) + roles, err := utilTestController.getInfrastructureRoles(&test.secretName) if err != test.expectedError { if err != nil && test.expectedError != nil && err.Error() == test.expectedError.Error() { continue diff --git a/pkg/util/constants/annotations.go b/pkg/util/constants/annotations.go index 0b93fc2e1..fc5a84fa5 100644 --- a/pkg/util/constants/annotations.go +++ b/pkg/util/constants/annotations.go @@ -7,4 +7,5 @@ const ( ElbTimeoutAnnotationValue = "3600" KubeIAmAnnotation = "iam.amazonaws.com/role" VolumeStorateProvisionerAnnotation = "pv.kubernetes.io/provisioned-by" + PostgresqlControllerAnnotationKey = "acid.zalan.do/controller" ) From 9ddee8f3029ed086d2123d678bcb0147745edafd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20=C3=98strem?= Date: Wed, 18 Mar 2020 10:28:39 +0100 Subject: [PATCH 064/104] Use cryptographically secure password generation (#854) The current password generation algorithm is extremely deterministic, due to being based on the standard random number generator with a deterministic seed based on the current Unix timestamp (in seconds). This can lead to a number of security issues, including: The same passwords being used in different Kubernetes clusters if the operator is deployed in parallel. (This issue was discovered because of four deployments having the same generated passwords due to automatically being deployed in parallel.) The passwords being easily guessable based on the time the operator pod started when the database was created. (This would typically be present in logs, metrics, etc., that may typically be accessible to more people than should have database access.) Fix this issue by replacing the current randomness source with crypto/rand, which should produce cryptographically secure random data that is virtually unguessable. This will avoid both of the above problems as each deployment will be guaranteed to have unique, indeterministic passwords. --- pkg/util/util.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index ad6de14a2..d9803ab48 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -2,8 +2,10 @@ package util import ( "crypto/md5" // #nosec we need it to for PostgreSQL md5 passwords + cryptoRand "crypto/rand" "encoding/hex" "fmt" + "math/big" "math/rand" "regexp" "strings" @@ -37,13 +39,17 @@ func False() *bool { return &b } -// RandomPassword generates random alphanumeric password of a given length. +// RandomPassword generates a secure, random alphanumeric password of a given length. func RandomPassword(n int) string { b := make([]byte, n) for i := range b { - b[i] = passwordChars[rand.Intn(len(passwordChars))] + maxN := big.NewInt(int64(len(passwordChars))) + if n, err := cryptoRand.Int(cryptoRand.Reader, maxN); err != nil { + panic(fmt.Errorf("Unable to generate secure, random password: %v", err)) + } else { + b[i] = passwordChars[n.Int64()] + } } - return string(b) } From 07c5da35e3572b8302f33e3dd8fbf5e8ca846229 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 18 Mar 2020 15:02:13 +0100 Subject: [PATCH 065/104] fix minor issues in docs and manifests (#866) * fix minor issues in docs and manifests * double retry_timeout_sec --- docs/reference/cluster_manifest.md | 20 ++++++++++---------- docs/reference/operator_parameters.md | 10 +++++----- docs/user.md | 8 ++++---- e2e/tests/test_e2e.py | 2 +- manifests/complete-postgres-manifest.yaml | 2 +- manifests/minimal-postgres-manifest.yaml | 2 +- manifests/standby-manifest.yaml | 2 +- ui/manifests/deployment.yaml | 2 +- 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 92e457d7e..955622843 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -111,12 +111,12 @@ These parameters are grouped directly under the `spec` key in the manifest. value overrides the `pod_toleration` setting from the operator. Optional. * **podPriorityClassName** - a name of the [priority - class](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass) - that should be assigned to the cluster pods. When not specified, the value - is taken from the `pod_priority_class_name` operator parameter, if not set - then the default priority class is taken. The priority class itself must be - defined in advance. Optional. + a name of the [priority + class](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass) + that should be assigned to the cluster pods. When not specified, the value + is taken from the `pod_priority_class_name` operator parameter, if not set + then the default priority class is taken. The priority class itself must be + defined in advance. Optional. * **podAnnotations** A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) @@ -184,9 +184,9 @@ explanation of `ttl` and `loop_wait` parameters. ``` hostssl all +pamrole all pam ``` - , where pamrole is the name of the role for the pam authentication; any - custom `pg_hba` should include the pam line to avoid breaking pam - authentication. Optional. + where pamrole is the name of the role for the pam authentication; any + custom `pg_hba` should include the pam line to avoid breaking pam + authentication. Optional. * **ttl** Patroni `ttl` parameter value, optional. The default is set by the Spilo @@ -379,4 +379,4 @@ Those parameters are grouped under the `tls` top-level key. * **caFile** Optional filename to the CA certificate. Useful when the client connects - with `sslmode=verify-ca` or `sslmode=verify-full`. + with `sslmode=verify-ca` or `sslmode=verify-full`. Default is empty. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index ba8e73cf8..86eedd33c 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -285,11 +285,11 @@ configuration they are grouped under the `kubernetes` key. capability. The default is `false`. * **master_pod_move_timeout** - The period of time to wait for the success of migration of master pods from - an unschedulable node. The migration includes Patroni switchovers to - respective replicas on healthy nodes. The situation where master pods still - exist on the old node after this timeout expires has to be fixed manually. - The default is 20 minutes. + The period of time to wait for the success of migration of master pods from + an unschedulable node. The migration includes Patroni switchovers to + respective replicas on healthy nodes. The situation where master pods still + exist on the old node after this timeout expires has to be fixed manually. + The default is 20 minutes. * **enable_pod_antiaffinity** toggles [pod anti affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) diff --git a/docs/user.md b/docs/user.md index 6e71d0404..9a6752185 100644 --- a/docs/user.md +++ b/docs/user.md @@ -30,7 +30,7 @@ spec: databases: foo: zalando postgresql: - version: "11" + version: "12" ``` Once you cloned the Postgres Operator [repository](https://github.com/zalando/postgres-operator) @@ -515,9 +515,9 @@ executed. ## Custom TLS certificates By default, the spilo image generates its own TLS certificate during startup. -This certificate is not secure since it cannot be verified and thus doesn't -protect from active MITM attacks. In this section we show how a Kubernete -Secret resources can be loaded with a custom TLS certificate. +However, this certificate cannot be verified and thus doesn't protect from +active MITM attacks. In this section we show how to specify a custom TLS +certificate which is mounted in the database pods via a K8s Secret. Before applying these changes, the operator must also be configured with the `spilo_fsgroup` set to the GID matching the postgres user group. If the value diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index f6be8a600..f0d8a0b23 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -473,7 +473,7 @@ class K8s: Wraps around K8 api client and helper methods. ''' - RETRY_TIMEOUT_SEC = 5 + RETRY_TIMEOUT_SEC = 10 def __init__(self): self.api = K8sApi() diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index ceb27a5c3..c82f1eac5 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -24,7 +24,7 @@ spec: databases: foo: zalando postgresql: - version: "11" + version: "12" parameters: # Expert section shared_buffers: "32MB" max_connections: "10" diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml index 75dfdf07f..af0add8e6 100644 --- a/manifests/minimal-postgres-manifest.yaml +++ b/manifests/minimal-postgres-manifest.yaml @@ -16,4 +16,4 @@ spec: databases: foo: zalando # dbname: owner postgresql: - version: "11" + version: "12" diff --git a/manifests/standby-manifest.yaml b/manifests/standby-manifest.yaml index 2b621bd10..4c8d09650 100644 --- a/manifests/standby-manifest.yaml +++ b/manifests/standby-manifest.yaml @@ -9,7 +9,7 @@ spec: size: 1Gi numberOfInstances: 1 postgresql: - version: "11" + version: "12" # Make this a standby cluster and provide the s3 bucket path of source cluster for continuous streaming. standby: s3_wal_path: "s3://path/to/bucket/containing/wal/of/source/cluster/" diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index 477e4d655..6138ca1a8 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: postgres-operator-ui containers: - name: "service" - image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.3.0 + image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.4.0 ports: - containerPort: 8081 protocol: "TCP" From cc1ffdc7b6ed83ef2d1be575db8e4aa76f8c6e57 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 25 Mar 2020 09:31:30 +0100 Subject: [PATCH 066/104] enable controllerID for chart and allow configurable pod cluster role (#876) --- charts/postgres-operator/templates/_helpers.tpl | 14 ++++++++++++++ .../templates/clusterrole-postgres-pod.yaml | 2 +- .../postgres-operator/templates/configmap.yaml | 1 + .../postgres-operator/templates/deployment.yaml | 4 ++++ .../templates/operatorconfiguration.yaml | 1 + charts/postgres-operator/values-crd.yaml | 16 ++++++++++++++-- charts/postgres-operator/values.yaml | 16 ++++++++++++++-- 7 files changed, 49 insertions(+), 5 deletions(-) diff --git a/charts/postgres-operator/templates/_helpers.tpl b/charts/postgres-operator/templates/_helpers.tpl index 306613ac3..e49670763 100644 --- a/charts/postgres-operator/templates/_helpers.tpl +++ b/charts/postgres-operator/templates/_helpers.tpl @@ -31,6 +31,20 @@ Create a service account name. {{ default (include "postgres-operator.fullname" .) .Values.serviceAccount.name }} {{- end -}} +{{/* +Create a pod service account name. +*/}} +{{- define "postgres-pod.serviceAccountName" -}} +{{ default (printf "%s-%v" (include "postgres-operator.fullname" .) "pod") .Values.podServiceAccount.name }} +{{- end -}} + +{{/* +Create a controller ID. +*/}} +{{- define "postgres-operator.controllerID" -}} +{{ default (include "postgres-operator.fullname" .) .Values.controllerID.name }} +{{- end -}} + {{/* Create chart name and version as used by the chart label. */}} diff --git a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml index c327d9101..ef607ae3c 100644 --- a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml +++ b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml @@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: postgres-pod + name: {{ include "postgres-pod.serviceAccountName" . }} labels: app.kubernetes.io/name: {{ template "postgres-operator.name" . }} helm.sh/chart: {{ template "postgres-operator.chart" . }} diff --git a/charts/postgres-operator/templates/configmap.yaml b/charts/postgres-operator/templates/configmap.yaml index 0b976294e..00ebc6676 100644 --- a/charts/postgres-operator/templates/configmap.yaml +++ b/charts/postgres-operator/templates/configmap.yaml @@ -9,6 +9,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} data: + pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }} {{ toYaml .Values.configGeneral | indent 2 }} {{ toYaml .Values.configUsers | indent 2 }} {{ toYaml .Values.configKubernetes | indent 2 }} diff --git a/charts/postgres-operator/templates/deployment.yaml b/charts/postgres-operator/templates/deployment.yaml index 1f7e39bbc..2d8eebcb3 100644 --- a/charts/postgres-operator/templates/deployment.yaml +++ b/charts/postgres-operator/templates/deployment.yaml @@ -43,6 +43,10 @@ spec: {{- else }} - name: POSTGRES_OPERATOR_CONFIGURATION_OBJECT value: {{ template "postgres-operator.fullname" . }} + {{- end }} + {{- if .Values.controllerID.create }} + - name: CONTROLLER_ID + value: {{ template "postgres-operator.controllerID" . }} {{- end }} resources: {{ toYaml .Values.resources | indent 10 }} diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index 06e9c7605..ccbbf59c6 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -13,6 +13,7 @@ configuration: users: {{ toYaml .Values.configUsers | indent 4 }} kubernetes: + pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }} oauth_token_secret_name: {{ template "postgres-operator.fullname" . }} {{ toYaml .Values.configKubernetes | indent 4 }} postgres_pod_resources: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index d170e0b77..f9c20d8d6 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -103,8 +103,6 @@ configKubernetes: # service account definition as JSON/YAML string to be used by postgres cluster pods # pod_service_account_definition: "" - # name of service account to be used by postgres cluster pods - pod_service_account_name: "postgres-pod" # role binding definition as JSON/YAML string to be used by pod service account # pod_service_account_role_binding_definition: "" @@ -284,6 +282,11 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: +podServiceAccount: + # The name of the ServiceAccount to be used by postgres cluster pods + # If not set a name is generated using the fullname template and "-pod" suffix + name: "postgres-pod" + priorityClassName: "" resources: @@ -305,3 +308,12 @@ tolerations: [] # Node labels for pod assignment # Ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: {} + +controllerID: + # Specifies whether a controller ID should be defined for the operator + # Note, all postgres manifest must then contain the following annotation to be found by this operator + # "acid.zalan.do/controller": + create: false + # The name of the controller ID to use. + # If not set and create is true, a name is generated using the fullname template + name: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index b6f18f305..ea1028f23 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -96,8 +96,6 @@ configKubernetes: # service account definition as JSON/YAML string to be used by postgres cluster pods # pod_service_account_definition: "" - # name of service account to be used by postgres cluster pods - pod_service_account_name: "postgres-pod" # role binding definition as JSON/YAML string to be used by pod service account # pod_service_account_role_binding_definition: "" @@ -260,6 +258,11 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: +podServiceAccount: + # The name of the ServiceAccount to be used by postgres cluster pods + # If not set a name is generated using the fullname template and "-pod" suffix + name: "postgres-pod" + priorityClassName: "" resources: @@ -281,3 +284,12 @@ tolerations: [] # Node labels for pod assignment # Ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: {} + +controllerID: + # Specifies whether a controller ID should be defined for the operator + # Note, all postgres manifest must then contain the following annotation to be found by this operator + # "acid.zalan.do/controller": + create: false + # The name of the controller ID to use. + # If not set and create is true, a name is generated using the fullname template + name: From 579f78864bbe67ad4068e02967c272d6c68c19a9 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 25 Mar 2020 09:59:54 +0100 Subject: [PATCH 067/104] pass cluster labels as JSON to Spilo (#877) --- pkg/cluster/k8sres.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index aaa27384a..39b5c16a0 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -549,10 +549,6 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri Name: "KUBERNETES_ROLE_LABEL", Value: c.OpConfig.PodRoleLabel, }, - { - Name: "KUBERNETES_LABELS", - Value: labels.Set(c.OpConfig.ClusterLabels).String(), - }, { Name: "PGPASSWORD_SUPERUSER", ValueFrom: &v1.EnvVarSource{ @@ -588,6 +584,12 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri Value: c.OpConfig.PamRoleName, }, } + // Spilo expects cluster labels as JSON + if clusterLabels, err := json.Marshal(labels.Set(c.OpConfig.ClusterLabels)); err != nil { + envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_LABELS", Value: labels.Set(c.OpConfig.ClusterLabels).String()}) + } else { + envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_LABELS", Value: string(clusterLabels)}) + } if spiloConfiguration != "" { envVars = append(envVars, v1.EnvVar{Name: "SPILO_CONFIGURATION", Value: spiloConfiguration}) } From 9dfa433363cbe997baf69a99c7046166fa966fa3 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Wed, 25 Mar 2020 12:57:26 +0100 Subject: [PATCH 068/104] Connection pooler (#799) Connection pooler support Add support for a connection pooler. The idea is to make it generic enough to be able to switch between different implementations (e.g. pgbouncer or odyssey). Operator needs to create a deployment with pooler and a service for it to access. For connection pool to work properly, a database needs to be prepared by operator, namely a separate user have to be created with an access to an installed lookup function (to fetch credential for other users). This setups is supposed to be used only by robot/application users. Usually a connection pool implementation is more CPU bounded, so it makes sense to create several pods for connection pool with more emphasize on cpu resources. At the moment there are no special affinity or tolerations assigned to bring those pods closer to the database. For availability purposes minimal number of connection pool pods is 2, ideally they have to be distributed between different nodes/AZ, but it's not enforced in the operator itself. Available configuration supposed to be ergonomic and in the normal case require minimum changes to a manifest to enable connection pool. To have more control over the configuration and functionality on the pool side one can customize the corresponding docker image. Co-authored-by: Felix Kunde --- .../crds/operatorconfigurations.yaml | 41 ++ .../postgres-operator/crds/postgresqls.yaml | 51 +++ .../templates/clusterrole.yaml | 1 + .../templates/configmap.yaml | 1 + .../templates/operatorconfiguration.yaml | 2 + charts/postgres-operator/values-crd.yaml | 19 + charts/postgres-operator/values.yaml | 20 + docker/DebugDockerfile | 13 +- docs/reference/cluster_manifest.md | 34 ++ docs/reference/operator_parameters.md | 37 ++ docs/user.md | 53 +++ e2e/tests/test_e2e.py | 165 +++++++- manifests/configmap.yaml | 10 + manifests/operator-service-account-rbac.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 41 ++ ...gresql-operator-default-configuration.yaml | 11 + manifests/postgresql.crd.yaml | 51 +++ pkg/apis/acid.zalan.do/v1/crds.go | 124 +++++- .../v1/operator_configuration_type.go | 29 +- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 26 ++ .../acid.zalan.do/v1/zz_generated.deepcopy.go | 64 +++ pkg/cluster/cluster.go | 205 +++++++++- pkg/cluster/cluster_test.go | 18 + pkg/cluster/database.go | 125 +++++- pkg/cluster/k8sres.go | 380 +++++++++++++++++- pkg/cluster/k8sres_test.go | 372 +++++++++++++++++ pkg/cluster/resources.go | 157 ++++++++ pkg/cluster/resources_test.go | 127 ++++++ pkg/cluster/sync.go | 182 ++++++++- pkg/cluster/sync_test.go | 212 ++++++++++ pkg/cluster/types.go | 4 + pkg/cluster/util.go | 39 +- pkg/controller/operator_config.go | 51 +++ pkg/spec/types.go | 6 +- pkg/util/config/config.go | 21 + pkg/util/constants/pooler.go | 18 + pkg/util/constants/roles.go | 23 +- pkg/util/k8sutil/k8sutil.go | 170 +++++++- pkg/util/util.go | 34 ++ 39 files changed, 2885 insertions(+), 53 deletions(-) create mode 100644 pkg/cluster/resources_test.go create mode 100644 pkg/cluster/sync_test.go create mode 100644 pkg/util/constants/pooler.go diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 9725c2708..7e3b607c0 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -318,6 +318,47 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string + connection_pool: + type: object + properties: + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" + connection_pool_image: + type: string + #default: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pool_max_db_connections: + type: integer + #default: 60 + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pool_number_of_instances: + type: integer + minimum: 2 + #default: 2 + connection_pool_default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pool_default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "500m" + connection_pool_default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100Mi" + connection_pool_default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100Mi" status: type: object additionalProperties: diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index af535e2c8..a4c0e4f3a 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -106,6 +106,55 @@ spec: uid: format: uuid type: string + connectionPool: + type: object + properties: + dockerImage: + type: string + maxDBConnections: + type: integer + mode: + type: string + enum: + - "session" + - "transaction" + numberOfInstances: + type: integer + minimum: 2 + resources: + type: object + required: + - requests + - limits + properties: + limits: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + schema: + type: string + user: + type: string databases: type: object additionalProperties: @@ -113,6 +162,8 @@ spec: # Note: usernames specified here as database owners must be declared in the users key of the spec key. dockerImage: type: string + enableConnectionPool: + type: boolean enableLogicalBackup: type: boolean enableMasterLoadBalancer: diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 7b3dd462d..38ce85e7a 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -128,6 +128,7 @@ rules: - apps resources: - statefulsets + - deployments verbs: - create - delete diff --git a/charts/postgres-operator/templates/configmap.yaml b/charts/postgres-operator/templates/configmap.yaml index 00ebc6676..e8a805db7 100644 --- a/charts/postgres-operator/templates/configmap.yaml +++ b/charts/postgres-operator/templates/configmap.yaml @@ -20,4 +20,5 @@ data: {{ toYaml .Values.configDebug | indent 2 }} {{ toYaml .Values.configLoggingRestApi | indent 2 }} {{ toYaml .Values.configTeamsApi | indent 2 }} +{{ toYaml .Values.configConnectionPool | indent 2 }} {{- end }} diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index ccbbf59c6..b52b3d664 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -34,4 +34,6 @@ configuration: {{ toYaml .Values.configLoggingRestApi | indent 4 }} scalyr: {{ toYaml .Values.configScalyr | indent 4 }} + connection_pool: +{{ toYaml .Values.configConnectionPool | indent 4 }} {{- end }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index f9c20d8d6..cf9fbab15 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -267,6 +267,25 @@ configScalyr: # Memory request value for the Scalyr sidecar scalyr_memory_request: 50Mi +configConnectionPool: + # db schema to install lookup function into + connection_pool_schema: "pooler" + # db user for pooler to use + connection_pool_user: "pooler" + # docker image + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + # max db connections the pooler should hold + connection_pool_max_db_connections: 60 + # default pooling mode + connection_pool_mode: "transaction" + # number of pooler instances + connection_pool_number_of_instances: 2 + # default resources + connection_pool_default_cpu_request: 500m + connection_pool_default_memory_request: 100Mi + connection_pool_default_cpu_limit: "1" + connection_pool_default_memory_limit: 100Mi + rbac: # Specifies whether RBAC resources should be created create: true diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index ea1028f23..503bf4562 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -243,6 +243,26 @@ configTeamsApi: # URL of the Teams API service # teams_api_url: http://fake-teams-api.default.svc.cluster.local +# configure connection pooler deployment created by the operator +configConnectionPool: + # db schema to install lookup function into + connection_pool_schema: "pooler" + # db user for pooler to use + connection_pool_user: "pooler" + # docker image + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + # max db connections the pooler should hold + connection_pool_max_db_connections: 60 + # default pooling mode + connection_pool_mode: "transaction" + # number of pooler instances + connection_pool_number_of_instances: 2 + # default resources + connection_pool_default_cpu_request: 500m + connection_pool_default_memory_request: 100Mi + connection_pool_default_cpu_limit: "1" + connection_pool_default_memory_limit: 100Mi + rbac: # Specifies whether RBAC resources should be created create: true diff --git a/docker/DebugDockerfile b/docker/DebugDockerfile index 76dadf6df..0c11fe3b4 100644 --- a/docker/DebugDockerfile +++ b/docker/DebugDockerfile @@ -3,8 +3,17 @@ MAINTAINER Team ACID @ Zalando # We need root certificates to deal with teams api over https RUN apk --no-cache add ca-certificates go git musl-dev -RUN go get github.com/derekparker/delve/cmd/dlv COPY build/* / -CMD ["/root/go/bin/dlv", "--listen=:7777", "--headless=true", "--api-version=2", "exec", "/postgres-operator"] +RUN addgroup -g 1000 pgo +RUN adduser -D -u 1000 -G pgo -g 'Postgres Operator' pgo + +RUN go get github.com/derekparker/delve/cmd/dlv +RUN cp /root/go/bin/dlv /dlv +RUN chown -R pgo:pgo /dlv + +USER pgo:pgo +RUN ls -l / + +CMD ["/dlv", "--listen=:7777", "--headless=true", "--api-version=2", "exec", "/postgres-operator"] diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 955622843..4400cb666 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -140,6 +140,11 @@ These parameters are grouped directly under the `spec` key in the manifest. is `false`, then no volume will be mounted no matter how operator was configured (so you can override the operator configuration). Optional. +* **enableConnectionPool** + Tells the operator to create a connection pool with a database. If this + field is true, a connection pool deployment will be created even if + `connectionPool` section is empty. Optional, not set by default. + * **enableLogicalBackup** Determines if the logical backup of this cluster should be taken and uploaded to S3. Default: false. Optional. @@ -360,6 +365,35 @@ CPU and memory limits for the sidecar container. memory limits for the sidecar container. Optional, overrides the `default_memory_limits` operator configuration parameter. Optional. +## Connection pool + +Parameters are grouped under the `connectionPool` top-level key and specify +configuration for connection pool. If this section is not empty, a connection +pool will be created for a database even if `enableConnectionPool` is not +present. + +* **numberOfInstances** + How many instances of connection pool to create. + +* **schema** + Schema to create for credentials lookup function. + +* **user** + User to create for connection pool to be able to connect to a database. + +* **dockerImage** + Which docker image to use for connection pool deployment. + +* **maxDBConnections** + How many connections the pooler can max hold. This value is divided among the + pooler pods. + +* **mode** + In which mode to run connection pool, transaction or session. + +* **resources** + Resource configuration for connection pool deployment. + ## Custom TLS certificates Those parameters are grouped under the `tls` top-level key. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 86eedd33c..848fa1cf2 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -595,3 +595,40 @@ scalyr sidecar. In the CRD-based configuration they are grouped under the * **scalyr_memory_limit** Memory limit value for the Scalyr sidecar. The default is `500Mi`. + +## Connection pool configuration + +Parameters are grouped under the `connection_pool` top-level key and specify +default configuration for connection pool, if a postgres manifest requests it +but do not specify some of the parameters. All of them are optional with the +operator being able to provide some reasonable defaults. + +* **connection_pool_number_of_instances** + How many instances of connection pool to create. Default is 2 which is also + the required minimum. + +* **connection_pool_schema** + Schema to create for credentials lookup function. Default is `pooler`. + +* **connection_pool_user** + User to create for connection pool to be able to connect to a database. + Default is `pooler`. + +* **connection_pool_image** + Docker image to use for connection pool deployment. + Default: "registry.opensource.zalan.do/acid/pgbouncer" + +* **connection_pool_max_db_connections** + How many connections the pooler can max hold. This value is divided among the + pooler pods. Default is 60 which will make up 30 connections per pod for the + default setup with two instances. + +* **connection_pool_mode** + Default pool mode, `session` or `transaction`. Default is `transaction`. + +* **connection_pool_default_cpu_request** + **connection_pool_default_memory_reques** + **connection_pool_default_cpu_limit** + **connection_pool_default_memory_limit** + Default resource configuration for connection pool deployment. The internal + default for memory request and limit is `100Mi`, for CPU it is `500m` and `1`. diff --git a/docs/user.md b/docs/user.md index 9a6752185..8c79bb485 100644 --- a/docs/user.md +++ b/docs/user.md @@ -512,6 +512,59 @@ monitoring is outside the scope of operator responsibilities. See [administrator documentation](administrator.md) for details on how backups are executed. +## Connection pool + +The operator can create a database side connection pool for those applications, +where an application side pool is not feasible, but a number of connections is +high. To create a connection pool together with a database, modify the +manifest: + +```yaml +spec: + enableConnectionPool: true +``` + +This will tell the operator to create a connection pool with default +configuration, through which one can access the master via a separate service +`{cluster-name}-pooler`. In most of the cases provided default configuration +should be good enough. + +To configure a new connection pool, specify: + +``` +spec: + connectionPool: + # how many instances of connection pool to create + number_of_instances: 2 + + # in which mode to run, session or transaction + mode: "transaction" + + # schema, which operator will create to install credentials lookup + # function + schema: "pooler" + + # user, which operator will create for connection pool + user: "pooler" + + # resources for each instance + resources: + requests: + cpu: 500m + memory: 100Mi + limits: + cpu: "1" + memory: 100Mi +``` + +By default `pgbouncer` is used to create a connection pool. To find out about +pool modes see [docs](https://www.pgbouncer.org/config.html#pool_mode) (but it +should be general approach between different implementation). + +Note, that using `pgbouncer` means meaningful resource CPU limit should be less +than 1 core (there is a way to utilize more than one, but in K8S it's easier +just to spin up more instances). + ## Custom TLS certificates By default, the spilo image generates its own TLS certificate during startup. diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index f0d8a0b23..cc90aa5e2 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -9,6 +9,10 @@ import yaml from kubernetes import client, config +def to_selector(labels): + return ",".join(["=".join(l) for l in labels.items()]) + + class EndToEndTestCase(unittest.TestCase): ''' Test interaction of the operator with multiple K8s components. @@ -47,7 +51,8 @@ class EndToEndTestCase(unittest.TestCase): for filename in ["operator-service-account-rbac.yaml", "configmap.yaml", "postgres-operator.yaml"]: - k8s.create_with_kubectl("manifests/" + filename) + result = k8s.create_with_kubectl("manifests/" + filename) + print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) k8s.wait_for_operator_pod_start() @@ -55,9 +60,14 @@ class EndToEndTestCase(unittest.TestCase): 'default', label_selector='name=postgres-operator').items[0].spec.containers[0].image print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish - k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") - k8s.wait_for_pod_start('spilo-role=master') - k8s.wait_for_pod_start('spilo-role=replica') + result = k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") + print('stdout: {}, stderr: {}'.format(result.stdout, result.stderr)) + try: + k8s.wait_for_pod_start('spilo-role=master') + k8s.wait_for_pod_start('spilo-role=replica') + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_enable_load_balancer(self): @@ -66,7 +76,7 @@ class EndToEndTestCase(unittest.TestCase): ''' k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' # enable load balancer services pg_patch_enable_lbs = { @@ -178,7 +188,7 @@ class EndToEndTestCase(unittest.TestCase): Lower resource limits below configured minimum and let operator fix it ''' k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' labels = 'spilo-role=master,' + cluster_label _, failover_targets = k8s.get_pg_nodes(cluster_label) @@ -247,7 +257,7 @@ class EndToEndTestCase(unittest.TestCase): Remove node readiness label from master node. This must cause a failover. ''' k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' labels = 'spilo-role=master,' + cluster_label readiness_label = 'lifecycle-status' readiness_value = 'ready' @@ -289,7 +299,7 @@ class EndToEndTestCase(unittest.TestCase): Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. ''' k8s = self.k8s - labels = "cluster-name=acid-minimal-cluster" + labels = "application=spilo,cluster-name=acid-minimal-cluster" k8s.wait_for_pg_to_scale(3) self.assertEqual(3, k8s.count_pods_with_label(labels)) @@ -339,13 +349,99 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(unpatch_custom_service_annotations) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_enable_disable_connection_pool(self): + ''' + For a database without connection pool, then turns it on, scale up, + turn off and on again. Test with different ways of doing this (via + enableConnectionPool or connectionPool configuration section). At the + end turn the connection pool off to not interfere with other tests. + ''' + k8s = self.k8s + service_labels = { + 'cluster-name': 'acid-minimal-cluster', + } + pod_labels = dict({ + 'connection-pool': 'acid-minimal-cluster-pooler', + }) + + pod_selector = to_selector(pod_labels) + service_selector = to_selector(service_labels) + + try: + # enable connection pool + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': True, + } + }) + k8s.wait_for_pod_start(pod_selector) + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=pod_selector + ).items + + self.assertTrue(pods, 'No connection pool pods') + + k8s.wait_for_service(service_selector) + services = k8s.api.core_v1.list_namespaced_service( + 'default', label_selector=service_selector + ).items + services = [ + s for s in services + if s.metadata.name.endswith('pooler') + ] + + self.assertTrue(services, 'No connection pool service') + + # scale up connection pool deployment + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'connectionPool': { + 'numberOfInstances': 2, + }, + } + }) + + k8s.wait_for_running_pods(pod_selector, 2) + + # turn it off, keeping configuration section + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': False, + } + }) + k8s.wait_for_pods_to_stop(pod_selector) + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': True, + } + }) + k8s.wait_for_pod_start(pod_selector) + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): ''' Add taint "postgres=:NoExecute" to node with master. This must cause a failover. ''' k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' # get nodes of master and replica(s) (expected target of new master) current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) @@ -496,12 +592,39 @@ class K8s: # for local execution ~ 10 seconds suffices time.sleep(60) + def get_operator_pod(self): + pods = self.api.core_v1.list_namespaced_pod( + 'default', label_selector='name=postgres-operator' + ).items + + if pods: + return pods[0] + + return None + + def get_operator_log(self): + operator_pod = self.get_operator_pod() + pod_name = operator_pod.metadata.name + return self.api.core_v1.read_namespaced_pod_log( + name=pod_name, + namespace='default' + ) + def wait_for_pod_start(self, pod_labels, namespace='default'): pod_phase = 'No pod running' while pod_phase != 'Running': pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels).items if pods: pod_phase = pods[0].status.phase + + if pods and pod_phase != 'Running': + pod_name = pods[0].metadata.name + response = self.api.core_v1.read_namespaced_pod( + name=pod_name, + namespace=namespace + ) + print("Pod description {}".format(response)) + time.sleep(self.RETRY_TIMEOUT_SEC) def get_service_type(self, svc_labels, namespace='default'): @@ -531,10 +654,27 @@ class K8s: _ = self.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body) - labels = 'cluster-name=acid-minimal-cluster' + labels = 'application=spilo,cluster-name=acid-minimal-cluster' while self.count_pods_with_label(labels) != number_of_instances: time.sleep(self.RETRY_TIMEOUT_SEC) + def wait_for_running_pods(self, labels, number, namespace=''): + while self.count_pods_with_label(labels) != number: + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_pods_to_stop(self, labels, namespace=''): + while self.count_pods_with_label(labels) != 0: + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_service(self, labels, namespace='default'): + def get_services(): + return self.api.core_v1.list_namespaced_service( + namespace, label_selector=labels + ).items + + while not get_services(): + time.sleep(self.RETRY_TIMEOUT_SEC) + def count_pods_with_label(self, labels, namespace='default'): return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) @@ -571,7 +711,10 @@ class K8s: self.wait_for_operator_pod_start() def create_with_kubectl(self, path): - subprocess.run(["kubectl", "create", "-f", path]) + return subprocess.run( + ["kubectl", "create", "-f", path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) if __name__ == '__main__': diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 0300b5495..fdc2d5d56 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -11,6 +11,16 @@ data: cluster_history_entries: "1000" cluster_labels: application:spilo cluster_name_label: cluster-name + # connection_pool_default_cpu_limit: "1" + # connection_pool_default_cpu_request: "500m" + # connection_pool_default_memory_limit: 100Mi + # connection_pool_default_memory_request: 100Mi + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + # connection_pool_max_db_connections: 60 + # connection_pool_mode: "transaction" + # connection_pool_number_of_instances: 2 + # connection_pool_schema: "pooler" + # connection_pool_user: "pooler" # custom_service_annotations: "keyx:valuez,keya:valuea" # custom_pod_annotations: "keya:valuea,keyb:valueb" db_hosted_zone: db.example.com diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index e5bc49f83..83cd721e7 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -129,6 +129,7 @@ rules: - apps resources: - statefulsets + - deployments verbs: - create - delete diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 7bd5c529c..4e6858af8 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -294,6 +294,47 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string + connection_pool: + type: object + properties: + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" + connection_pool_image: + type: string + #default: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pool_max_db_connections: + type: integer + #default: 60 + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pool_number_of_instances: + type: integer + minimum: 2 + #default: 2 + connection_pool_default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pool_default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "500m" + connection_pool_default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100Mi" + connection_pool_default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100Mi" status: type: object additionalProperties: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 33838b2a9..d4c9b518f 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -121,3 +121,14 @@ configuration: scalyr_memory_limit: 500Mi scalyr_memory_request: 50Mi # scalyr_server_url: "" + connection_pool: + connection_pool_default_cpu_limit: "1" + connection_pool_default_cpu_request: "500m" + connection_pool_default_memory_limit: 100Mi + connection_pool_default_memory_request: 100Mi + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + # connection_pool_max_db_connections: 60 + connection_pool_mode: "transaction" + connection_pool_number_of_instances: 2 + # connection_pool_schema: "pooler" + # connection_pool_user: "pooler" diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 04c789fb9..06434da14 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -70,6 +70,55 @@ spec: uid: format: uuid type: string + connectionPool: + type: object + properties: + dockerImage: + type: string + maxDBConnections: + type: integer + mode: + type: string + enum: + - "session" + - "transaction" + numberOfInstances: + type: integer + minimum: 2 + resources: + type: object + required: + - requests + - limits + properties: + limits: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + schema: + type: string + user: + type: string databases: type: object additionalProperties: @@ -77,6 +126,8 @@ spec: # Note: usernames specified here as database owners must be declared in the users key of the spec key. dockerImage: type: string + enableConnectionPool: + type: boolean enableLogicalBackup: type: boolean enableMasterLoadBalancer: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index eff47c255..dc552d3f4 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -105,6 +105,7 @@ var OperatorConfigCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefin var min0 = 0.0 var min1 = 1.0 +var min2 = 2.0 var minDisable = -1.0 // PostgresCRDResourceValidation to check applied manifest parameters @@ -176,6 +177,76 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "connectionPool": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "dockerImage": { + Type: "string", + }, + "maxDBConnections": { + Type: "integer", + }, + "mode": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"session"`), + }, + { + Raw: []byte(`"transaction"`), + }, + }, + }, + "numberOfInstances": { + Type: "integer", + Minimum: &min2, + }, + "resources": { + Type: "object", + Required: []string{"requests", "limits"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "limits": { + Type: "object", + Required: []string{"cpu", "memory"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "cpu": { + Type: "string", + Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "memory": { + Type: "string", + Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + }, + }, + "requests": { + Type: "object", + Required: []string{"cpu", "memory"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "cpu": { + Type: "string", + Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "memory": { + Type: "string", + Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + }, + }, + }, + }, + "schema": { + Type: "string", + }, + "user": { + Type: "string", + }, + }, + }, "databases": { Type: "object", AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ @@ -188,6 +259,9 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "dockerImage": { Type: "string", }, + "enableConnectionPool": { + Type: "boolean", + }, "enableLogicalBackup": { Type: "boolean", }, @@ -418,7 +492,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ Type: "string", }, "tls": { - Type: "object", + Type: "object", Required: []string{"secretName"}, Properties: map[string]apiextv1beta1.JSONSchemaProps{ "secretName": { @@ -1055,6 +1129,54 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "connection_pool": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "connection_pool_default_cpu_limit": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "connection_pool_default_cpu_request": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "connection_pool_default_memory_limit": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "connection_pool_default_memory_request": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "connection_pool_image": { + Type: "string", + }, + "connection_pool_max_db_connections": { + Type: "integer", + }, + "connection_pool_mode": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"session"`), + }, + { + Raw: []byte(`"transaction"`), + }, + }, + }, + "connection_pool_number_of_instances": { + Type: "integer", + Minimum: &min2, + }, + "connection_pool_schema": { + Type: "string", + }, + "connection_pool_user": { + Type: "string", + }, + }, + }, }, }, "status": { diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 8463be6bd..7c1d2b7e8 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -1,5 +1,7 @@ package v1 +// Operator configuration CRD definition, please use snake_case for field names. + import ( "github.com/zalando/postgres-operator/pkg/util/config" @@ -65,12 +67,12 @@ type KubernetesMetaConfiguration struct { // TODO: use a proper toleration structure? PodToleration map[string]string `json:"toleration,omitempty"` // TODO: use namespacedname - PodEnvironmentConfigMap string `json:"pod_environment_configmap,omitempty"` - PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` - MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` - EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` - PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` - PodManagementPolicy string `json:"pod_management_policy,omitempty"` + PodEnvironmentConfigMap string `json:"pod_environment_configmap,omitempty"` + PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` + MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` + EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` + PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` + PodManagementPolicy string `json:"pod_management_policy,omitempty"` } // PostgresPodResourcesDefaults defines the spec of default resources @@ -152,6 +154,20 @@ type ScalyrConfiguration struct { ScalyrMemoryLimit string `json:"scalyr_memory_limit,omitempty"` } +// Defines default configuration for connection pool +type ConnectionPoolConfiguration struct { + NumberOfInstances *int32 `json:"connection_pool_number_of_instances,omitempty"` + Schema string `json:"connection_pool_schema,omitempty"` + User string `json:"connection_pool_user,omitempty"` + Image string `json:"connection_pool_image,omitempty"` + Mode string `json:"connection_pool_mode,omitempty"` + MaxDBConnections *int32 `json:"connection_pool_max_db_connections,omitempty"` + DefaultCPURequest string `json:"connection_pool_default_cpu_request,omitempty"` + DefaultMemoryRequest string `json:"connection_pool_default_memory_request,omitempty"` + DefaultCPULimit string `json:"connection_pool_default_cpu_limit,omitempty"` + DefaultMemoryLimit string `json:"connection_pool_default_memory_limit,omitempty"` +} + // OperatorLogicalBackupConfiguration defines configuration for logical backup type OperatorLogicalBackupConfiguration struct { Schedule string `json:"logical_backup_schedule,omitempty"` @@ -188,6 +204,7 @@ type OperatorConfigurationData struct { LoggingRESTAPI LoggingRESTAPIConfiguration `json:"logging_rest_api"` Scalyr ScalyrConfiguration `json:"scalyr"` LogicalBackup OperatorLogicalBackupConfiguration `json:"logical_backup"` + ConnectionPool ConnectionPoolConfiguration `json:"connection_pool"` } //Duration shortens this frequently used name diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 862db6a4e..1784f8235 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -1,5 +1,7 @@ package v1 +// Postgres CRD definition, please use CamelCase for field names. + import ( "time" @@ -27,6 +29,9 @@ type PostgresSpec struct { Patroni `json:"patroni,omitempty"` Resources `json:"resources,omitempty"` + EnableConnectionPool *bool `json:"enableConnectionPool,omitempty"` + ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` + TeamID string `json:"teamId"` DockerImage string `json:"dockerImage,omitempty"` @@ -162,3 +167,24 @@ type UserFlags []string type PostgresStatus struct { PostgresClusterStatus string `json:"PostgresClusterStatus"` } + +// Options for connection pooler +// +// TODO: prepared snippets of configuration, one can choose via type, e.g. +// pgbouncer-large (with higher resources) or odyssey-small (with smaller +// resources) +// Type string `json:"type,omitempty"` +// +// TODO: figure out what other important parameters of the connection pool it +// makes sense to expose. E.g. pool size (min/max boundaries), max client +// connections etc. +type ConnectionPool struct { + NumberOfInstances *int32 `json:"numberOfInstances,omitempty"` + Schema string `json:"schema,omitempty"` + User string `json:"user,omitempty"` + Mode string `json:"mode,omitempty"` + DockerImage string `json:"dockerImage,omitempty"` + MaxDBConnections *int32 `json:"maxDBConnections,omitempty"` + + Resources `json:"resources,omitempty"` +} diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 753b0490c..fcab394ca 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -68,6 +68,59 @@ func (in *CloneDescription) DeepCopy() *CloneDescription { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectionPool) DeepCopyInto(out *ConnectionPool) { + *out = *in + if in.NumberOfInstances != nil { + in, out := &in.NumberOfInstances, &out.NumberOfInstances + *out = new(int32) + **out = **in + } + if in.MaxDBConnections != nil { + in, out := &in.MaxDBConnections, &out.MaxDBConnections + *out = new(int32) + **out = **in + } + out.Resources = in.Resources + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPool. +func (in *ConnectionPool) DeepCopy() *ConnectionPool { + if in == nil { + return nil + } + out := new(ConnectionPool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectionPoolConfiguration) DeepCopyInto(out *ConnectionPoolConfiguration) { + *out = *in + if in.NumberOfInstances != nil { + in, out := &in.NumberOfInstances, &out.NumberOfInstances + *out = new(int32) + **out = **in + } + if in.MaxDBConnections != nil { + in, out := &in.MaxDBConnections, &out.MaxDBConnections + *out = new(int32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPoolConfiguration. +func (in *ConnectionPoolConfiguration) DeepCopy() *ConnectionPoolConfiguration { + if in == nil { + return nil + } + out := new(ConnectionPoolConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfiguration) { *out = *in @@ -254,6 +307,7 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData out.LoggingRESTAPI = in.LoggingRESTAPI out.Scalyr = in.Scalyr out.LogicalBackup = in.LogicalBackup + in.ConnectionPool.DeepCopyInto(&out.ConnectionPool) return } @@ -416,6 +470,16 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { out.Volume = in.Volume in.Patroni.DeepCopyInto(&out.Patroni) out.Resources = in.Resources + if in.EnableConnectionPool != nil { + in, out := &in.EnableConnectionPool, &out.EnableConnectionPool + *out = new(bool) + **out = **in + } + if in.ConnectionPool != nil { + in, out := &in.ConnectionPool, &out.ConnectionPool + *out = new(ConnectionPool) + (*in).DeepCopyInto(*out) + } if in.SpiloFSGroup != nil { in, out := &in.SpiloFSGroup, &out.SpiloFSGroup *out = new(int64) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index d740260d2..dba67c142 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/r3labs/diff" "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -48,11 +49,26 @@ type Config struct { PodServiceAccountRoleBinding *rbacv1.RoleBinding } +// K8S objects that are belongs to a connection pool +type ConnectionPoolObjects struct { + Deployment *appsv1.Deployment + Service *v1.Service + + // It could happen that a connection pool was enabled, but the operator was + // not able to properly process a corresponding event or was restarted. In + // this case we will miss missing/require situation and a lookup function + // will not be installed. To avoid synchronizing it all the time to prevent + // this, we can remember the result in memory at least until the next + // restart. + LookupFunction bool +} + type kubeResources struct { Services map[PostgresRole]*v1.Service Endpoints map[PostgresRole]*v1.Endpoints Secrets map[types.UID]*v1.Secret Statefulset *appsv1.StatefulSet + ConnectionPool *ConnectionPoolObjects PodDisruptionBudget *policybeta1.PodDisruptionBudget //Pods are treated separately //PVCs are treated separately @@ -184,7 +200,8 @@ func (c *Cluster) isNewCluster() bool { func (c *Cluster) initUsers() error { c.setProcessName("initializing users") - // clear our the previous state of the cluster users (in case we are running a sync). + // clear our the previous state of the cluster users (in case we are + // running a sync). c.systemUsers = map[string]spec.PgUser{} c.pgUsers = map[string]spec.PgUser{} @@ -292,8 +309,10 @@ func (c *Cluster) Create() error { } c.logger.Infof("pods are ready") - // create database objects unless we are running without pods or disabled that feature explicitly + // create database objects unless we are running without pods or disabled + // that feature explicitly if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&c.Spec) <= 0 || c.Spec.StandbyCluster != nil) { + c.logger.Infof("Create roles") if err = c.createRoles(); err != nil { return fmt.Errorf("could not create users: %v", err) } @@ -316,6 +335,26 @@ func (c *Cluster) Create() error { c.logger.Errorf("could not list resources: %v", err) } + // Create connection pool deployment and services if necessary. Since we + // need to peform some operations with the database itself (e.g. install + // lookup function), do it as the last step, when everything is available. + // + // Do not consider connection pool as a strict requirement, and if + // something fails, report warning + if c.needConnectionPool() { + if c.ConnectionPool != nil { + c.logger.Warning("Connection pool already exists in the cluster") + return nil + } + connPool, err := c.createConnectionPool(c.installLookupFunction) + if err != nil { + c.logger.Warningf("could not create connection pool: %v", err) + return nil + } + c.logger.Infof("connection pool %q has been successfully created", + util.NameFromMeta(connPool.Deployment.ObjectMeta)) + } + return nil } @@ -571,7 +610,11 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } - if !reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) { + // connection pool needs one system user created, which is done in + // initUsers. Check if it needs to be called. + sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) + needConnPool := c.needConnectionPoolWorker(&newSpec.Spec) + if !sameUsers || needConnPool { c.logger.Debugf("syncing secrets") if err := c.initUsers(); err != nil { c.logger.Errorf("could not init users: %v", err) @@ -695,6 +738,11 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } + // sync connection pool + if err := c.syncConnectionPool(oldSpec, newSpec, c.installLookupFunction); err != nil { + return fmt.Errorf("could not sync connection pool: %v", err) + } + return nil } @@ -746,6 +794,12 @@ func (c *Cluster) Delete() { c.logger.Warningf("could not remove leftover patroni objects; %v", err) } + // Delete connection pool objects anyway, even if it's not mentioned in the + // manifest, just to not keep orphaned components in case if something went + // wrong + if err := c.deleteConnectionPool(); err != nil { + c.logger.Warningf("could not remove connection pool: %v", err) + } } //NeedsRepair returns true if the cluster should be included in the repair scan (based on its in-memory status). @@ -812,6 +866,35 @@ func (c *Cluster) initSystemUsers() { Name: c.OpConfig.ReplicationUsername, Password: util.RandomPassword(constants.PasswordLength), } + + // Connection pool user is an exception, if requested it's going to be + // created by operator as a normal pgUser + if c.needConnectionPool() { + // initialize empty connection pool if not done yet + if c.Spec.ConnectionPool == nil { + c.Spec.ConnectionPool = &acidv1.ConnectionPool{} + } + + username := util.Coalesce( + c.Spec.ConnectionPool.User, + c.OpConfig.ConnectionPool.User) + + // connection pooler application should be able to login with this role + connPoolUser := spec.PgUser{ + Origin: spec.RoleConnectionPool, + Name: username, + Flags: []string{constants.RoleFlagLogin}, + Password: util.RandomPassword(constants.PasswordLength), + } + + if _, exists := c.pgUsers[username]; !exists { + c.pgUsers[username] = connPoolUser + } + + if _, exists := c.systemUsers[constants.ConnectionPoolUserKeyName]; !exists { + c.systemUsers[constants.ConnectionPoolUserKeyName] = connPoolUser + } + } } func (c *Cluster) initRobotUsers() error { @@ -1138,3 +1221,119 @@ func (c *Cluster) deletePatroniClusterConfigMaps() error { return c.deleteClusterObject(get, deleteConfigMapFn, "configmap") } + +// Test if two connection pool configuration needs to be synced. For simplicity +// compare not the actual K8S objects, but the configuration itself and request +// sync if there is any difference. +func (c *Cluster) needSyncConnPoolSpecs(oldSpec, newSpec *acidv1.ConnectionPool) (sync bool, reasons []string) { + reasons = []string{} + sync = false + + changelog, err := diff.Diff(oldSpec, newSpec) + if err != nil { + c.logger.Infof("Cannot get diff, do not do anything, %+v", err) + return false, reasons + } + + if len(changelog) > 0 { + sync = true + } + + for _, change := range changelog { + msg := fmt.Sprintf("%s %+v from '%+v' to '%+v'", + change.Type, change.Path, change.From, change.To) + reasons = append(reasons, msg) + } + + return sync, reasons +} + +func syncResources(a, b *v1.ResourceRequirements) bool { + for _, res := range []v1.ResourceName{ + v1.ResourceCPU, + v1.ResourceMemory, + } { + if !a.Limits[res].Equal(b.Limits[res]) || + !a.Requests[res].Equal(b.Requests[res]) { + return true + } + } + + return false +} + +// Check if we need to synchronize connection pool deployment due to new +// defaults, that are different from what we see in the DeploymentSpec +func (c *Cluster) needSyncConnPoolDefaults( + spec *acidv1.ConnectionPool, + deployment *appsv1.Deployment) (sync bool, reasons []string) { + + reasons = []string{} + sync = false + + config := c.OpConfig.ConnectionPool + podTemplate := deployment.Spec.Template + poolContainer := podTemplate.Spec.Containers[constants.ConnPoolContainer] + + if spec == nil { + spec = &acidv1.ConnectionPool{} + } + + if spec.NumberOfInstances == nil && + *deployment.Spec.Replicas != *config.NumberOfInstances { + + sync = true + msg := fmt.Sprintf("NumberOfInstances is different (having %d, required %d)", + *deployment.Spec.Replicas, *config.NumberOfInstances) + reasons = append(reasons, msg) + } + + if spec.DockerImage == "" && + poolContainer.Image != config.Image { + + sync = true + msg := fmt.Sprintf("DockerImage is different (having %s, required %s)", + poolContainer.Image, config.Image) + reasons = append(reasons, msg) + } + + expectedResources, err := generateResourceRequirements(spec.Resources, + c.makeDefaultConnPoolResources()) + + // An error to generate expected resources means something is not quite + // right, but for the purpose of robustness do not panic here, just report + // and ignore resources comparison (in the worst case there will be no + // updates for new resource values). + if err == nil && syncResources(&poolContainer.Resources, expectedResources) { + sync = true + msg := fmt.Sprintf("Resources are different (having %+v, required %+v)", + poolContainer.Resources, expectedResources) + reasons = append(reasons, msg) + } + + if err != nil { + c.logger.Warningf("Cannot generate expected resources, %v", err) + } + + for _, env := range poolContainer.Env { + if spec.User == "" && env.Name == "PGUSER" { + ref := env.ValueFrom.SecretKeyRef.LocalObjectReference + + if ref.Name != c.credentialSecretName(config.User) { + sync = true + msg := fmt.Sprintf("Pool user is different (having %s, required %s)", + ref.Name, config.User) + reasons = append(reasons, msg) + } + } + + if spec.Schema == "" && env.Name == "PGSCHEMA" && env.Value != config.Schema { + sync = true + msg := fmt.Sprintf("Pool schema is different (having %s, required %s)", + env.Value, config.Schema) + reasons = append(reasons, msg) + } + } + + return sync, reasons +} diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 9efbc51c6..a1b361642 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -9,6 +9,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/teams" v1 "k8s.io/api/core/v1" @@ -704,3 +705,20 @@ func TestServiceAnnotations(t *testing.T) { }) } } + +func TestInitSystemUsers(t *testing.T) { + testName := "Test system users initialization" + + // default cluster without connection pool + cl.initSystemUsers() + if _, exist := cl.systemUsers[constants.ConnectionPoolUserKeyName]; exist { + t.Errorf("%s, connection pool user is present", testName) + } + + // cluster with connection pool + cl.Spec.EnableConnectionPool = boolToPointer(true) + cl.initSystemUsers() + if _, exist := cl.systemUsers[constants.ConnectionPoolUserKeyName]; !exist { + t.Errorf("%s, connection pool user is not present", testName) + } +} diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 07ea011a6..bca68c188 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -1,10 +1,12 @@ package cluster import ( + "bytes" "database/sql" "fmt" "net" "strings" + "text/template" "time" "github.com/lib/pq" @@ -28,13 +30,37 @@ const ( getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` + connectionPoolLookup = ` + CREATE SCHEMA IF NOT EXISTS {{.pool_schema}}; + + CREATE OR REPLACE FUNCTION {{.pool_schema}}.user_lookup( + in i_username text, out uname text, out phash text) + RETURNS record AS $$ + BEGIN + SELECT usename, passwd FROM pg_catalog.pg_shadow + WHERE usename = i_username INTO uname, phash; + RETURN; + END; + $$ LANGUAGE plpgsql SECURITY DEFINER; + + REVOKE ALL ON FUNCTION {{.pool_schema}}.user_lookup(text) + FROM public, {{.pool_user}}; + GRANT EXECUTE ON FUNCTION {{.pool_schema}}.user_lookup(text) + TO {{.pool_user}}; + GRANT USAGE ON SCHEMA {{.pool_schema}} TO {{.pool_user}}; + ` ) -func (c *Cluster) pgConnectionString() string { +func (c *Cluster) pgConnectionString(dbname string) string { password := c.systemUsers[constants.SuperuserKeyName].Password - return fmt.Sprintf("host='%s' dbname=postgres sslmode=require user='%s' password='%s' connect_timeout='%d'", + if dbname == "" { + dbname = "postgres" + } + + return fmt.Sprintf("host='%s' dbname='%s' sslmode=require user='%s' password='%s' connect_timeout='%d'", fmt.Sprintf("%s.%s.svc.%s", c.Name, c.Namespace, c.OpConfig.ClusterDomain), + dbname, c.systemUsers[constants.SuperuserKeyName].Name, strings.Replace(password, "$", "\\$", -1), constants.PostgresConnectTimeout/time.Second) @@ -49,13 +75,17 @@ func (c *Cluster) databaseAccessDisabled() bool { } func (c *Cluster) initDbConn() error { + return c.initDbConnWithName("") +} + +func (c *Cluster) initDbConnWithName(dbname string) error { c.setProcessName("initializing db connection") if c.pgDb != nil { return nil } var conn *sql.DB - connstring := c.pgConnectionString() + connstring := c.pgConnectionString(dbname) finalerr := retryutil.Retry(constants.PostgresConnectTimeout, constants.PostgresConnectRetryTimeout, func() (bool, error) { @@ -94,6 +124,10 @@ func (c *Cluster) initDbConn() error { return nil } +func (c *Cluster) connectionIsClosed() bool { + return c.pgDb == nil +} + func (c *Cluster) closeDbConn() (err error) { c.setProcessName("closing db connection") if c.pgDb != nil { @@ -243,3 +277,88 @@ func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin return result } + +// Creates a connection pool credentials lookup function in every database to +// perform remote authentification. +func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { + var stmtBytes bytes.Buffer + c.logger.Info("Installing lookup function") + + if err := c.initDbConn(); err != nil { + return fmt.Errorf("could not init database connection") + } + defer func() { + if c.connectionIsClosed() { + return + } + + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + }() + + currentDatabases, err := c.getDatabases() + if err != nil { + msg := "could not get databases to install pool lookup function: %v" + return fmt.Errorf(msg, err) + } + + templater := template.Must(template.New("sql").Parse(connectionPoolLookup)) + + for dbname, _ := range currentDatabases { + if dbname == "template0" || dbname == "template1" { + continue + } + + if err := c.initDbConnWithName(dbname); err != nil { + return fmt.Errorf("could not init database connection to %s", dbname) + } + + c.logger.Infof("Install pool lookup function into %s", dbname) + + params := TemplateParams{ + "pool_schema": poolSchema, + "pool_user": poolUser, + } + + if err := templater.Execute(&stmtBytes, params); err != nil { + c.logger.Errorf("could not prepare sql statement %+v: %v", + params, err) + // process other databases + continue + } + + // golang sql will do retries couple of times if pq driver reports + // connections issues (driver.ErrBadConn), but since our query is + // idempotent, we can retry in a view of other errors (e.g. due to + // failover a db is temporary in a read-only mode or so) to make sure + // it was applied. + execErr := retryutil.Retry( + constants.PostgresConnectTimeout, + constants.PostgresConnectRetryTimeout, + func() (bool, error) { + if _, err := c.pgDb.Exec(stmtBytes.String()); err != nil { + msg := fmt.Errorf("could not execute sql statement %s: %v", + stmtBytes.String(), err) + return false, msg + } + + return true, nil + }) + + if execErr != nil { + c.logger.Errorf("could not execute after retries %s: %v", + stmtBytes.String(), err) + // process other databases + continue + } + + c.logger.Infof("Pool lookup function installed into %s", dbname) + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + } + + c.ConnectionPool.LookupFunction = true + return nil +} diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 39b5c16a0..34b409d4e 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -21,6 +21,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" batchv1 "k8s.io/api/batch/v1" batchv1beta1 "k8s.io/api/batch/v1beta1" "k8s.io/apimachinery/pkg/labels" @@ -31,10 +32,12 @@ const ( patroniPGBinariesParameterName = "bin_dir" patroniPGParametersParameterName = "parameters" patroniPGHBAConfParameterName = "pg_hba" + localHost = "127.0.0.1/32" + connectionPoolContainer = "connection-pool" + pgPort = 5432 // the gid of the postgres user in the default spilo image spiloPostgresGID = 103 - localHost = "127.0.0.1/32" ) type pgUser struct { @@ -70,6 +73,10 @@ func (c *Cluster) statefulSetName() string { return c.Name } +func (c *Cluster) connPoolName() string { + return c.Name + "-pooler" +} + func (c *Cluster) endpointName(role PostgresRole) string { name := c.Name if role == Replica { @@ -88,6 +95,28 @@ func (c *Cluster) serviceName(role PostgresRole) string { return name } +func (c *Cluster) serviceAddress(role PostgresRole) string { + service, exist := c.Services[role] + + if exist { + return service.ObjectMeta.Name + } + + c.logger.Warningf("No service for role %s", role) + return "" +} + +func (c *Cluster) servicePort(role PostgresRole) string { + service, exist := c.Services[role] + + if exist { + return fmt.Sprint(service.Spec.Ports[0].Port) + } + + c.logger.Warningf("No service for role %s", role) + return "" +} + func (c *Cluster) podDisruptionBudgetName() string { return c.OpConfig.PDBNameFormat.Format("cluster", c.Name) } @@ -96,10 +125,39 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources { config := c.OpConfig - defaultRequests := acidv1.ResourceDescription{CPU: config.DefaultCPURequest, Memory: config.DefaultMemoryRequest} - defaultLimits := acidv1.ResourceDescription{CPU: config.DefaultCPULimit, Memory: config.DefaultMemoryLimit} + defaultRequests := acidv1.ResourceDescription{ + CPU: config.Resources.DefaultCPURequest, + Memory: config.Resources.DefaultMemoryRequest, + } + defaultLimits := acidv1.ResourceDescription{ + CPU: config.Resources.DefaultCPULimit, + Memory: config.Resources.DefaultMemoryLimit, + } - return acidv1.Resources{ResourceRequests: defaultRequests, ResourceLimits: defaultLimits} + return acidv1.Resources{ + ResourceRequests: defaultRequests, + ResourceLimits: defaultLimits, + } +} + +// Generate default resource section for connection pool deployment, to be used +// if nothing custom is specified in the manifest +func (c *Cluster) makeDefaultConnPoolResources() acidv1.Resources { + config := c.OpConfig + + defaultRequests := acidv1.ResourceDescription{ + CPU: config.ConnectionPool.ConnPoolDefaultCPURequest, + Memory: config.ConnectionPool.ConnPoolDefaultMemoryRequest, + } + defaultLimits := acidv1.ResourceDescription{ + CPU: config.ConnectionPool.ConnPoolDefaultCPULimit, + Memory: config.ConnectionPool.ConnPoolDefaultMemoryLimit, + } + + return acidv1.Resources{ + ResourceRequests: defaultRequests, + ResourceLimits: defaultLimits, + } } func generateResourceRequirements(resources acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) { @@ -319,7 +377,11 @@ func tolerations(tolerationsSpec *[]v1.Toleration, podToleration map[string]stri return *tolerationsSpec } - if len(podToleration["key"]) > 0 || len(podToleration["operator"]) > 0 || len(podToleration["value"]) > 0 || len(podToleration["effect"]) > 0 { + if len(podToleration["key"]) > 0 || + len(podToleration["operator"]) > 0 || + len(podToleration["value"]) > 0 || + len(podToleration["effect"]) > 0 { + return []v1.Toleration{ { Key: podToleration["key"], @@ -786,12 +848,12 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef request := spec.Resources.ResourceRequests.Memory if request == "" { - request = c.OpConfig.DefaultMemoryRequest + request = c.OpConfig.Resources.DefaultMemoryRequest } limit := spec.Resources.ResourceLimits.Memory if limit == "" { - limit = c.OpConfig.DefaultMemoryLimit + limit = c.OpConfig.Resources.DefaultMemoryLimit } isSmaller, err := util.IsSmallerQuantity(request, limit) @@ -813,12 +875,12 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // TODO #413 sidecarRequest := sidecar.Resources.ResourceRequests.Memory if request == "" { - request = c.OpConfig.DefaultMemoryRequest + request = c.OpConfig.Resources.DefaultMemoryRequest } sidecarLimit := sidecar.Resources.ResourceLimits.Memory if limit == "" { - limit = c.OpConfig.DefaultMemoryLimit + limit = c.OpConfig.Resources.DefaultMemoryLimit } isSmaller, err := util.IsSmallerQuantity(sidecarRequest, sidecarLimit) @@ -1774,6 +1836,306 @@ func (c *Cluster) getLogicalBackupJobName() (jobName string) { return "logical-backup-" + c.clusterName().Name } +// Generate pool size related environment variables. +// +// MAX_DB_CONN would specify the global maximum for connections to a target +// database. +// +// MAX_CLIENT_CONN is not configurable at the moment, just set it high enough. +// +// DEFAULT_SIZE is a pool size per db/user (having in mind the use case when +// most of the queries coming through a connection pooler are from the same +// user to the same db). In case if we want to spin up more connection pool +// instances, take this into account and maintain the same number of +// connections. +// +// MIN_SIZE is a pool minimal size, to prevent situation when sudden workload +// have to wait for spinning up a new connections. +// +// RESERVE_SIZE is how many additional connections to allow for a pool. +func (c *Cluster) getConnPoolEnvVars(spec *acidv1.PostgresSpec) []v1.EnvVar { + effectiveMode := util.Coalesce( + spec.ConnectionPool.Mode, + c.OpConfig.ConnectionPool.Mode) + + numberOfInstances := spec.ConnectionPool.NumberOfInstances + if numberOfInstances == nil { + numberOfInstances = util.CoalesceInt32( + c.OpConfig.ConnectionPool.NumberOfInstances, + k8sutil.Int32ToPointer(1)) + } + + effectiveMaxDBConn := util.CoalesceInt32( + spec.ConnectionPool.MaxDBConnections, + c.OpConfig.ConnectionPool.MaxDBConnections) + + if effectiveMaxDBConn == nil { + effectiveMaxDBConn = k8sutil.Int32ToPointer( + constants.ConnPoolMaxDBConnections) + } + + maxDBConn := *effectiveMaxDBConn / *numberOfInstances + + defaultSize := maxDBConn / 2 + minSize := defaultSize / 2 + reserveSize := minSize + + return []v1.EnvVar{ + { + Name: "CONNECTION_POOL_PORT", + Value: fmt.Sprint(pgPort), + }, + { + Name: "CONNECTION_POOL_MODE", + Value: effectiveMode, + }, + { + Name: "CONNECTION_POOL_DEFAULT_SIZE", + Value: fmt.Sprint(defaultSize), + }, + { + Name: "CONNECTION_POOL_MIN_SIZE", + Value: fmt.Sprint(minSize), + }, + { + Name: "CONNECTION_POOL_RESERVE_SIZE", + Value: fmt.Sprint(reserveSize), + }, + { + Name: "CONNECTION_POOL_MAX_CLIENT_CONN", + Value: fmt.Sprint(constants.ConnPoolMaxClientConnections), + }, + { + Name: "CONNECTION_POOL_MAX_DB_CONN", + Value: fmt.Sprint(maxDBConn), + }, + } +} + +func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( + *v1.PodTemplateSpec, error) { + + gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) + resources, err := generateResourceRequirements( + spec.ConnectionPool.Resources, + c.makeDefaultConnPoolResources()) + + effectiveDockerImage := util.Coalesce( + spec.ConnectionPool.DockerImage, + c.OpConfig.ConnectionPool.Image) + + effectiveSchema := util.Coalesce( + spec.ConnectionPool.Schema, + c.OpConfig.ConnectionPool.Schema) + + if err != nil { + return nil, fmt.Errorf("could not generate resource requirements: %v", err) + } + + secretSelector := func(key string) *v1.SecretKeySelector { + effectiveUser := util.Coalesce( + spec.ConnectionPool.User, + c.OpConfig.ConnectionPool.User) + + return &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: c.credentialSecretName(effectiveUser), + }, + Key: key, + } + } + + envVars := []v1.EnvVar{ + { + Name: "PGHOST", + Value: c.serviceAddress(Master), + }, + { + Name: "PGPORT", + Value: c.servicePort(Master), + }, + { + Name: "PGUSER", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("username"), + }, + }, + // the convention is to use the same schema name as + // connection pool username + { + Name: "PGSCHEMA", + Value: effectiveSchema, + }, + { + Name: "PGPASSWORD", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("password"), + }, + }, + } + + envVars = append(envVars, c.getConnPoolEnvVars(spec)...) + + poolerContainer := v1.Container{ + Name: connectionPoolContainer, + Image: effectiveDockerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Resources: *resources, + Ports: []v1.ContainerPort{ + { + ContainerPort: pgPort, + Protocol: v1.ProtocolTCP, + }, + }, + Env: envVars, + } + + podTemplate := &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: c.connPoolLabelsSelector().MatchLabels, + Namespace: c.Namespace, + Annotations: c.generatePodAnnotations(spec), + }, + Spec: v1.PodSpec{ + ServiceAccountName: c.OpConfig.PodServiceAccountName, + TerminationGracePeriodSeconds: &gracePeriod, + Containers: []v1.Container{poolerContainer}, + // TODO: add tolerations to scheduler pooler on the same node + // as database + //Tolerations: *tolerationsSpec, + }, + } + + return podTemplate, nil +} + +// Return an array of ownerReferences to make an arbitraty object dependent on +// the StatefulSet. Dependency is made on StatefulSet instead of PostgreSQL CRD +// while the former is represent the actual state, and only it's deletion means +// we delete the cluster (e.g. if CRD was deleted, StatefulSet somehow +// survived, we can't delete an object because it will affect the functioning +// cluster). +func (c *Cluster) ownerReferences() []metav1.OwnerReference { + controller := true + + if c.Statefulset == nil { + c.logger.Warning("Cannot get owner reference, no statefulset") + return []metav1.OwnerReference{} + } + + return []metav1.OwnerReference{ + { + UID: c.Statefulset.ObjectMeta.UID, + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: c.Statefulset.ObjectMeta.Name, + Controller: &controller, + }, + } +} + +func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( + *appsv1.Deployment, error) { + + // there are two ways to enable connection pooler, either to specify a + // connectionPool section or enableConnectionPool. In the second case + // spec.connectionPool will be nil, so to make it easier to calculate + // default values, initialize it to an empty structure. It could be done + // anywhere, but here is the earliest common entry point between sync and + // create code, so init here. + if spec.ConnectionPool == nil { + spec.ConnectionPool = &acidv1.ConnectionPool{} + } + + podTemplate, err := c.generateConnPoolPodTemplate(spec) + numberOfInstances := spec.ConnectionPool.NumberOfInstances + if numberOfInstances == nil { + numberOfInstances = util.CoalesceInt32( + c.OpConfig.ConnectionPool.NumberOfInstances, + k8sutil.Int32ToPointer(1)) + } + + if *numberOfInstances < constants.ConnPoolMinInstances { + msg := "Adjusted number of connection pool instances from %d to %d" + c.logger.Warningf(msg, numberOfInstances, constants.ConnPoolMinInstances) + + *numberOfInstances = constants.ConnPoolMinInstances + } + + if err != nil { + return nil, err + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.connPoolName(), + Namespace: c.Namespace, + Labels: c.connPoolLabelsSelector().MatchLabels, + Annotations: map[string]string{}, + // make StatefulSet object its owner to represent the dependency. + // By itself StatefulSet is being deleted with "Orphaned" + // propagation policy, which means that it's deletion will not + // clean up this deployment, but there is a hope that this object + // will be garbage collected if something went wrong and operator + // didn't deleted it. + OwnerReferences: c.ownerReferences(), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: numberOfInstances, + Selector: c.connPoolLabelsSelector(), + Template: *podTemplate, + }, + } + + return deployment, nil +} + +func (c *Cluster) generateConnPoolService(spec *acidv1.PostgresSpec) *v1.Service { + + // there are two ways to enable connection pooler, either to specify a + // connectionPool section or enableConnectionPool. In the second case + // spec.connectionPool will be nil, so to make it easier to calculate + // default values, initialize it to an empty structure. It could be done + // anywhere, but here is the earliest common entry point between sync and + // create code, so init here. + if spec.ConnectionPool == nil { + spec.ConnectionPool = &acidv1.ConnectionPool{} + } + + serviceSpec := v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: c.connPoolName(), + Port: pgPort, + TargetPort: intstr.IntOrString{StrVal: c.servicePort(Master)}, + }, + }, + Type: v1.ServiceTypeClusterIP, + Selector: map[string]string{ + "connection-pool": c.connPoolName(), + }, + } + + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.connPoolName(), + Namespace: c.Namespace, + Labels: c.connPoolLabelsSelector().MatchLabels, + Annotations: map[string]string{}, + // make StatefulSet object its owner to represent the dependency. + // By itself StatefulSet is being deleted with "Orphaned" + // propagation policy, which means that it's deletion will not + // clean up this service, but there is a hope that this object will + // be garbage collected if something went wrong and operator didn't + // deleted it. + OwnerReferences: c.ownerReferences(), + }, + Spec: serviceSpec, + } + + return service +} + func ensurePath(file string, defaultDir string, defaultFile string) string { if file == "" { return path.Join(defaultDir, defaultFile) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 25e0f7af4..e04b281ba 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1,6 +1,8 @@ package cluster import ( + "errors" + "fmt" "reflect" "testing" @@ -13,6 +15,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -582,6 +585,375 @@ func TestSecretVolume(t *testing.T) { } } +func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + cpuReq := podSpec.Spec.Containers[0].Resources.Requests["cpu"] + if cpuReq.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPURequest { + return fmt.Errorf("CPU request doesn't match, got %s, expected %s", + cpuReq.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPURequest) + } + + memReq := podSpec.Spec.Containers[0].Resources.Requests["memory"] + if memReq.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryRequest { + return fmt.Errorf("Memory request doesn't match, got %s, expected %s", + memReq.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryRequest) + } + + cpuLim := podSpec.Spec.Containers[0].Resources.Limits["cpu"] + if cpuLim.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPULimit { + return fmt.Errorf("CPU limit doesn't match, got %s, expected %s", + cpuLim.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPULimit) + } + + memLim := podSpec.Spec.Containers[0].Resources.Limits["memory"] + if memLim.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryLimit { + return fmt.Errorf("Memory limit doesn't match, got %s, expected %s", + memLim.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryLimit) + } + + return nil +} + +func testLabels(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + poolLabels := podSpec.ObjectMeta.Labels["connection-pool"] + + if poolLabels != cluster.connPoolLabelsSelector().MatchLabels["connection-pool"] { + return fmt.Errorf("Pod labels do not match, got %+v, expected %+v", + podSpec.ObjectMeta.Labels, cluster.connPoolLabelsSelector().MatchLabels) + } + + return nil +} + +func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + required := map[string]bool{ + "PGHOST": false, + "PGPORT": false, + "PGUSER": false, + "PGSCHEMA": false, + "PGPASSWORD": false, + "CONNECTION_POOL_MODE": false, + "CONNECTION_POOL_PORT": false, + } + + envs := podSpec.Spec.Containers[0].Env + for _, env := range envs { + required[env.Name] = true + } + + for env, value := range required { + if !value { + return fmt.Errorf("Environment variable %s is not present", env) + } + } + + return nil +} + +func testCustomPodTemplate(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + if podSpec.ObjectMeta.Name != "test-pod-template" { + return fmt.Errorf("Custom pod template is not used, current spec %+v", + podSpec) + } + + return nil +} + +func TestConnPoolPodSpec(t *testing.T) { + testName := "Test connection pool pod template generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + MaxDBConnections: int32ToPointer(60), + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + var clusterNoDefaultRes = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{}, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + noCheck := func(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { return nil } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + expected error + cluster *Cluster + check func(cluster *Cluster, podSpec *v1.PodTemplateSpec) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: noCheck, + }, + { + subTest: "no default resources", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: errors.New(`could not generate resource requirements: could not fill resource requests: could not parse default CPU quantity: quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'`), + cluster: clusterNoDefaultRes, + check: noCheck, + }, + { + subTest: "default resources are set", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testResources, + }, + { + subTest: "labels for service", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testLabels, + }, + { + subTest: "required envs", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testEnvs, + }, + } + for _, tt := range tests { + podSpec, err := tt.cluster.generateConnPoolPodTemplate(tt.spec) + + if err != tt.expected && err.Error() != tt.expected.Error() { + t.Errorf("%s [%s]: Could not generate pod template,\n %+v, expected\n %+v", + testName, tt.subTest, err, tt.expected) + } + + err = tt.check(cluster, podSpec) + if err != nil { + t.Errorf("%s [%s]: Pod spec is incorrect, %+v", + testName, tt.subTest, err) + } + } +} + +func testDeploymentOwnwerReference(cluster *Cluster, deployment *appsv1.Deployment) error { + owner := deployment.ObjectMeta.OwnerReferences[0] + + if owner.Name != cluster.Statefulset.ObjectMeta.Name { + return fmt.Errorf("Ownere reference is incorrect, got %s, expected %s", + owner.Name, cluster.Statefulset.ObjectMeta.Name) + } + + return nil +} + +func testSelector(cluster *Cluster, deployment *appsv1.Deployment) error { + labels := deployment.Spec.Selector.MatchLabels + expected := cluster.connPoolLabelsSelector().MatchLabels + + if labels["connection-pool"] != expected["connection-pool"] { + return fmt.Errorf("Labels are incorrect, got %+v, expected %+v", + labels, expected) + } + + return nil +} + +func TestConnPoolDeploymentSpec(t *testing.T) { + testName := "Test connection pool deployment spec generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + noCheck := func(cluster *Cluster, deployment *appsv1.Deployment) error { + return nil + } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + expected error + cluster *Cluster + check func(cluster *Cluster, deployment *appsv1.Deployment) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: noCheck, + }, + { + subTest: "owner reference", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testDeploymentOwnwerReference, + }, + { + subTest: "selector", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testSelector, + }, + } + for _, tt := range tests { + deployment, err := tt.cluster.generateConnPoolDeployment(tt.spec) + + if err != tt.expected && err.Error() != tt.expected.Error() { + t.Errorf("%s [%s]: Could not generate deployment spec,\n %+v, expected\n %+v", + testName, tt.subTest, err, tt.expected) + } + + err = tt.check(cluster, deployment) + if err != nil { + t.Errorf("%s [%s]: Deployment spec is incorrect, %+v", + testName, tt.subTest, err) + } + } +} + +func testServiceOwnwerReference(cluster *Cluster, service *v1.Service) error { + owner := service.ObjectMeta.OwnerReferences[0] + + if owner.Name != cluster.Statefulset.ObjectMeta.Name { + return fmt.Errorf("Ownere reference is incorrect, got %s, expected %s", + owner.Name, cluster.Statefulset.ObjectMeta.Name) + } + + return nil +} + +func testServiceSelector(cluster *Cluster, service *v1.Service) error { + selector := service.Spec.Selector + + if selector["connection-pool"] != cluster.connPoolName() { + return fmt.Errorf("Selector is incorrect, got %s, expected %s", + selector["connection-pool"], cluster.connPoolName()) + } + + return nil +} + +func TestConnPoolServiceSpec(t *testing.T) { + testName := "Test connection pool service spec generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + noCheck := func(cluster *Cluster, deployment *v1.Service) error { + return nil + } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + cluster *Cluster + check func(cluster *Cluster, deployment *v1.Service) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + cluster: cluster, + check: noCheck, + }, + { + subTest: "owner reference", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + cluster: cluster, + check: testServiceOwnwerReference, + }, + { + subTest: "selector", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + cluster: cluster, + check: testServiceSelector, + }, + } + for _, tt := range tests { + service := tt.cluster.generateConnPoolService(tt.spec) + + if err := tt.check(cluster, service); err != nil { + t.Errorf("%s [%s]: Service spec is incorrect, %+v", + testName, tt.subTest, err) + } + } +} + func TestTLS(t *testing.T) { var err error var spec acidv1.PostgresSpec diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index d6c2149bf..2e02a9a83 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -90,6 +90,132 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { return statefulSet, nil } +// Prepare the database for connection pool to be used, i.e. install lookup +// function (do it first, because it should be fast and if it didn't succeed, +// it doesn't makes sense to create more K8S objects. At this moment we assume +// that necessary connection pool user exists. +// +// After that create all the objects for connection pool, namely a deployment +// with a chosen pooler and a service to expose it. +func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolObjects, error) { + var msg string + c.setProcessName("creating connection pool") + + schema := c.Spec.ConnectionPool.Schema + if schema == "" { + schema = c.OpConfig.ConnectionPool.Schema + } + + user := c.Spec.ConnectionPool.User + if user == "" { + user = c.OpConfig.ConnectionPool.User + } + + err := lookup(schema, user) + + if err != nil { + msg = "could not prepare database for connection pool: %v" + return nil, fmt.Errorf(msg, err) + } + + deploymentSpec, err := c.generateConnPoolDeployment(&c.Spec) + if err != nil { + msg = "could not generate deployment for connection pool: %v" + return nil, fmt.Errorf(msg, err) + } + + // client-go does retry 10 times (with NoBackoff by default) when the API + // believe a request can be retried and returns Retry-After header. This + // should be good enough to not think about it here. + deployment, err := c.KubeClient. + Deployments(deploymentSpec.Namespace). + Create(deploymentSpec) + + if err != nil { + return nil, err + } + + serviceSpec := c.generateConnPoolService(&c.Spec) + service, err := c.KubeClient. + Services(serviceSpec.Namespace). + Create(serviceSpec) + + if err != nil { + return nil, err + } + + c.ConnectionPool = &ConnectionPoolObjects{ + Deployment: deployment, + Service: service, + } + c.logger.Debugf("created new connection pool %q, uid: %q", + util.NameFromMeta(deployment.ObjectMeta), deployment.UID) + + return c.ConnectionPool, nil +} + +func (c *Cluster) deleteConnectionPool() (err error) { + c.setProcessName("deleting connection pool") + c.logger.Debugln("deleting connection pool") + + // Lack of connection pooler objects is not a fatal error, just log it if + // it was present before in the manifest + if c.ConnectionPool == nil { + c.logger.Infof("No connection pool to delete") + return nil + } + + // Clean up the deployment object. If deployment resource we've remembered + // is somehow empty, try to delete based on what would we generate + deploymentName := c.connPoolName() + deployment := c.ConnectionPool.Deployment + + if deployment != nil { + deploymentName = deployment.Name + } + + // set delete propagation policy to foreground, so that replica set will be + // also deleted. + policy := metav1.DeletePropagationForeground + options := metav1.DeleteOptions{PropagationPolicy: &policy} + err = c.KubeClient. + Deployments(c.Namespace). + Delete(deploymentName, &options) + + if !k8sutil.ResourceNotFound(err) { + c.logger.Debugf("Connection pool deployment was already deleted") + } else if err != nil { + return fmt.Errorf("could not delete deployment: %v", err) + } + + c.logger.Infof("Connection pool deployment %q has been deleted", deploymentName) + + // Repeat the same for the service object + service := c.ConnectionPool.Service + serviceName := c.connPoolName() + + if service != nil { + serviceName = service.Name + } + + // set delete propagation policy to foreground, so that all the dependant + // will be deleted. + err = c.KubeClient. + Services(c.Namespace). + Delete(serviceName, &options) + + if !k8sutil.ResourceNotFound(err) { + c.logger.Debugf("Connection pool service was already deleted") + } else if err != nil { + return fmt.Errorf("could not delete service: %v", err) + } + + c.logger.Infof("Connection pool service %q has been deleted", serviceName) + + c.ConnectionPool = nil + return nil +} + func getPodIndex(podName string) (int32, error) { parts := strings.Split(podName, "-") if len(parts) == 0 { @@ -674,3 +800,34 @@ func (c *Cluster) GetStatefulSet() *appsv1.StatefulSet { func (c *Cluster) GetPodDisruptionBudget() *policybeta1.PodDisruptionBudget { return c.PodDisruptionBudget } + +// Perform actual patching of a connection pool deployment, assuming that all +// the check were already done before. +func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) { + c.setProcessName("updating connection pool") + if c.ConnectionPool == nil || c.ConnectionPool.Deployment == nil { + return nil, fmt.Errorf("there is no connection pool in the cluster") + } + + patchData, err := specPatch(newDeployment.Spec) + if err != nil { + return nil, fmt.Errorf("could not form patch for the deployment: %v", err) + } + + // An update probably requires RetryOnConflict, but since only one operator + // worker at one time will try to update it chances of conflicts are + // minimal. + deployment, err := c.KubeClient. + Deployments(c.ConnectionPool.Deployment.Namespace). + Patch( + c.ConnectionPool.Deployment.Name, + types.MergePatchType, + patchData, "") + if err != nil { + return nil, fmt.Errorf("could not patch deployment: %v", err) + } + + c.ConnectionPool.Deployment = deployment + + return deployment, nil +} diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go new file mode 100644 index 000000000..f06e96e65 --- /dev/null +++ b/pkg/cluster/resources_test.go @@ -0,0 +1,127 @@ +package cluster + +import ( + "testing" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func mockInstallLookupFunction(schema string, user string) error { + return nil +} + +func boolToPointer(value bool) *bool { + return &value +} + +func TestConnPoolCreationAndDeletion(t *testing.T) { + testName := "Test connection pool creation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) + + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + cluster.Spec = acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + } + poolResources, err := cluster.createConnectionPool(mockInstallLookupFunction) + + if err != nil { + t.Errorf("%s: Cannot create connection pool, %s, %+v", + testName, err, poolResources) + } + + if poolResources.Deployment == nil { + t.Errorf("%s: Connection pool deployment is empty", testName) + } + + if poolResources.Service == nil { + t.Errorf("%s: Connection pool service is empty", testName) + } + + err = cluster.deleteConnectionPool() + if err != nil { + t.Errorf("%s: Cannot delete connection pool, %s", testName, err) + } +} + +func TestNeedConnPool(t *testing.T) { + testName := "Test how connection pool can be enabled" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", + }, + }, + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) + + cluster.Spec = acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + } + + if !cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is not enabled with full definition", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPool: boolToPointer(true), + } + + if !cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is not enabled with flag", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPool: boolToPointer(false), + ConnectionPool: &acidv1.ConnectionPool{}, + } + + if cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is still enabled with flag being false", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPool: boolToPointer(true), + ConnectionPool: &acidv1.ConnectionPool{}, + } + + if !cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is not enabled with flag and full", + testName) + } +} diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index b04ff863b..a7c933ae7 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -23,6 +23,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { c.mu.Lock() defer c.mu.Unlock() + oldSpec := c.Postgresql c.setSpec(newSpec) defer func() { @@ -108,6 +109,11 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } + // sync connection pool + if err = c.syncConnectionPool(&oldSpec, newSpec, c.installLookupFunction); err != nil { + return fmt.Errorf("could not sync connection pool: %v", err) + } + return err } @@ -424,7 +430,9 @@ func (c *Cluster) syncSecrets() error { } pwdUser := userMap[secretUsername] // if this secret belongs to the infrastructure role and the password has changed - replace it in the secret - if pwdUser.Password != string(secret.Data["password"]) && pwdUser.Origin == spec.RoleOriginInfrastructure { + if pwdUser.Password != string(secret.Data["password"]) && + pwdUser.Origin == spec.RoleOriginInfrastructure { + c.logger.Debugf("updating the secret %q from the infrastructure roles", secretSpec.Name) if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(secretSpec); err != nil { return fmt.Errorf("could not update infrastructure role secret for role %q: %v", secretUsername, err) @@ -468,6 +476,16 @@ func (c *Cluster) syncRoles() (err error) { for _, u := range c.pgUsers { userNames = append(userNames, u.Name) } + + if c.needConnectionPool() { + connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] + userNames = append(userNames, connPoolUser.Name) + + if _, exists := c.pgUsers[connPoolUser.Name]; !exists { + c.pgUsers[connPoolUser.Name] = connPoolUser + } + } + dbUsers, err = c.readPgUsersFromDatabase(userNames) if err != nil { return fmt.Errorf("error getting users from the database: %v", err) @@ -600,3 +618,165 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } + +func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup InstallFunction) error { + if c.ConnectionPool == nil { + c.ConnectionPool = &ConnectionPoolObjects{} + } + + newNeedConnPool := c.needConnectionPoolWorker(&newSpec.Spec) + oldNeedConnPool := c.needConnectionPoolWorker(&oldSpec.Spec) + + if newNeedConnPool { + // Try to sync in any case. If we didn't needed connection pool before, + // it means we want to create it. If it was already present, still sync + // since it could happen that there is no difference in specs, and all + // the resources are remembered, but the deployment was manualy deleted + // in between + c.logger.Debug("syncing connection pool") + + // in this case also do not forget to install lookup function as for + // creating cluster + if !oldNeedConnPool || !c.ConnectionPool.LookupFunction { + newConnPool := newSpec.Spec.ConnectionPool + + specSchema := "" + specUser := "" + + if newConnPool != nil { + specSchema = newConnPool.Schema + specUser = newConnPool.User + } + + schema := util.Coalesce( + specSchema, + c.OpConfig.ConnectionPool.Schema) + + user := util.Coalesce( + specUser, + c.OpConfig.ConnectionPool.User) + + if err := lookup(schema, user); err != nil { + return err + } + } + + if err := c.syncConnectionPoolWorker(oldSpec, newSpec); err != nil { + c.logger.Errorf("could not sync connection pool: %v", err) + return err + } + } + + if oldNeedConnPool && !newNeedConnPool { + // delete and cleanup resources + if err := c.deleteConnectionPool(); err != nil { + c.logger.Warningf("could not remove connection pool: %v", err) + } + } + + if !oldNeedConnPool && !newNeedConnPool { + // delete and cleanup resources if not empty + if c.ConnectionPool != nil && + (c.ConnectionPool.Deployment != nil || + c.ConnectionPool.Service != nil) { + + if err := c.deleteConnectionPool(); err != nil { + c.logger.Warningf("could not remove connection pool: %v", err) + } + } + } + + return nil +} + +// Synchronize connection pool resources. Effectively we're interested only in +// synchronizing the corresponding deployment, but in case of deployment or +// service is missing, create it. After checking, also remember an object for +// the future references. +func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) error { + deployment, err := c.KubeClient. + Deployments(c.Namespace). + Get(c.connPoolName(), metav1.GetOptions{}) + + if err != nil && k8sutil.ResourceNotFound(err) { + msg := "Deployment %s for connection pool synchronization is not found, create it" + c.logger.Warningf(msg, c.connPoolName()) + + deploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) + if err != nil { + msg = "could not generate deployment for connection pool: %v" + return fmt.Errorf(msg, err) + } + + deployment, err := c.KubeClient. + Deployments(deploymentSpec.Namespace). + Create(deploymentSpec) + + if err != nil { + return err + } + + c.ConnectionPool.Deployment = deployment + } else if err != nil { + return fmt.Errorf("could not get connection pool deployment to sync: %v", err) + } else { + c.ConnectionPool.Deployment = deployment + + // actual synchronization + oldConnPool := oldSpec.Spec.ConnectionPool + newConnPool := newSpec.Spec.ConnectionPool + specSync, specReason := c.needSyncConnPoolSpecs(oldConnPool, newConnPool) + defaultsSync, defaultsReason := c.needSyncConnPoolDefaults(newConnPool, deployment) + reason := append(specReason, defaultsReason...) + if specSync || defaultsSync { + c.logger.Infof("Update connection pool deployment %s, reason: %+v", + c.connPoolName(), reason) + + newDeploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) + if err != nil { + msg := "could not generate deployment for connection pool: %v" + return fmt.Errorf(msg, err) + } + + oldDeploymentSpec := c.ConnectionPool.Deployment + + deployment, err := c.updateConnPoolDeployment( + oldDeploymentSpec, + newDeploymentSpec) + + if err != nil { + return err + } + + c.ConnectionPool.Deployment = deployment + return nil + } + } + + service, err := c.KubeClient. + Services(c.Namespace). + Get(c.connPoolName(), metav1.GetOptions{}) + + if err != nil && k8sutil.ResourceNotFound(err) { + msg := "Service %s for connection pool synchronization is not found, create it" + c.logger.Warningf(msg, c.connPoolName()) + + serviceSpec := c.generateConnPoolService(&newSpec.Spec) + service, err := c.KubeClient. + Services(serviceSpec.Namespace). + Create(serviceSpec) + + if err != nil { + return err + } + + c.ConnectionPool.Service = service + } else if err != nil { + return fmt.Errorf("could not get connection pool service to sync: %v", err) + } else { + // Service updates are not supported and probably not that useful anyway + c.ConnectionPool.Service = service + } + + return nil +} diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go new file mode 100644 index 000000000..483d4ba58 --- /dev/null +++ b/pkg/cluster/sync_test.go @@ -0,0 +1,212 @@ +package cluster + +import ( + "fmt" + "testing" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func int32ToPointer(value int32) *int32 { + return &value +} + +func deploymentUpdated(cluster *Cluster, err error) error { + if cluster.ConnectionPool.Deployment.Spec.Replicas == nil || + *cluster.ConnectionPool.Deployment.Spec.Replicas != 2 { + return fmt.Errorf("Wrong nubmer of instances") + } + + return nil +} + +func objectsAreSaved(cluster *Cluster, err error) error { + if cluster.ConnectionPool == nil { + return fmt.Errorf("Connection pool resources are empty") + } + + if cluster.ConnectionPool.Deployment == nil { + return fmt.Errorf("Deployment was not saved") + } + + if cluster.ConnectionPool.Service == nil { + return fmt.Errorf("Service was not saved") + } + + return nil +} + +func objectsAreDeleted(cluster *Cluster, err error) error { + if cluster.ConnectionPool != nil { + return fmt.Errorf("Connection pool was not deleted") + } + + return nil +} + +func TestConnPoolSynchronization(t *testing.T) { + testName := "Test connection pool synchronization" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", + NumberOfInstances: int32ToPointer(1), + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + clusterMissingObjects := *cluster + clusterMissingObjects.KubeClient = k8sutil.ClientMissingObjects() + + clusterMock := *cluster + clusterMock.KubeClient = k8sutil.NewMockKubernetesClient() + + clusterDirtyMock := *cluster + clusterDirtyMock.KubeClient = k8sutil.NewMockKubernetesClient() + clusterDirtyMock.ConnectionPool = &ConnectionPoolObjects{ + Deployment: &appsv1.Deployment{}, + Service: &v1.Service{}, + } + + clusterNewDefaultsMock := *cluster + clusterNewDefaultsMock.KubeClient = k8sutil.NewMockKubernetesClient() + cluster.OpConfig.ConnectionPool.Image = "pooler:2.0" + cluster.OpConfig.ConnectionPool.NumberOfInstances = int32ToPointer(2) + + tests := []struct { + subTest string + oldSpec *acidv1.Postgresql + newSpec *acidv1.Postgresql + cluster *Cluster + check func(cluster *Cluster, err error) error + }{ + { + subTest: "create if doesn't exist", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + cluster: &clusterMissingObjects, + check: objectsAreSaved, + }, + { + subTest: "create if doesn't exist with a flag", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPool: boolToPointer(true), + }, + }, + cluster: &clusterMissingObjects, + check: objectsAreSaved, + }, + { + subTest: "create from scratch", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + cluster: &clusterMissingObjects, + check: objectsAreSaved, + }, + { + subTest: "delete if not needed", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + cluster: &clusterMock, + check: objectsAreDeleted, + }, + { + subTest: "cleanup if still there", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + cluster: &clusterDirtyMock, + check: objectsAreDeleted, + }, + { + subTest: "update deployment", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{ + NumberOfInstances: int32ToPointer(1), + }, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{ + NumberOfInstances: int32ToPointer(2), + }, + }, + }, + cluster: &clusterMock, + check: deploymentUpdated, + }, + { + subTest: "update image from changed defaults", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + cluster: &clusterNewDefaultsMock, + check: deploymentUpdated, + }, + } + for _, tt := range tests { + err := tt.cluster.syncConnectionPool(tt.oldSpec, tt.newSpec, mockInstallLookupFunction) + + if err := tt.check(tt.cluster, err); err != nil { + t.Errorf("%s [%s]: Could not synchronize, %+v", + testName, tt.subTest, err) + } + } +} diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 138b7015c..04d00cb58 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -69,3 +69,7 @@ type ClusterStatus struct { Spec acidv1.PostgresSpec Error error } + +type TemplateParams map[string]interface{} + +type InstallFunction func(schema string, user string) error diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 8c02fed2e..dc1e93954 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -408,7 +408,32 @@ func (c *Cluster) labelsSet(shouldAddExtraLabels bool) labels.Set { } func (c *Cluster) labelsSelector() *metav1.LabelSelector { - return &metav1.LabelSelector{MatchLabels: c.labelsSet(false), MatchExpressions: nil} + return &metav1.LabelSelector{ + MatchLabels: c.labelsSet(false), + MatchExpressions: nil, + } +} + +// Return connection pool labels selector, which should from one point of view +// inherit most of the labels from the cluster itself, but at the same time +// have e.g. different `application` label, so that recreatePod operation will +// not interfere with it (it lists all the pods via labels, and if there would +// be no difference, it will recreate also pooler pods). +func (c *Cluster) connPoolLabelsSelector() *metav1.LabelSelector { + connPoolLabels := labels.Set(map[string]string{}) + + extraLabels := labels.Set(map[string]string{ + "connection-pool": c.connPoolName(), + "application": "db-connection-pool", + }) + + connPoolLabels = labels.Merge(connPoolLabels, c.labelsSet(false)) + connPoolLabels = labels.Merge(connPoolLabels, extraLabels) + + return &metav1.LabelSelector{ + MatchLabels: connPoolLabels, + MatchExpressions: nil, + } } func (c *Cluster) roleLabelsSet(shouldAddExtraLabels bool, role PostgresRole) labels.Set { @@ -483,3 +508,15 @@ func (c *Cluster) GetSpec() (*acidv1.Postgresql, error) { func (c *Cluster) patroniUsesKubernetes() bool { return c.OpConfig.EtcdHost == "" } + +func (c *Cluster) needConnectionPoolWorker(spec *acidv1.PostgresSpec) bool { + if spec.EnableConnectionPool == nil { + return spec.ConnectionPool != nil + } else { + return *spec.EnableConnectionPool + } +} + +func (c *Cluster) needConnectionPool() bool { + return c.needConnectionPoolWorker(&c.Spec) +} diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 03602c3bd..970eef701 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -8,6 +8,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -21,6 +22,10 @@ func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, con return config, nil } +func int32ToPointer(value int32) *int32 { + return &value +} + // importConfigurationFromCRD is a transitional function that converts CRD configuration to the one based on the configmap func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigurationData) *config.Config { result := &config.Config{} @@ -143,5 +148,51 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ScalyrCPULimit = fromCRD.Scalyr.ScalyrCPULimit result.ScalyrMemoryLimit = fromCRD.Scalyr.ScalyrMemoryLimit + // Connection pool. Looks like we can't use defaulting in CRD before 1.17, + // so ensure default values here. + result.ConnectionPool.NumberOfInstances = util.CoalesceInt32( + fromCRD.ConnectionPool.NumberOfInstances, + int32ToPointer(2)) + + result.ConnectionPool.NumberOfInstances = util.MaxInt32( + result.ConnectionPool.NumberOfInstances, + int32ToPointer(2)) + + result.ConnectionPool.Schema = util.Coalesce( + fromCRD.ConnectionPool.Schema, + constants.ConnectionPoolSchemaName) + + result.ConnectionPool.User = util.Coalesce( + fromCRD.ConnectionPool.User, + constants.ConnectionPoolUserName) + + result.ConnectionPool.Image = util.Coalesce( + fromCRD.ConnectionPool.Image, + "registry.opensource.zalan.do/acid/pgbouncer") + + result.ConnectionPool.Mode = util.Coalesce( + fromCRD.ConnectionPool.Mode, + constants.ConnectionPoolDefaultMode) + + result.ConnectionPool.ConnPoolDefaultCPURequest = util.Coalesce( + fromCRD.ConnectionPool.DefaultCPURequest, + constants.ConnectionPoolDefaultCpuRequest) + + result.ConnectionPool.ConnPoolDefaultMemoryRequest = util.Coalesce( + fromCRD.ConnectionPool.DefaultMemoryRequest, + constants.ConnectionPoolDefaultMemoryRequest) + + result.ConnectionPool.ConnPoolDefaultCPULimit = util.Coalesce( + fromCRD.ConnectionPool.DefaultCPULimit, + constants.ConnectionPoolDefaultCpuLimit) + + result.ConnectionPool.ConnPoolDefaultMemoryLimit = util.Coalesce( + fromCRD.ConnectionPool.DefaultMemoryLimit, + constants.ConnectionPoolDefaultMemoryLimit) + + result.ConnectionPool.MaxDBConnections = util.CoalesceInt32( + fromCRD.ConnectionPool.MaxDBConnections, + int32ToPointer(constants.ConnPoolMaxDBConnections)) + return result } diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 3e6bec8db..36783204d 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -23,13 +23,15 @@ const fileWithNamespace = "/var/run/secrets/kubernetes.io/serviceaccount/namespa // RoleOrigin contains the code of the origin of a role type RoleOrigin int -// The rolesOrigin constant values must be sorted by the role priority for resolveNameConflict(...) to work. +// The rolesOrigin constant values must be sorted by the role priority for +// resolveNameConflict(...) to work. const ( RoleOriginUnknown RoleOrigin = iota RoleOriginManifest RoleOriginInfrastructure RoleOriginTeamsAPI RoleOriginSystem + RoleConnectionPool ) type syncUserOperation int @@ -178,6 +180,8 @@ func (r RoleOrigin) String() string { return "teams API role" case RoleOriginSystem: return "system role" + case RoleConnectionPool: + return "connection pool role" default: panic(fmt.Sprintf("bogus role origin value %d", r)) } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index fee65be81..e0e095617 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util/constants" ) // CRD describes CustomResourceDefinition specific configuration parameters @@ -83,6 +84,20 @@ type LogicalBackup struct { LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:"AES256"` } +// Operator options for connection pooler +type ConnectionPool struct { + NumberOfInstances *int32 `name:"connection_pool_number_of_instances" default:"2"` + Schema string `name:"connection_pool_schema" default:"pooler"` + User string `name:"connection_pool_user" default:"pooler"` + Image string `name:"connection_pool_image" default:"registry.opensource.zalan.do/acid/pgbouncer"` + Mode string `name:"connection_pool_mode" default:"transaction"` + MaxDBConnections *int32 `name:"connection_pool_max_db_connections" default:"60"` + ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"500m"` + ConnPoolDefaultMemoryRequest string `name:"connection_pool_default_memory_request" default:"100Mi"` + ConnPoolDefaultCPULimit string `name:"connection_pool_default_cpu_limit" default:"1"` + ConnPoolDefaultMemoryLimit string `name:"connection_pool_default_memory_limit" default:"100Mi"` +} + // Config describes operator config type Config struct { CRD @@ -90,6 +105,7 @@ type Config struct { Auth Scalyr LogicalBackup + ConnectionPool WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS @@ -196,5 +212,10 @@ func validate(cfg *Config) (err error) { if cfg.Workers == 0 { err = fmt.Errorf("number of workers should be higher than 0") } + + if *cfg.ConnectionPool.NumberOfInstances < constants.ConnPoolMinInstances { + msg := "number of connection pool instances should be higher than %d" + err = fmt.Errorf(msg, constants.ConnPoolMinInstances) + } return } diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go new file mode 100644 index 000000000..540d64e2c --- /dev/null +++ b/pkg/util/constants/pooler.go @@ -0,0 +1,18 @@ +package constants + +// Connection pool specific constants +const ( + ConnectionPoolUserName = "pooler" + ConnectionPoolSchemaName = "pooler" + ConnectionPoolDefaultType = "pgbouncer" + ConnectionPoolDefaultMode = "transaction" + ConnectionPoolDefaultCpuRequest = "500m" + ConnectionPoolDefaultCpuLimit = "1" + ConnectionPoolDefaultMemoryRequest = "100Mi" + ConnectionPoolDefaultMemoryLimit = "100Mi" + + ConnPoolContainer = 0 + ConnPoolMaxDBConnections = 60 + ConnPoolMaxClientConnections = 10000 + ConnPoolMinInstances = 2 +) diff --git a/pkg/util/constants/roles.go b/pkg/util/constants/roles.go index 2c20d69db..3d201142c 100644 --- a/pkg/util/constants/roles.go +++ b/pkg/util/constants/roles.go @@ -2,15 +2,16 @@ package constants // Roles specific constants const ( - PasswordLength = 64 - SuperuserKeyName = "superuser" - ReplicationUserKeyName = "replication" - RoleFlagSuperuser = "SUPERUSER" - RoleFlagInherit = "INHERIT" - RoleFlagLogin = "LOGIN" - RoleFlagNoLogin = "NOLOGIN" - RoleFlagCreateRole = "CREATEROLE" - RoleFlagCreateDB = "CREATEDB" - RoleFlagReplication = "REPLICATION" - RoleFlagByPassRLS = "BYPASSRLS" + PasswordLength = 64 + SuperuserKeyName = "superuser" + ConnectionPoolUserKeyName = "pooler" + ReplicationUserKeyName = "replication" + RoleFlagSuperuser = "SUPERUSER" + RoleFlagInherit = "INHERIT" + RoleFlagLogin = "LOGIN" + RoleFlagNoLogin = "NOLOGIN" + RoleFlagCreateRole = "CREATEROLE" + RoleFlagCreateDB = "CREATEDB" + RoleFlagReplication = "REPLICATION" + RoleFlagByPassRLS = "BYPASSRLS" ) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 509b12c19..75b99ec7c 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -9,11 +9,13 @@ import ( batchv1beta1 "k8s.io/api/batch/v1beta1" clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" + apiappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policybeta1 "k8s.io/api/policy/v1beta1" apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apiextbeta1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" @@ -26,6 +28,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func Int32ToPointer(value int32) *int32 { + return &value +} + // KubernetesClient describes getters for Kubernetes objects type KubernetesClient struct { corev1.SecretsGetter @@ -39,6 +45,7 @@ type KubernetesClient struct { corev1.NamespacesGetter corev1.ServiceAccountsGetter appsv1.StatefulSetsGetter + appsv1.DeploymentsGetter rbacv1.RoleBindingsGetter policyv1beta1.PodDisruptionBudgetsGetter apiextbeta1.CustomResourceDefinitionsGetter @@ -55,6 +62,34 @@ type mockSecret struct { type MockSecretGetter struct { } +type mockDeployment struct { + appsv1.DeploymentInterface +} + +type mockDeploymentNotExist struct { + appsv1.DeploymentInterface +} + +type MockDeploymentGetter struct { +} + +type MockDeploymentNotExistGetter struct { +} + +type mockService struct { + corev1.ServiceInterface +} + +type mockServiceNotExist struct { + corev1.ServiceInterface +} + +type MockServiceGetter struct { +} + +type MockServiceNotExistGetter struct { +} + type mockConfigMap struct { corev1.ConfigMapInterface } @@ -101,6 +136,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.NodesGetter = client.CoreV1() kubeClient.NamespacesGetter = client.CoreV1() kubeClient.StatefulSetsGetter = client.AppsV1() + kubeClient.DeploymentsGetter = client.AppsV1() kubeClient.PodDisruptionBudgetsGetter = client.PolicyV1beta1() kubeClient.RESTClient = client.CoreV1().RESTClient() kubeClient.RoleBindingsGetter = client.RbacV1() @@ -230,19 +266,145 @@ func (c *mockConfigMap) Get(name string, options metav1.GetOptions) (*v1.ConfigM } // Secrets to be mocked -func (c *MockSecretGetter) Secrets(namespace string) corev1.SecretInterface { +func (mock *MockSecretGetter) Secrets(namespace string) corev1.SecretInterface { return &mockSecret{} } // ConfigMaps to be mocked -func (c *MockConfigMapsGetter) ConfigMaps(namespace string) corev1.ConfigMapInterface { +func (mock *MockConfigMapsGetter) ConfigMaps(namespace string) corev1.ConfigMapInterface { return &mockConfigMap{} } +func (mock *MockDeploymentGetter) Deployments(namespace string) appsv1.DeploymentInterface { + return &mockDeployment{} +} + +func (mock *MockDeploymentNotExistGetter) Deployments(namespace string) appsv1.DeploymentInterface { + return &mockDeploymentNotExist{} +} + +func (mock *mockDeployment) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + Spec: apiappsv1.DeploymentSpec{ + Replicas: Int32ToPointer(1), + }, + }, nil +} + +func (mock *mockDeployment) Delete(name string, opts *metav1.DeleteOptions) error { + return nil +} + +func (mock *mockDeployment) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + Spec: apiappsv1.DeploymentSpec{ + Replicas: Int32ToPointer(1), + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + Image: "pooler:1.0", + }, + }, + }, + }, + }, + }, nil +} + +func (mock *mockDeployment) Patch(name string, t types.PatchType, data []byte, subres ...string) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + Spec: apiappsv1.DeploymentSpec{ + Replicas: Int32ToPointer(2), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + }, nil +} + +func (mock *mockDeploymentNotExist) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { + return nil, &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonNotFound, + }, + } +} + +func (mock *mockDeploymentNotExist) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + Spec: apiappsv1.DeploymentSpec{ + Replicas: Int32ToPointer(1), + }, + }, nil +} + +func (mock *MockServiceGetter) Services(namespace string) corev1.ServiceInterface { + return &mockService{} +} + +func (mock *MockServiceNotExistGetter) Services(namespace string) corev1.ServiceInterface { + return &mockServiceNotExist{} +} + +func (mock *mockService) Create(*v1.Service) (*v1.Service, error) { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + }, + }, nil +} + +func (mock *mockService) Delete(name string, opts *metav1.DeleteOptions) error { + return nil +} + +func (mock *mockService) Get(name string, opts metav1.GetOptions) (*v1.Service, error) { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + }, + }, nil +} + +func (mock *mockServiceNotExist) Create(*v1.Service) (*v1.Service, error) { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + }, + }, nil +} + +func (mock *mockServiceNotExist) Get(name string, opts metav1.GetOptions) (*v1.Service, error) { + return nil, &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonNotFound, + }, + } +} + // NewMockKubernetesClient for other tests func NewMockKubernetesClient() KubernetesClient { return KubernetesClient{ - SecretsGetter: &MockSecretGetter{}, - ConfigMapsGetter: &MockConfigMapsGetter{}, + SecretsGetter: &MockSecretGetter{}, + ConfigMapsGetter: &MockConfigMapsGetter{}, + DeploymentsGetter: &MockDeploymentGetter{}, + ServicesGetter: &MockServiceGetter{}, + } +} + +func ClientMissingObjects() KubernetesClient { + return KubernetesClient{ + DeploymentsGetter: &MockDeploymentNotExistGetter{}, + ServicesGetter: &MockServiceNotExistGetter{}, } } diff --git a/pkg/util/util.go b/pkg/util/util.go index d9803ab48..46df5d345 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -147,6 +147,40 @@ func Coalesce(val, defaultVal string) string { return val } +// Yeah, golang +func CoalesceInt32(val, defaultVal *int32) *int32 { + if val == nil { + return defaultVal + } + return val +} + +// Test if any of the values is nil +func testNil(values ...*int32) bool { + for _, v := range values { + if v == nil { + return true + } + } + + return false +} + +// Return maximum of two integers provided via pointers. If one value is not +// defined, return the other one. If both are not defined, result is also +// undefined, caller needs to check for that. +func MaxInt32(a, b *int32) *int32 { + if testNil(a, b) { + return nil + } + + if *a > *b { + return a + } + + return b +} + // IsSmallerQuantity : checks if first resource is of a smaller quantity than the second func IsSmallerQuantity(requestStr, limitStr string) (bool, error) { From ba9cf686502077babbda1803e2b54b83c6a153b9 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 25 Mar 2020 15:59:31 +0100 Subject: [PATCH 069/104] Change type of pod environment config map to NamespacedName (#870) * allow PodEnvironmentConfigMap in other namespaces * update codegen * update docs and comments --- charts/postgres-operator/values-crd.yaml | 8 +-- charts/postgres-operator/values.yaml | 8 +-- docs/administrator.md | 15 +++--- docs/reference/operator_parameters.md | 17 ++++--- manifests/configmap.yaml | 2 +- ...gresql-operator-default-configuration.yaml | 2 +- .../v1/operator_configuration_type.go | 15 +++--- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 1 + pkg/cluster/k8sres.go | 19 ++++--- pkg/util/config/config.go | 50 +++++++++---------- 10 files changed, 73 insertions(+), 64 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index cf9fbab15..79940b236 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -71,7 +71,7 @@ configKubernetes: enable_pod_disruption_budget: true # enables sidecar containers to run alongside Spilo in the same pod enable_sidecars: true - # name of the secret containing infrastructure roles names and passwords + # namespaced name of the secret containing infrastructure roles names and passwords # infrastructure_roles_secret_name: postgresql-infrastructure-roles # list of labels that can be inherited from the cluster manifest @@ -86,15 +86,15 @@ configKubernetes: # node_readiness_label: # status: ready - # name of the secret containing the OAuth2 token to pass to the teams API + # namespaced name of the secret containing the OAuth2 token to pass to the teams API # oauth_token_secret_name: postgresql-operator # defines the template for PDB (Pod Disruption Budget) names pdb_name_format: "postgres-{cluster}-pdb" # override topology key for pod anti affinity pod_antiaffinity_topology_key: "kubernetes.io/hostname" - # name of the ConfigMap with environment variables to populate on every pod - # pod_environment_configmap: "" + # namespaced name of the ConfigMap with environment variables to populate on every pod + # pod_environment_configmap: "default/my-custom-config" # specify the pod management policy of stateful sets of Postgres clusters pod_management_policy: "ordered_ready" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 503bf4562..29f85339d 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -67,7 +67,7 @@ configKubernetes: enable_pod_disruption_budget: "true" # enables sidecar containers to run alongside Spilo in the same pod enable_sidecars: "true" - # name of the secret containing infrastructure roles names and passwords + # namespaced name of the secret containing infrastructure roles names and passwords # infrastructure_roles_secret_name: postgresql-infrastructure-roles # list of labels that can be inherited from the cluster manifest @@ -79,15 +79,15 @@ configKubernetes: # set of labels that a running and active node should possess to be considered ready # node_readiness_label: "" - # name of the secret containing the OAuth2 token to pass to the teams API + # namespaced name of the secret containing the OAuth2 token to pass to the teams API # oauth_token_secret_name: postgresql-operator # defines the template for PDB (Pod Disruption Budget) names pdb_name_format: "postgres-{cluster}-pdb" # override topology key for pod anti affinity pod_antiaffinity_topology_key: "kubernetes.io/hostname" - # name of the ConfigMap with environment variables to populate on every pod - # pod_environment_configmap: "" + # namespaced name of the ConfigMap with environment variables to populate on every pod + # pod_environment_configmap: "default/my-custom-config" # specify the pod management policy of stateful sets of Postgres clusters pod_management_policy: "ordered_ready" diff --git a/docs/administrator.md b/docs/administrator.md index a3a0f70cc..93adf2eb1 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -321,11 +321,12 @@ spec: ## Custom Pod Environment Variables It is possible to configure a ConfigMap which is used by the Postgres pods as -an additional provider for environment variables. - -One use case is to customize the Spilo image and configure it with environment -variables. The ConfigMap with the additional settings is configured in the -operator's main ConfigMap: +an additional provider for environment variables. One use case is to customize +the Spilo image and configure it with environment variables. The ConfigMap with +the additional settings is referenced in the operator's main configuration. +A namespace can be specified along with the name. If left out, the configured +default namespace of your K8s client will be used and if the ConfigMap is not +found there, the Postgres cluster's namespace is taken when different: **postgres-operator ConfigMap** @@ -336,7 +337,7 @@ metadata: name: postgres-operator data: # referencing config map with custom settings - pod_environment_configmap: postgres-pod-config + pod_environment_configmap: default/postgres-pod-config ``` **OperatorConfiguration** @@ -349,7 +350,7 @@ metadata: configuration: kubernetes: # referencing config map with custom settings - pod_environment_configmap: postgres-pod-config + pod_environment_configmap: default/postgres-pod-config ``` **referenced ConfigMap `postgres-pod-config`** diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 848fa1cf2..1ab92a287 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -221,11 +221,12 @@ configuration they are grouped under the `kubernetes` key. to the Postgres clusters after creation. * **oauth_token_secret_name** - a name of the secret containing the `OAuth2` token to pass to the teams API. - The default is `postgresql-operator`. + namespaced name of the secret containing the `OAuth2` token to pass to the + teams API. The default is `postgresql-operator`. * **infrastructure_roles_secret_name** - name of the secret containing infrastructure roles names and passwords. + namespaced name of the secret containing infrastructure roles names and + passwords. * **pod_role_label** name of the label assigned to the Postgres pods (and services/endpoints) by @@ -262,11 +263,11 @@ configuration they are grouped under the `kubernetes` key. for details on taints and tolerations. The default is empty. * **pod_environment_configmap** - a name of the ConfigMap with environment variables to populate on every pod. - Right now this ConfigMap is searched in the namespace of the Postgres cluster. - All variables from that ConfigMap are injected to the pod's environment, on - conflicts they are overridden by the environment variables generated by the - operator. The default is empty. + namespaced name of the ConfigMap with environment variables to populate on + every pod. Right now this ConfigMap is searched in the namespace of the + Postgres cluster. All variables from that ConfigMap are injected to the pod's + environment, on conflicts they are overridden by the environment variables + generated by the operator. The default is empty. * **pod_priority_class_name** a name of the [priority class](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass) diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index fdc2d5d56..67c3368f3 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -69,7 +69,7 @@ data: pdb_name_format: "postgres-{cluster}-pdb" # pod_antiaffinity_topology_key: "kubernetes.io/hostname" pod_deletion_wait_timeout: 10m - # pod_environment_configmap: "" + # pod_environment_configmap: "default/my-custom-config" pod_label_wait_timeout: 10m pod_management_policy: "ordered_ready" pod_role_label: spilo-role diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index d4c9b518f..9d609713c 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -40,7 +40,7 @@ configuration: oauth_token_secret_name: postgresql-operator pdb_name_format: "postgres-{cluster}-pdb" pod_antiaffinity_topology_key: "kubernetes.io/hostname" - # pod_environment_configmap: "" + # pod_environment_configmap: "default/my-custom-config" pod_management_policy: "ordered_ready" # pod_priority_class_name: "" pod_role_label: spilo-role diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 7c1d2b7e8..3dbe96b7f 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -65,14 +65,13 @@ type KubernetesMetaConfiguration struct { NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` // TODO: use a proper toleration structure? - PodToleration map[string]string `json:"toleration,omitempty"` - // TODO: use namespacedname - PodEnvironmentConfigMap string `json:"pod_environment_configmap,omitempty"` - PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` - MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` - EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` - PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` - PodManagementPolicy string `json:"pod_management_policy,omitempty"` + PodToleration map[string]string `json:"toleration,omitempty"` + PodEnvironmentConfigMap spec.NamespacedName `json:"pod_environment_configmap,omitempty"` + PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` + MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` + EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` + PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` + PodManagementPolicy string `json:"pod_management_policy,omitempty"` } // PostgresPodResourcesDefaults defines the spec of default resources diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index fcab394ca..65a19600a 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -179,6 +179,7 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura (*out)[key] = val } } + out.PodEnvironmentConfigMap = in.PodEnvironmentConfigMap return } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 34b409d4e..2c40bb0ba 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -18,6 +18,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" + pkgspec "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" @@ -485,9 +486,9 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar, // Check whether or not we're requested to mount an shm volume, // taking into account that PostgreSQL manifest has precedence. -func mountShmVolumeNeeded(opConfig config.Config, pgSpec *acidv1.PostgresSpec) *bool { - if pgSpec.ShmVolume != nil && *pgSpec.ShmVolume { - return pgSpec.ShmVolume +func mountShmVolumeNeeded(opConfig config.Config, spec *acidv1.PostgresSpec) *bool { + if spec.ShmVolume != nil && *spec.ShmVolume { + return spec.ShmVolume } return opConfig.ShmVolume @@ -911,11 +912,17 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef customPodEnvVarsList := make([]v1.EnvVar, 0) - if c.OpConfig.PodEnvironmentConfigMap != "" { + if c.OpConfig.PodEnvironmentConfigMap != (pkgspec.NamespacedName{}) { var cm *v1.ConfigMap - cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get(c.OpConfig.PodEnvironmentConfigMap, metav1.GetOptions{}) + cm, err = c.KubeClient.ConfigMaps(c.OpConfig.PodEnvironmentConfigMap.Namespace).Get(c.OpConfig.PodEnvironmentConfigMap.Name, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err) + // if not found, try again using the cluster's namespace if it's different (old behavior) + if k8sutil.ResourceNotFound(err) && c.Namespace != c.OpConfig.PodEnvironmentConfigMap.Namespace { + cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get(c.OpConfig.PodEnvironmentConfigMap.Name, metav1.GetOptions{}) + } + if err != nil { + return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err) + } } for k, v := range cm.Data { customPodEnvVarsList = append(customPodEnvVarsList, v1.EnvVar{Name: k, Value: v}) diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index e0e095617..403615f06 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -22,31 +22,31 @@ type CRD struct { // Resources describes kubernetes resource specific configuration parameters type Resources struct { - ResourceCheckInterval time.Duration `name:"resource_check_interval" default:"3s"` - ResourceCheckTimeout time.Duration `name:"resource_check_timeout" default:"10m"` - PodLabelWaitTimeout time.Duration `name:"pod_label_wait_timeout" default:"10m"` - PodDeletionWaitTimeout time.Duration `name:"pod_deletion_wait_timeout" default:"10m"` - PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` - SpiloFSGroup *int64 `name:"spilo_fsgroup"` - PodPriorityClassName string `name:"pod_priority_class_name"` - ClusterDomain string `name:"cluster_domain" default:"cluster.local"` - SpiloPrivileged bool `name:"spilo_privileged" default:"false"` - ClusterLabels map[string]string `name:"cluster_labels" default:"application:spilo"` - InheritedLabels []string `name:"inherited_labels" default:""` - ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` - PodRoleLabel string `name:"pod_role_label" default:"spilo-role"` - PodToleration map[string]string `name:"toleration" default:""` - DefaultCPURequest string `name:"default_cpu_request" default:"100m"` - DefaultMemoryRequest string `name:"default_memory_request" default:"100Mi"` - DefaultCPULimit string `name:"default_cpu_limit" default:"1"` - DefaultMemoryLimit string `name:"default_memory_limit" default:"500Mi"` - MinCPULimit string `name:"min_cpu_limit" default:"250m"` - MinMemoryLimit string `name:"min_memory_limit" default:"250Mi"` - PodEnvironmentConfigMap string `name:"pod_environment_configmap" default:""` - NodeReadinessLabel map[string]string `name:"node_readiness_label" default:""` - MaxInstances int32 `name:"max_instances" default:"-1"` - MinInstances int32 `name:"min_instances" default:"-1"` - ShmVolume *bool `name:"enable_shm_volume" default:"true"` + ResourceCheckInterval time.Duration `name:"resource_check_interval" default:"3s"` + ResourceCheckTimeout time.Duration `name:"resource_check_timeout" default:"10m"` + PodLabelWaitTimeout time.Duration `name:"pod_label_wait_timeout" default:"10m"` + PodDeletionWaitTimeout time.Duration `name:"pod_deletion_wait_timeout" default:"10m"` + PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` + SpiloFSGroup *int64 `name:"spilo_fsgroup"` + PodPriorityClassName string `name:"pod_priority_class_name"` + ClusterDomain string `name:"cluster_domain" default:"cluster.local"` + SpiloPrivileged bool `name:"spilo_privileged" default:"false"` + ClusterLabels map[string]string `name:"cluster_labels" default:"application:spilo"` + InheritedLabels []string `name:"inherited_labels" default:""` + ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` + PodRoleLabel string `name:"pod_role_label" default:"spilo-role"` + PodToleration map[string]string `name:"toleration" default:""` + DefaultCPURequest string `name:"default_cpu_request" default:"100m"` + DefaultMemoryRequest string `name:"default_memory_request" default:"100Mi"` + DefaultCPULimit string `name:"default_cpu_limit" default:"1"` + DefaultMemoryLimit string `name:"default_memory_limit" default:"500Mi"` + MinCPULimit string `name:"min_cpu_limit" default:"250m"` + MinMemoryLimit string `name:"min_memory_limit" default:"250Mi"` + PodEnvironmentConfigMap spec.NamespacedName `name:"pod_environment_configmap"` + NodeReadinessLabel map[string]string `name:"node_readiness_label" default:""` + MaxInstances int32 `name:"max_instances" default:"-1"` + MinInstances int32 `name:"min_instances" default:"-1"` + ShmVolume *bool `name:"enable_shm_volume" default:"true"` } // Auth describes authentication specific configuration parameters From 66f2cda87fb041a6cc0151d2b442aad2c9755075 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 30 Mar 2020 15:50:17 +0200 Subject: [PATCH 070/104] Move operator to go 1.14 (#882) * update go modules march 2020 * update to GO 1.14 * reflect k8s client API changes --- .travis.yml | 2 +- delivery.yaml | 2 +- go.mod | 31 ++-- go.sum | 168 +++++++++--------- pkg/cluster/cluster.go | 20 ++- pkg/cluster/exec.go | 5 +- pkg/cluster/k8sres.go | 11 +- pkg/cluster/pod.go | 21 ++- pkg/cluster/resources.go | 85 +++++---- pkg/cluster/sync.go | 33 ++-- pkg/cluster/util.go | 11 +- pkg/cluster/volumes.go | 11 +- pkg/controller/controller.go | 5 +- pkg/controller/node.go | 7 +- pkg/controller/operator_config.go | 4 +- pkg/controller/pod.go | 8 +- pkg/controller/postgresql.go | 11 +- pkg/controller/util.go | 13 +- .../clientset/versioned/clientset.go | 2 +- .../v1/fake/fake_operatorconfiguration.go | 4 +- .../acid.zalan.do/v1/fake/fake_postgresql.go | 22 +-- .../acid.zalan.do/v1/operatorconfiguration.go | 8 +- .../typed/acid.zalan.do/v1/postgresql.go | 72 ++++---- .../acid.zalan.do/v1/postgresql.go | 5 +- pkg/util/k8sutil/k8sutil.go | 27 +-- pkg/util/teams/teams_test.go | 4 +- 26 files changed, 319 insertions(+), 273 deletions(-) diff --git a/.travis.yml b/.travis.yml index 589eb03a4..091c875ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ branches: language: go go: - - "1.12.x" + - "1.14.x" before_install: - go get github.com/mattn/goveralls diff --git a/delivery.yaml b/delivery.yaml index 144448ea9..d0884f982 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -12,7 +12,7 @@ pipeline: - desc: 'Install go' cmd: | cd /tmp - wget -q https://storage.googleapis.com/golang/go1.12.linux-amd64.tar.gz -O go.tar.gz + wget -q https://storage.googleapis.com/golang/go1.14.linux-amd64.tar.gz -O go.tar.gz tar -xf go.tar.gz mv go /usr/local ln -s /usr/local/go/bin/go /usr/bin/go diff --git a/go.mod b/go.mod index be1ec32d4..8b9f55509 100644 --- a/go.mod +++ b/go.mod @@ -1,26 +1,19 @@ module github.com/zalando/postgres-operator -go 1.12 +go 1.14 require ( - github.com/aws/aws-sdk-go v1.25.44 - github.com/emicklei/go-restful v2.9.6+incompatible // indirect - github.com/evanphx/json-patch v4.5.0+incompatible // indirect - github.com/googleapis/gnostic v0.3.0 // indirect - github.com/imdario/mergo v0.3.8 // indirect - github.com/lib/pq v1.2.0 + github.com/aws/aws-sdk-go v1.29.33 + github.com/lib/pq v1.3.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d - github.com/sirupsen/logrus v1.4.2 + github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a + github.com/sirupsen/logrus v1.5.0 github.com/stretchr/testify v1.4.0 - golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect - golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect - golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect - golang.org/x/tools v0.0.0-20191209225234-22774f7dae43 // indirect - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v2 v2.2.4 - k8s.io/api v0.0.0-20191121015604-11707872ac1c - k8s.io/apiextensions-apiserver v0.0.0-20191204090421-cd61debedab5 - k8s.io/apimachinery v0.0.0-20191203211716-adc6f4cd9e7d - k8s.io/client-go v0.0.0-20191204082520-bc9b51d240b2 - k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e + golang.org/x/tools v0.0.0-20200326210457-5d86d385bf88 // indirect + gopkg.in/yaml.v2 v2.2.8 + k8s.io/api v0.18.0 + k8s.io/apiextensions-apiserver v0.18.0 + k8s.io/apimachinery v0.18.0 + k8s.io/client-go v0.18.0 + k8s.io/code-generator v0.18.0 ) diff --git a/go.sum b/go.sum index 0737d3a5d..e8607922b 100644 --- a/go.sum +++ b/go.sum @@ -11,7 +11,6 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -27,12 +26,13 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.25.44 h1:n9ahFoiyn66smjF34hYr3tb6/ZdBcLuFz7BCDhHyJ7I= -github.com/aws/aws-sdk-go v1.25.44/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.29.33 h1:WP85+WHalTFQR2wYp5xR2sjiVAZXew2bBQXGU1QJBXI= +github.com/aws/aws-sdk-go v1.29.33/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -46,7 +46,6 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -58,15 +57,15 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QL github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w= -github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= -github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -124,11 +123,12 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -136,6 +136,7 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -144,18 +145,20 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= -github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -169,15 +172,14 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -195,8 +197,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -214,7 +216,6 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -227,8 +228,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -236,9 +237,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= @@ -246,26 +246,30 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a h1:2v4Ipjxa3sh+xn6GvtgrMub2ci4ZLQMvTaYIba2lfdc= +github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a/go.mod h1:ozniNEFS3j1qCwHKdvraMn1WJOsUxHd7lYfukEIS4cs= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= +github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -273,7 +277,6 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -286,6 +289,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -302,19 +306,17 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 h1:I6A9Ag9FpEKOjcKrRNjQkPHawoXIhKyTGfvvjFAiiAk= -golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -332,8 +334,9 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -343,6 +346,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -352,20 +356,21 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -375,23 +380,20 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191209225234-22774f7dae43 h1:NfPq5mgc5ArFgVLCpeS4z07IoxSAqVfV/gQ5vxdgaxI= -golang.org/x/tools v0.0.0-20191209225234-22774f7dae43/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200326210457-5d86d385bf88 h1:F7fM2kxXfuWw820fa+MMCCLH6hmYe+jtLnZpwoiLK4Q= +golang.org/x/tools v0.0.0-20200326210457-5d86d385bf88/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 h1:OB/uP/Puiu5vS5QMRPrXCDWUPb+kt8f1KW8oQzFejQw= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e h1:jRyg0XfpwWlhEV8mDfdNGBeSJM2fuyh9Yjrnd8kF2Ts= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -400,14 +402,15 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -423,45 +426,40 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.0.0-20191121015604-11707872ac1c h1:Z87my3sF4WhG0OMxzARkWY/IKBtOr+MhXZAb4ts6qFc= -k8s.io/api v0.0.0-20191121015604-11707872ac1c/go.mod h1:R/s4gKT0V/cWEnbQa9taNRJNbWUK57/Dx6cPj6MD3A0= -k8s.io/apiextensions-apiserver v0.0.0-20191204090421-cd61debedab5 h1:g+GvnbGqLU1Jxb/9iFm/BFcmkqG9HdsGh52+wHirpsM= -k8s.io/apiextensions-apiserver v0.0.0-20191204090421-cd61debedab5/go.mod h1:CPw0IHz1YrWGy0+8mG/76oTHXvChlgCb3EAezKQKB2I= -k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.0.0-20191123233150-4c4803ed55e3/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.0.0-20191128180518-03184f823e28/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.0.0-20191203211716-adc6f4cd9e7d h1:q+OZmYewHJeMCzwpHkXlNTtk5bvaUMPCikKvf77RBlo= -k8s.io/apimachinery v0.0.0-20191203211716-adc6f4cd9e7d/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apiserver v0.0.0-20191204084332-137a9d3b886b/go.mod h1:itgfam5HJbT/4b2BGfpUkkxfheMmDH+Ix+tEAP3uqZk= -k8s.io/client-go v0.0.0-20191204082517-8c19b9f4a642/go.mod h1:HMVIZ0dPop3WCrPEaJ+v5/94cjt56avdDFshpX0Fjvo= -k8s.io/client-go v0.0.0-20191204082519-e9644b2e3edc/go.mod h1:5lSG1yeDZVwDYAHe9VK48SCe5zmcnkAcf2Mx59TuhmM= -k8s.io/client-go v0.0.0-20191204082520-bc9b51d240b2 h1:T2HGghBOPAOEjWuIyFSeCsWEwsxa6unkBvy3PHfqonM= -k8s.io/client-go v0.0.0-20191204082520-bc9b51d240b2/go.mod h1:5lSG1yeDZVwDYAHe9VK48SCe5zmcnkAcf2Mx59TuhmM= -k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e h1:HB9Zu5ZUvJfNpLiTPhz+CebVKV8C39qTBMQkAgAZLNw= -k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= -k8s.io/component-base v0.0.0-20191204083903-0d4d24e738e4/go.mod h1:8VIh1jErItC4bg9hLBkPneyS77Tin8KwSzbYepHJnQI= -k8s.io/component-base v0.0.0-20191204083906-3ac1376c73aa/go.mod h1:mECWvHCPhJudDVDMtBl+AIf/YnTMp5r1F947OYFUwP0= +k8s.io/api v0.18.0 h1:lwYk8Vt7rsVTwjRU6pzEsa9YNhThbmbocQlKvNBB4EQ= +k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= +k8s.io/apiextensions-apiserver v0.18.0 h1:HN4/P8vpGZFvB5SOMuPPH2Wt9Y/ryX+KRvIyAkchu1Q= +k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= +k8s.io/apimachinery v0.18.0 h1:fuPfYpk3cs1Okp/515pAf0dNhL66+8zk8RLbSX+EgAE= +k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= +k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= +k8s.io/client-go v0.18.0 h1:yqKw4cTUQraZK3fcVCMeSa+lqKwcjZ5wtcOIPnxQno4= +k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8= +k8s.io/code-generator v0.18.0 h1:0xIRWzym+qMgVpGmLESDeMfz/orwgxwxFFAo1xfGNtQ= +k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= +k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM= -k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4= +k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= -k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= -k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= -modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= -sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c h1:/KUFqjjqAcY4Us6luF5RDNZ16KJtb49HfR3ZHB9qYXM= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index dba67c142..45ee55300 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -3,6 +3,7 @@ package cluster // Postgres CustomResourceDefinition object i.e. Spilo import ( + "context" "database/sql" "encoding/json" "fmt" @@ -88,7 +89,7 @@ type Cluster struct { pgDb *sql.DB mu sync.Mutex userSyncStrategy spec.UserSyncer - deleteOptions *metav1.DeleteOptions + deleteOptions metav1.DeleteOptions podEventsQueue *cache.FIFO teamsAPIClient teams.Interface @@ -131,7 +132,7 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres Services: make(map[PostgresRole]*v1.Service), Endpoints: make(map[PostgresRole]*v1.Endpoints)}, userSyncStrategy: users.DefaultUserSyncStrategy{}, - deleteOptions: &metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}, + deleteOptions: metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}, podEventsQueue: podEventsQueue, KubeClient: kubeClient, } @@ -182,7 +183,8 @@ func (c *Cluster) setStatus(status string) { // we cannot do a full scale update here without fetching the previous manifest (as the resourceVersion may differ), // however, we could do patch without it. In the future, once /status subresource is there (starting Kubernetes 1.11) // we should take advantage of it. - newspec, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.clusterNamespace()).Patch(c.Name, types.MergePatchType, patch, "status") + newspec, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.clusterNamespace()).Patch( + context.TODO(), c.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "status") if err != nil { c.logger.Errorf("could not update status: %v", err) // return as newspec is empty, see PR654 @@ -1185,12 +1187,12 @@ func (c *Cluster) deleteClusterObject( func (c *Cluster) deletePatroniClusterServices() error { get := func(name string) (spec.NamespacedName, error) { - svc, err := c.KubeClient.Services(c.Namespace).Get(name, metav1.GetOptions{}) + svc, err := c.KubeClient.Services(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) return util.NameFromMeta(svc.ObjectMeta), err } deleteServiceFn := func(name string) error { - return c.KubeClient.Services(c.Namespace).Delete(name, c.deleteOptions) + return c.KubeClient.Services(c.Namespace).Delete(context.TODO(), name, c.deleteOptions) } return c.deleteClusterObject(get, deleteServiceFn, "service") @@ -1198,12 +1200,12 @@ func (c *Cluster) deletePatroniClusterServices() error { func (c *Cluster) deletePatroniClusterEndpoints() error { get := func(name string) (spec.NamespacedName, error) { - ep, err := c.KubeClient.Endpoints(c.Namespace).Get(name, metav1.GetOptions{}) + ep, err := c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) return util.NameFromMeta(ep.ObjectMeta), err } deleteEndpointFn := func(name string) error { - return c.KubeClient.Endpoints(c.Namespace).Delete(name, c.deleteOptions) + return c.KubeClient.Endpoints(c.Namespace).Delete(context.TODO(), name, c.deleteOptions) } return c.deleteClusterObject(get, deleteEndpointFn, "endpoint") @@ -1211,12 +1213,12 @@ func (c *Cluster) deletePatroniClusterEndpoints() error { func (c *Cluster) deletePatroniClusterConfigMaps() error { get := func(name string) (spec.NamespacedName, error) { - cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(name, metav1.GetOptions{}) + cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) return util.NameFromMeta(cm.ObjectMeta), err } deleteConfigMapFn := func(name string) error { - return c.KubeClient.ConfigMaps(c.Namespace).Delete(name, c.deleteOptions) + return c.KubeClient.ConfigMaps(c.Namespace).Delete(context.TODO(), name, c.deleteOptions) } return c.deleteClusterObject(get, deleteConfigMapFn, "configmap") diff --git a/pkg/cluster/exec.go b/pkg/cluster/exec.go index 8dd6bd91d..8b5089b4e 100644 --- a/pkg/cluster/exec.go +++ b/pkg/cluster/exec.go @@ -2,10 +2,11 @@ package cluster import ( "bytes" + "context" "fmt" "strings" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/remotecommand" @@ -23,7 +24,7 @@ func (c *Cluster) ExecCommand(podName *spec.NamespacedName, command ...string) ( execErr bytes.Buffer ) - pod, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{}) + pod, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{}) if err != nil { return "", fmt.Errorf("could not get pod info: %v", err) } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 2c40bb0ba..c4919c62d 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1,6 +1,7 @@ package cluster import ( + "context" "encoding/json" "fmt" "path" @@ -914,11 +915,17 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef if c.OpConfig.PodEnvironmentConfigMap != (pkgspec.NamespacedName{}) { var cm *v1.ConfigMap - cm, err = c.KubeClient.ConfigMaps(c.OpConfig.PodEnvironmentConfigMap.Namespace).Get(c.OpConfig.PodEnvironmentConfigMap.Name, metav1.GetOptions{}) + cm, err = c.KubeClient.ConfigMaps(c.OpConfig.PodEnvironmentConfigMap.Namespace).Get( + context.TODO(), + c.OpConfig.PodEnvironmentConfigMap.Name, + metav1.GetOptions{}) if err != nil { // if not found, try again using the cluster's namespace if it's different (old behavior) if k8sutil.ResourceNotFound(err) && c.Namespace != c.OpConfig.PodEnvironmentConfigMap.Namespace { - cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get(c.OpConfig.PodEnvironmentConfigMap.Name, metav1.GetOptions{}) + cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get( + context.TODO(), + c.OpConfig.PodEnvironmentConfigMap.Name, + metav1.GetOptions{}) } if err != nil { return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err) diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index 095f859f0..9991621cc 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -1,6 +1,7 @@ package cluster import ( + "context" "fmt" "math/rand" @@ -17,7 +18,7 @@ func (c *Cluster) listPods() ([]v1.Pod, error) { LabelSelector: c.labelsSet(false).String(), } - pods, err := c.KubeClient.Pods(c.Namespace).List(listOptions) + pods, err := c.KubeClient.Pods(c.Namespace).List(context.TODO(), listOptions) if err != nil { return nil, fmt.Errorf("could not get list of pods: %v", err) } @@ -30,7 +31,7 @@ func (c *Cluster) getRolePods(role PostgresRole) ([]v1.Pod, error) { LabelSelector: c.roleLabelsSet(false, role).String(), } - pods, err := c.KubeClient.Pods(c.Namespace).List(listOptions) + pods, err := c.KubeClient.Pods(c.Namespace).List(context.TODO(), listOptions) if err != nil { return nil, fmt.Errorf("could not get list of pods: %v", err) } @@ -73,7 +74,7 @@ func (c *Cluster) deletePod(podName spec.NamespacedName) error { ch := c.registerPodSubscriber(podName) defer c.unregisterPodSubscriber(podName) - if err := c.KubeClient.Pods(podName.Namespace).Delete(podName.Name, c.deleteOptions); err != nil { + if err := c.KubeClient.Pods(podName.Namespace).Delete(context.TODO(), podName.Name, c.deleteOptions); err != nil { return err } @@ -183,7 +184,7 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { eol bool ) - oldMaster, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{}) + oldMaster, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{}) if err != nil { return fmt.Errorf("could not get pod: %v", err) @@ -206,7 +207,9 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { // we must have a statefulset in the cluster for the migration to work if c.Statefulset == nil { var sset *appsv1.StatefulSet - if sset, err = c.KubeClient.StatefulSets(c.Namespace).Get(c.statefulSetName(), + if sset, err = c.KubeClient.StatefulSets(c.Namespace).Get( + context.TODO(), + c.statefulSetName(), metav1.GetOptions{}); err != nil { return fmt.Errorf("could not retrieve cluster statefulset: %v", err) } @@ -247,7 +250,7 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { // MigrateReplicaPod recreates pod on a new node func (c *Cluster) MigrateReplicaPod(podName spec.NamespacedName, fromNodeName string) error { - replicaPod, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{}) + replicaPod, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{}) if err != nil { return fmt.Errorf("could not get pod: %v", err) } @@ -276,7 +279,7 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) { defer c.unregisterPodSubscriber(podName) stopChan := make(chan struct{}) - if err := c.KubeClient.Pods(podName.Namespace).Delete(podName.Name, c.deleteOptions); err != nil { + if err := c.KubeClient.Pods(podName.Namespace).Delete(context.TODO(), podName.Name, c.deleteOptions); err != nil { return nil, fmt.Errorf("could not delete pod: %v", err) } @@ -300,7 +303,7 @@ func (c *Cluster) recreatePods() error { LabelSelector: ls.String(), } - pods, err := c.KubeClient.Pods(namespace).List(listOptions) + pods, err := c.KubeClient.Pods(namespace).List(context.TODO(), listOptions) if err != nil { return fmt.Errorf("could not get the list of pods: %v", err) } @@ -349,7 +352,7 @@ func (c *Cluster) recreatePods() error { } func (c *Cluster) podIsEndOfLife(pod *v1.Pod) (bool, error) { - node, err := c.KubeClient.Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{}) + node, err := c.KubeClient.Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{}) if err != nil { return false, err } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 2e02a9a83..c0c731ed8 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -1,6 +1,7 @@ package cluster import ( + "context" "fmt" "strconv" "strings" @@ -80,7 +81,10 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { if err != nil { return nil, fmt.Errorf("could not generate statefulset: %v", err) } - statefulSet, err := c.KubeClient.StatefulSets(statefulSetSpec.Namespace).Create(statefulSetSpec) + statefulSet, err := c.KubeClient.StatefulSets(statefulSetSpec.Namespace).Create( + context.TODO(), + statefulSetSpec, + metav1.CreateOptions{}) if err != nil { return nil, err } @@ -129,7 +133,7 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolO // should be good enough to not think about it here. deployment, err := c.KubeClient. Deployments(deploymentSpec.Namespace). - Create(deploymentSpec) + Create(context.TODO(), deploymentSpec, metav1.CreateOptions{}) if err != nil { return nil, err @@ -138,7 +142,7 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolO serviceSpec := c.generateConnPoolService(&c.Spec) service, err := c.KubeClient. Services(serviceSpec.Namespace). - Create(serviceSpec) + Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) if err != nil { return nil, err @@ -180,7 +184,7 @@ func (c *Cluster) deleteConnectionPool() (err error) { options := metav1.DeleteOptions{PropagationPolicy: &policy} err = c.KubeClient. Deployments(c.Namespace). - Delete(deploymentName, &options) + Delete(context.TODO(), deploymentName, options) if !k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pool deployment was already deleted") @@ -202,7 +206,7 @@ func (c *Cluster) deleteConnectionPool() (err error) { // will be deleted. err = c.KubeClient. Services(c.Namespace). - Delete(serviceName, &options) + Delete(context.TODO(), serviceName, options) if !k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pool service was already deleted") @@ -251,7 +255,7 @@ func (c *Cluster) preScaleDown(newStatefulSet *appsv1.StatefulSet) error { } podName := fmt.Sprintf("%s-0", c.Statefulset.Name) - masterCandidatePod, err := c.KubeClient.Pods(c.clusterNamespace()).Get(podName, metav1.GetOptions{}) + masterCandidatePod, err := c.KubeClient.Pods(c.clusterNamespace()).Get(context.TODO(), podName, metav1.GetOptions{}) if err != nil { return fmt.Errorf("could not get master candidate pod: %v", err) } @@ -350,9 +354,12 @@ func (c *Cluster) updateStatefulSetAnnotations(annotations map[string]string) (* return nil, fmt.Errorf("could not form patch for the statefulset metadata: %v", err) } result, err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Patch( + context.TODO(), c.Statefulset.Name, types.MergePatchType, - []byte(patchData), "") + []byte(patchData), + metav1.PatchOptions{}, + "") if err != nil { return nil, fmt.Errorf("could not patch statefulset annotations %q: %v", patchData, err) } @@ -380,9 +387,12 @@ func (c *Cluster) updateStatefulSet(newStatefulSet *appsv1.StatefulSet) error { } statefulSet, err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Patch( + context.TODO(), c.Statefulset.Name, types.MergePatchType, - patchData, "") + patchData, + metav1.PatchOptions{}, + "") if err != nil { return fmt.Errorf("could not patch statefulset spec %q: %v", statefulSetName, err) } @@ -414,7 +424,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error { oldStatefulset := c.Statefulset options := metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy} - err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Delete(oldStatefulset.Name, &options) + err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Delete(context.TODO(), oldStatefulset.Name, options) if err != nil { return fmt.Errorf("could not delete statefulset %q: %v", statefulSetName, err) } @@ -425,7 +435,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error { err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, func() (bool, error) { - _, err2 := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Get(oldStatefulset.Name, metav1.GetOptions{}) + _, err2 := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Get(context.TODO(), oldStatefulset.Name, metav1.GetOptions{}) if err2 == nil { return false, nil } @@ -439,7 +449,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error { } // create the new statefulset with the desired spec. It would take over the remaining pods. - createdStatefulset, err := c.KubeClient.StatefulSets(newStatefulSet.Namespace).Create(newStatefulSet) + createdStatefulset, err := c.KubeClient.StatefulSets(newStatefulSet.Namespace).Create(context.TODO(), newStatefulSet, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create statefulset %q: %v", statefulSetName, err) } @@ -460,7 +470,7 @@ func (c *Cluster) deleteStatefulSet() error { return fmt.Errorf("there is no statefulset in the cluster") } - err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Delete(c.Statefulset.Name, c.deleteOptions) + err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Delete(context.TODO(), c.Statefulset.Name, c.deleteOptions) if err != nil { return err } @@ -482,7 +492,7 @@ func (c *Cluster) createService(role PostgresRole) (*v1.Service, error) { c.setProcessName("creating %v service", role) serviceSpec := c.generateService(role, &c.Spec) - service, err := c.KubeClient.Services(serviceSpec.Namespace).Create(serviceSpec) + service, err := c.KubeClient.Services(serviceSpec.Namespace).Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) if err != nil { return nil, err } @@ -509,9 +519,12 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error if len(newService.ObjectMeta.Annotations) > 0 { if annotationsPatchData, err := metaAnnotationsPatch(newService.ObjectMeta.Annotations); err == nil { _, err = c.KubeClient.Services(serviceName.Namespace).Patch( + context.TODO(), serviceName.Name, types.MergePatchType, - []byte(annotationsPatchData), "") + []byte(annotationsPatchData), + metav1.PatchOptions{}, + "") if err != nil { return fmt.Errorf("could not replace annotations for the service %q: %v", serviceName, err) @@ -528,7 +541,7 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error if newServiceType == "ClusterIP" && newServiceType != oldServiceType { newService.ResourceVersion = c.Services[role].ResourceVersion newService.Spec.ClusterIP = c.Services[role].Spec.ClusterIP - svc, err = c.KubeClient.Services(serviceName.Namespace).Update(newService) + svc, err = c.KubeClient.Services(serviceName.Namespace).Update(context.TODO(), newService, metav1.UpdateOptions{}) if err != nil { return fmt.Errorf("could not update service %q: %v", serviceName, err) } @@ -539,9 +552,7 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error } svc, err = c.KubeClient.Services(serviceName.Namespace).Patch( - serviceName.Name, - types.MergePatchType, - patchData, "") + context.TODO(), serviceName.Name, types.MergePatchType, patchData, metav1.PatchOptions{}, "") if err != nil { return fmt.Errorf("could not patch service %q: %v", serviceName, err) } @@ -560,7 +571,7 @@ func (c *Cluster) deleteService(role PostgresRole) error { return nil } - if err := c.KubeClient.Services(service.Namespace).Delete(service.Name, c.deleteOptions); err != nil { + if err := c.KubeClient.Services(service.Namespace).Delete(context.TODO(), service.Name, c.deleteOptions); err != nil { return err } @@ -584,7 +595,7 @@ func (c *Cluster) createEndpoint(role PostgresRole) (*v1.Endpoints, error) { } endpointsSpec := c.generateEndpoint(role, subsets) - endpoints, err := c.KubeClient.Endpoints(endpointsSpec.Namespace).Create(endpointsSpec) + endpoints, err := c.KubeClient.Endpoints(endpointsSpec.Namespace).Create(context.TODO(), endpointsSpec, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("could not create %s endpoint: %v", role, err) } @@ -626,7 +637,7 @@ func (c *Cluster) createPodDisruptionBudget() (*policybeta1.PodDisruptionBudget, podDisruptionBudgetSpec := c.generatePodDisruptionBudget() podDisruptionBudget, err := c.KubeClient. PodDisruptionBudgets(podDisruptionBudgetSpec.Namespace). - Create(podDisruptionBudgetSpec) + Create(context.TODO(), podDisruptionBudgetSpec, metav1.CreateOptions{}) if err != nil { return nil, err @@ -647,7 +658,7 @@ func (c *Cluster) updatePodDisruptionBudget(pdb *policybeta1.PodDisruptionBudget newPdb, err := c.KubeClient. PodDisruptionBudgets(pdb.Namespace). - Create(pdb) + Create(context.TODO(), pdb, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create pod disruption budget: %v", err) } @@ -665,7 +676,7 @@ func (c *Cluster) deletePodDisruptionBudget() error { pdbName := util.NameFromMeta(c.PodDisruptionBudget.ObjectMeta) err := c.KubeClient. PodDisruptionBudgets(c.PodDisruptionBudget.Namespace). - Delete(c.PodDisruptionBudget.Name, c.deleteOptions) + Delete(context.TODO(), c.PodDisruptionBudget.Name, c.deleteOptions) if err != nil { return fmt.Errorf("could not delete pod disruption budget: %v", err) } @@ -674,7 +685,7 @@ func (c *Cluster) deletePodDisruptionBudget() error { err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, func() (bool, error) { - _, err2 := c.KubeClient.PodDisruptionBudgets(pdbName.Namespace).Get(pdbName.Name, metav1.GetOptions{}) + _, err2 := c.KubeClient.PodDisruptionBudgets(pdbName.Namespace).Get(context.TODO(), pdbName.Name, metav1.GetOptions{}) if err2 == nil { return false, nil } @@ -697,7 +708,8 @@ func (c *Cluster) deleteEndpoint(role PostgresRole) error { return fmt.Errorf("there is no %s endpoint in the cluster", role) } - if err := c.KubeClient.Endpoints(c.Endpoints[role].Namespace).Delete(c.Endpoints[role].Name, c.deleteOptions); err != nil { + if err := c.KubeClient.Endpoints(c.Endpoints[role].Namespace).Delete( + context.TODO(), c.Endpoints[role].Name, c.deleteOptions); err != nil { return fmt.Errorf("could not delete endpoint: %v", err) } @@ -711,7 +723,7 @@ func (c *Cluster) deleteEndpoint(role PostgresRole) error { func (c *Cluster) deleteSecret(secret *v1.Secret) error { c.setProcessName("deleting secret %q", util.NameFromMeta(secret.ObjectMeta)) c.logger.Debugf("deleting secret %q", util.NameFromMeta(secret.ObjectMeta)) - err := c.KubeClient.Secrets(secret.Namespace).Delete(secret.Name, c.deleteOptions) + err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions) if err != nil { return err } @@ -736,7 +748,7 @@ func (c *Cluster) createLogicalBackupJob() (err error) { } c.logger.Debugf("Generated cronJobSpec: %v", logicalBackupJobSpec) - _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Create(logicalBackupJobSpec) + _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Create(context.TODO(), logicalBackupJobSpec, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create k8s cron job: %v", err) } @@ -754,9 +766,12 @@ func (c *Cluster) patchLogicalBackupJob(newJob *batchv1beta1.CronJob) error { // update the backup job spec _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Patch( + context.TODO(), c.getLogicalBackupJobName(), types.MergePatchType, - patchData, "") + patchData, + metav1.PatchOptions{}, + "") if err != nil { return fmt.Errorf("could not patch logical backup job: %v", err) } @@ -768,7 +783,7 @@ func (c *Cluster) deleteLogicalBackupJob() error { c.logger.Info("removing the logical backup job") - return c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Delete(c.getLogicalBackupJobName(), c.deleteOptions) + return c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Delete(context.TODO(), c.getLogicalBackupJobName(), c.deleteOptions) } // GetServiceMaster returns cluster's kubernetes master Service @@ -818,11 +833,13 @@ func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *app // worker at one time will try to update it chances of conflicts are // minimal. deployment, err := c.KubeClient. - Deployments(c.ConnectionPool.Deployment.Namespace). - Patch( - c.ConnectionPool.Deployment.Name, - types.MergePatchType, - patchData, "") + Deployments(c.ConnectionPool.Deployment.Namespace).Patch( + context.TODO(), + c.ConnectionPool.Deployment.Name, + types.MergePatchType, + patchData, + metav1.PatchOptions{}, + "") if err != nil { return nil, fmt.Errorf("could not patch deployment: %v", err) } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index a7c933ae7..eb3835787 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -1,6 +1,7 @@ package cluster import ( + "context" "fmt" batchv1beta1 "k8s.io/api/batch/v1beta1" @@ -140,7 +141,7 @@ func (c *Cluster) syncService(role PostgresRole) error { ) c.setProcessName("syncing %s service", role) - if svc, err = c.KubeClient.Services(c.Namespace).Get(c.serviceName(role), metav1.GetOptions{}); err == nil { + if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err == nil { c.Services[role] = svc desiredSvc := c.generateService(role, &c.Spec) if match, reason := k8sutil.SameService(svc, desiredSvc); !match { @@ -166,7 +167,7 @@ func (c *Cluster) syncService(role PostgresRole) error { return fmt.Errorf("could not create missing %s service: %v", role, err) } c.logger.Infof("%s service %q already exists", role, util.NameFromMeta(svc.ObjectMeta)) - if svc, err = c.KubeClient.Services(c.Namespace).Get(c.serviceName(role), metav1.GetOptions{}); err != nil { + if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err != nil { return fmt.Errorf("could not fetch existing %s service: %v", role, err) } } @@ -181,7 +182,7 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error { ) c.setProcessName("syncing %s endpoint", role) - if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(c.endpointName(role), metav1.GetOptions{}); err == nil { + if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), c.endpointName(role), metav1.GetOptions{}); err == nil { // TODO: No syncing of endpoints here, is this covered completely by updateService? c.Endpoints[role] = ep return nil @@ -200,7 +201,7 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error { return fmt.Errorf("could not create missing %s endpoint: %v", role, err) } c.logger.Infof("%s endpoint %q already exists", role, util.NameFromMeta(ep.ObjectMeta)) - if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(c.endpointName(role), metav1.GetOptions{}); err != nil { + if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), c.endpointName(role), metav1.GetOptions{}); err != nil { return fmt.Errorf("could not fetch existing %s endpoint: %v", role, err) } } @@ -213,7 +214,7 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error { pdb *policybeta1.PodDisruptionBudget err error ) - if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(c.podDisruptionBudgetName(), metav1.GetOptions{}); err == nil { + if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.podDisruptionBudgetName(), metav1.GetOptions{}); err == nil { c.PodDisruptionBudget = pdb newPDB := c.generatePodDisruptionBudget() if match, reason := k8sutil.SamePDB(pdb, newPDB); !match { @@ -239,7 +240,7 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error { return fmt.Errorf("could not create pod disruption budget: %v", err) } c.logger.Infof("pod disruption budget %q already exists", util.NameFromMeta(pdb.ObjectMeta)) - if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(c.podDisruptionBudgetName(), metav1.GetOptions{}); err != nil { + if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.podDisruptionBudgetName(), metav1.GetOptions{}); err != nil { return fmt.Errorf("could not fetch existing %q pod disruption budget", util.NameFromMeta(pdb.ObjectMeta)) } } @@ -255,7 +256,7 @@ func (c *Cluster) syncStatefulSet() error { podsRollingUpdateRequired bool ) // NB: Be careful to consider the codepath that acts on podsRollingUpdateRequired before returning early. - sset, err := c.KubeClient.StatefulSets(c.Namespace).Get(c.statefulSetName(), metav1.GetOptions{}) + sset, err := c.KubeClient.StatefulSets(c.Namespace).Get(context.TODO(), c.statefulSetName(), metav1.GetOptions{}) if err != nil { if !k8sutil.ResourceNotFound(err) { return fmt.Errorf("could not get statefulset: %v", err) @@ -404,14 +405,14 @@ func (c *Cluster) syncSecrets() error { secrets := c.generateUserSecrets() for secretUsername, secretSpec := range secrets { - if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(secretSpec); err == nil { + if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(context.TODO(), secretSpec, metav1.CreateOptions{}); err == nil { c.Secrets[secret.UID] = secret c.logger.Debugf("created new secret %q, uid: %q", util.NameFromMeta(secret.ObjectMeta), secret.UID) continue } if k8sutil.ResourceAlreadyExists(err) { var userMap map[string]spec.PgUser - if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(secretSpec.Name, metav1.GetOptions{}); err != nil { + if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(context.TODO(), secretSpec.Name, metav1.GetOptions{}); err != nil { return fmt.Errorf("could not get current secret: %v", err) } if secretUsername != string(secret.Data["username"]) { @@ -434,7 +435,7 @@ func (c *Cluster) syncSecrets() error { pwdUser.Origin == spec.RoleOriginInfrastructure { c.logger.Debugf("updating the secret %q from the infrastructure roles", secretSpec.Name) - if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(secretSpec); err != nil { + if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(context.TODO(), secretSpec, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("could not update infrastructure role secret for role %q: %v", secretUsername, err) } } else { @@ -577,7 +578,7 @@ func (c *Cluster) syncLogicalBackupJob() error { // sync the job if it exists jobName := c.getLogicalBackupJobName() - if job, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(jobName, metav1.GetOptions{}); err == nil { + if job, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(context.TODO(), jobName, metav1.GetOptions{}); err == nil { desiredJob, err = c.generateLogicalBackupJob() if err != nil { @@ -611,7 +612,7 @@ func (c *Cluster) syncLogicalBackupJob() error { return fmt.Errorf("could not create missing logical backup job: %v", err) } c.logger.Infof("logical backup job %q already exists", jobName) - if _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(jobName, metav1.GetOptions{}); err != nil { + if _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(context.TODO(), jobName, metav1.GetOptions{}); err != nil { return fmt.Errorf("could not fetch existing logical backup job: %v", err) } } @@ -696,7 +697,7 @@ func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) error { deployment, err := c.KubeClient. Deployments(c.Namespace). - Get(c.connPoolName(), metav1.GetOptions{}) + Get(context.TODO(), c.connPoolName(), metav1.GetOptions{}) if err != nil && k8sutil.ResourceNotFound(err) { msg := "Deployment %s for connection pool synchronization is not found, create it" @@ -710,7 +711,7 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) deployment, err := c.KubeClient. Deployments(deploymentSpec.Namespace). - Create(deploymentSpec) + Create(context.TODO(), deploymentSpec, metav1.CreateOptions{}) if err != nil { return err @@ -755,7 +756,7 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) service, err := c.KubeClient. Services(c.Namespace). - Get(c.connPoolName(), metav1.GetOptions{}) + Get(context.TODO(), c.connPoolName(), metav1.GetOptions{}) if err != nil && k8sutil.ResourceNotFound(err) { msg := "Service %s for connection pool synchronization is not found, create it" @@ -764,7 +765,7 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) serviceSpec := c.generateConnPoolService(&newSpec.Spec) service, err := c.KubeClient. Services(serviceSpec.Namespace). - Create(serviceSpec) + Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) if err != nil { return err diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index dc1e93954..3c3dffcaf 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -2,6 +2,7 @@ package cluster import ( "bytes" + "context" "encoding/gob" "encoding/json" "fmt" @@ -47,7 +48,7 @@ func (g *SecretOauthTokenGetter) getOAuthToken() (string, error) { // Temporary getting postgresql-operator secret from the NamespaceDefault credentialsSecret, err := g.kubeClient. Secrets(g.OAuthTokenSecretName.Namespace). - Get(g.OAuthTokenSecretName.Name, metav1.GetOptions{}) + Get(context.TODO(), g.OAuthTokenSecretName.Name, metav1.GetOptions{}) if err != nil { return "", fmt.Errorf("could not get credentials secret: %v", err) @@ -278,7 +279,7 @@ func (c *Cluster) waitStatefulsetReady() error { listOptions := metav1.ListOptions{ LabelSelector: c.labelsSet(false).String(), } - ss, err := c.KubeClient.StatefulSets(c.Namespace).List(listOptions) + ss, err := c.KubeClient.StatefulSets(c.Namespace).List(context.TODO(), listOptions) if err != nil { return false, err } @@ -313,7 +314,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error { } podsNumber = 1 if !anyReplica { - pods, err := c.KubeClient.Pods(namespace).List(listOptions) + pods, err := c.KubeClient.Pods(namespace).List(context.TODO(), listOptions) if err != nil { return err } @@ -327,7 +328,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error { func() (bool, error) { masterCount := 0 if !anyReplica { - masterPods, err2 := c.KubeClient.Pods(namespace).List(masterListOption) + masterPods, err2 := c.KubeClient.Pods(namespace).List(context.TODO(), masterListOption) if err2 != nil { return false, err2 } @@ -337,7 +338,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error { } masterCount = len(masterPods.Items) } - replicaPods, err2 := c.KubeClient.Pods(namespace).List(replicaListOption) + replicaPods, err2 := c.KubeClient.Pods(namespace).List(context.TODO(), replicaListOption) if err2 != nil { return false, err2 } diff --git a/pkg/cluster/volumes.go b/pkg/cluster/volumes.go index d92ae6258..a5bfe6c2d 100644 --- a/pkg/cluster/volumes.go +++ b/pkg/cluster/volumes.go @@ -1,11 +1,12 @@ package cluster import ( + "context" "fmt" "strconv" "strings" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,7 +24,7 @@ func (c *Cluster) listPersistentVolumeClaims() ([]v1.PersistentVolumeClaim, erro LabelSelector: c.labelsSet(false).String(), } - pvcs, err := c.KubeClient.PersistentVolumeClaims(ns).List(listOptions) + pvcs, err := c.KubeClient.PersistentVolumeClaims(ns).List(context.TODO(), listOptions) if err != nil { return nil, fmt.Errorf("could not list of PersistentVolumeClaims: %v", err) } @@ -38,7 +39,7 @@ func (c *Cluster) deletePersistentVolumeClaims() error { } for _, pvc := range pvcs { c.logger.Debugf("deleting PVC %q", util.NameFromMeta(pvc.ObjectMeta)) - if err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, c.deleteOptions); err != nil { + if err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, c.deleteOptions); err != nil { c.logger.Warningf("could not delete PersistentVolumeClaim: %v", err) } } @@ -78,7 +79,7 @@ func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) { continue } } - pv, err := c.KubeClient.PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{}) + pv, err := c.KubeClient.PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("could not get PersistentVolume: %v", err) } @@ -143,7 +144,7 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu c.logger.Debugf("filesystem resize successful on volume %q", pv.Name) pv.Spec.Capacity[v1.ResourceStorage] = newQuantity c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name) - if _, err := c.KubeClient.PersistentVolumes().Update(pv); err != nil { + if _, err := c.KubeClient.PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("could not update persistent volume: %q", err) } c.logger.Debugf("successfully updated persistent volume %q", pv.Name) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 0ce0d026e..9c48b7ef2 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "os" "sync" @@ -99,7 +100,7 @@ func (c *Controller) initOperatorConfig() { if c.config.ConfigMapName != (spec.NamespacedName{}) { configMap, err := c.KubeClient.ConfigMaps(c.config.ConfigMapName.Namespace). - Get(c.config.ConfigMapName.Name, metav1.GetOptions{}) + Get(context.TODO(), c.config.ConfigMapName.Name, metav1.GetOptions{}) if err != nil { panic(err) } @@ -406,7 +407,7 @@ func (c *Controller) getEffectiveNamespace(namespaceFromEnvironment, namespaceFr } else { - if _, err := c.KubeClient.Namespaces().Get(namespace, metav1.GetOptions{}); err != nil { + if _, err := c.KubeClient.Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil { c.logger.Fatalf("Could not find the watched namespace %q", namespace) } else { c.logger.Infof("Listenting to the specific namespace %q", namespace) diff --git a/pkg/controller/node.go b/pkg/controller/node.go index 8052458c3..be41b79ab 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "time" @@ -22,7 +23,7 @@ func (c *Controller) nodeListFunc(options metav1.ListOptions) (runtime.Object, e TimeoutSeconds: options.TimeoutSeconds, } - return c.KubeClient.Nodes().List(opts) + return c.KubeClient.Nodes().List(context.TODO(), opts) } func (c *Controller) nodeWatchFunc(options metav1.ListOptions) (watch.Interface, error) { @@ -32,7 +33,7 @@ func (c *Controller) nodeWatchFunc(options metav1.ListOptions) (watch.Interface, TimeoutSeconds: options.TimeoutSeconds, } - return c.KubeClient.Nodes().Watch(opts) + return c.KubeClient.Nodes().Watch(context.TODO(), opts) } func (c *Controller) nodeAdd(obj interface{}) { @@ -87,7 +88,7 @@ func (c *Controller) attemptToMoveMasterPodsOffNode(node *v1.Node) error { opts := metav1.ListOptions{ LabelSelector: labels.Set(c.opConfig.ClusterLabels).String(), } - podList, err := c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(opts) + podList, err := c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(context.TODO(), opts) if err != nil { c.logger.Errorf("could not fetch list of the pods: %v", err) return err diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 970eef701..c9b3c5ea4 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "time" @@ -14,7 +15,8 @@ import ( func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, configObjectName string) (*acidv1.OperatorConfiguration, error) { - config, err := c.KubeClient.AcidV1ClientSet.AcidV1().OperatorConfigurations(configObjectNamespace).Get(configObjectName, metav1.GetOptions{}) + config, err := c.KubeClient.AcidV1ClientSet.AcidV1().OperatorConfigurations(configObjectNamespace).Get( + context.TODO(), configObjectName, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("could not get operator configuration object %q: %v", configObjectName, err) } diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 27fd6c956..0defe88b1 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -1,7 +1,9 @@ package controller import ( - "k8s.io/api/core/v1" + "context" + + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" @@ -19,7 +21,7 @@ func (c *Controller) podListFunc(options metav1.ListOptions) (runtime.Object, er TimeoutSeconds: options.TimeoutSeconds, } - return c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(opts) + return c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(context.TODO(), opts) } func (c *Controller) podWatchFunc(options metav1.ListOptions) (watch.Interface, error) { @@ -29,7 +31,7 @@ func (c *Controller) podWatchFunc(options metav1.ListOptions) (watch.Interface, TimeoutSeconds: options.TimeoutSeconds, } - return c.KubeClient.Pods(c.opConfig.WatchedNamespace).Watch(opts) + return c.KubeClient.Pods(c.opConfig.WatchedNamespace).Watch(context.TODO(), opts) } func (c *Controller) dispatchPodEvent(clusterName spec.NamespacedName, event cluster.PodEvent) { diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 5d48bac39..e81671c7d 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "reflect" "strings" @@ -43,7 +44,7 @@ func (c *Controller) listClusters(options metav1.ListOptions) (*acidv1.Postgresq var pgList acidv1.PostgresqlList // TODO: use the SharedInformer cache instead of quering Kubernetes API directly. - list, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.opConfig.WatchedNamespace).List(options) + list, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.opConfig.WatchedNamespace).List(context.TODO(), options) if err != nil { c.logger.Errorf("could not list postgresql objects: %v", err) } @@ -535,7 +536,7 @@ func (c *Controller) submitRBACCredentials(event ClusterEvent) error { func (c *Controller) createPodServiceAccount(namespace string) error { podServiceAccountName := c.opConfig.PodServiceAccountName - _, err := c.KubeClient.ServiceAccounts(namespace).Get(podServiceAccountName, metav1.GetOptions{}) + _, err := c.KubeClient.ServiceAccounts(namespace).Get(context.TODO(), podServiceAccountName, metav1.GetOptions{}) if k8sutil.ResourceNotFound(err) { c.logger.Infof(fmt.Sprintf("creating pod service account %q in the %q namespace", podServiceAccountName, namespace)) @@ -543,7 +544,7 @@ func (c *Controller) createPodServiceAccount(namespace string) error { // get a separate copy of service account // to prevent a race condition when setting a namespace for many clusters sa := *c.PodServiceAccount - if _, err = c.KubeClient.ServiceAccounts(namespace).Create(&sa); err != nil { + if _, err = c.KubeClient.ServiceAccounts(namespace).Create(context.TODO(), &sa, metav1.CreateOptions{}); err != nil { return fmt.Errorf("cannot deploy the pod service account %q defined in the configuration to the %q namespace: %v", podServiceAccountName, namespace, err) } @@ -560,7 +561,7 @@ func (c *Controller) createRoleBindings(namespace string) error { podServiceAccountName := c.opConfig.PodServiceAccountName podServiceAccountRoleBindingName := c.PodServiceAccountRoleBinding.Name - _, err := c.KubeClient.RoleBindings(namespace).Get(podServiceAccountRoleBindingName, metav1.GetOptions{}) + _, err := c.KubeClient.RoleBindings(namespace).Get(context.TODO(), podServiceAccountRoleBindingName, metav1.GetOptions{}) if k8sutil.ResourceNotFound(err) { c.logger.Infof("Creating the role binding %q in the %q namespace", podServiceAccountRoleBindingName, namespace) @@ -568,7 +569,7 @@ func (c *Controller) createRoleBindings(namespace string) error { // get a separate copy of role binding // to prevent a race condition when setting a namespace for many clusters rb := *c.PodServiceAccountRoleBinding - _, err = c.KubeClient.RoleBindings(namespace).Create(&rb) + _, err = c.KubeClient.RoleBindings(namespace).Create(context.TODO(), &rb, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("cannot bind the pod service account %q defined in the configuration to the cluster role in the %q namespace: %v", podServiceAccountName, namespace, err) } diff --git a/pkg/controller/util.go b/pkg/controller/util.go index 9b7dca063..511f02823 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -1,6 +1,7 @@ package controller import ( + "context" "encoding/json" "fmt" @@ -50,7 +51,7 @@ func (c *Controller) clusterWorkerID(clusterName spec.NamespacedName) uint32 { } func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefinition) error { - if _, err := c.KubeClient.CustomResourceDefinitions().Create(crd); err != nil { + if _, err := c.KubeClient.CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}); err != nil { if k8sutil.ResourceAlreadyExists(err) { c.logger.Infof("customResourceDefinition %q is already registered and will only be updated", crd.Name) @@ -58,7 +59,8 @@ func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefiniti if err != nil { return fmt.Errorf("could not marshal new customResourceDefintion: %v", err) } - if _, err := c.KubeClient.CustomResourceDefinitions().Patch(crd.Name, types.MergePatchType, patch); err != nil { + if _, err := c.KubeClient.CustomResourceDefinitions().Patch( + context.TODO(), crd.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { return fmt.Errorf("could not update customResourceDefinition: %v", err) } } else { @@ -69,7 +71,7 @@ func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefiniti } return wait.Poll(c.config.CRDReadyWaitInterval, c.config.CRDReadyWaitTimeout, func() (bool, error) { - c, err := c.KubeClient.CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{}) + c, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{}) if err != nil { return false, err } @@ -115,7 +117,7 @@ func (c *Controller) getInfrastructureRoles(rolesSecret *spec.NamespacedName) (m infraRolesSecret, err := c.KubeClient. Secrets(rolesSecret.Namespace). - Get(rolesSecret.Name, metav1.GetOptions{}) + Get(context.TODO(), rolesSecret.Name, metav1.GetOptions{}) if err != nil { c.logger.Debugf("infrastructure roles secret name: %q", *rolesSecret) return nil, fmt.Errorf("could not get infrastructure roles secret: %v", err) @@ -161,7 +163,8 @@ Users: } // perhaps we have some map entries with usernames, passwords, let's check if we have those users in the configmap - if infraRolesMap, err := c.KubeClient.ConfigMaps(rolesSecret.Namespace).Get(rolesSecret.Name, metav1.GetOptions{}); err == nil { + if infraRolesMap, err := c.KubeClient.ConfigMaps(rolesSecret.Namespace).Get( + context.TODO(), rolesSecret.Name, metav1.GetOptions{}); err == nil { // we have a configmap with username - json description, let's read and decode it for role, s := range infraRolesMap.Data { roleDescr, err := readDecodedRole(s) diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go index cb72ec50f..5f1e5880a 100644 --- a/pkg/generated/clientset/versioned/clientset.go +++ b/pkg/generated/clientset/versioned/clientset.go @@ -65,7 +65,7 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { configShallowCopy := *c if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { if configShallowCopy.Burst <= 0 { - return nil, fmt.Errorf("Burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") } configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) } diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go index 732b48250..d515a0080 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go @@ -25,6 +25,8 @@ SOFTWARE. package fake import ( + "context" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -42,7 +44,7 @@ var operatorconfigurationsResource = schema.GroupVersionResource{Group: "acid.za var operatorconfigurationsKind = schema.GroupVersionKind{Group: "acid.zalan.do", Version: "v1", Kind: "OperatorConfiguration"} // Get takes name of the operatorConfiguration, and returns the corresponding operatorConfiguration object, and an error if there is any. -func (c *FakeOperatorConfigurations) Get(name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) { +func (c *FakeOperatorConfigurations) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) { obj, err := c.Fake. Invokes(testing.NewGetAction(operatorconfigurationsResource, c.ns, name), &acidzalandov1.OperatorConfiguration{}) diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go index 1ab20dbfc..e4d72f882 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go @@ -25,6 +25,8 @@ SOFTWARE. package fake import ( + "context" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" @@ -45,7 +47,7 @@ var postgresqlsResource = schema.GroupVersionResource{Group: "acid.zalan.do", Ve var postgresqlsKind = schema.GroupVersionKind{Group: "acid.zalan.do", Version: "v1", Kind: "Postgresql"} // Get takes name of the postgresql, and returns the corresponding postgresql object, and an error if there is any. -func (c *FakePostgresqls) Get(name string, options v1.GetOptions) (result *acidzalandov1.Postgresql, err error) { +func (c *FakePostgresqls) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.Postgresql, err error) { obj, err := c.Fake. Invokes(testing.NewGetAction(postgresqlsResource, c.ns, name), &acidzalandov1.Postgresql{}) @@ -56,7 +58,7 @@ func (c *FakePostgresqls) Get(name string, options v1.GetOptions) (result *acidz } // List takes label and field selectors, and returns the list of Postgresqls that match those selectors. -func (c *FakePostgresqls) List(opts v1.ListOptions) (result *acidzalandov1.PostgresqlList, err error) { +func (c *FakePostgresqls) List(ctx context.Context, opts v1.ListOptions) (result *acidzalandov1.PostgresqlList, err error) { obj, err := c.Fake. Invokes(testing.NewListAction(postgresqlsResource, postgresqlsKind, c.ns, opts), &acidzalandov1.PostgresqlList{}) @@ -78,14 +80,14 @@ func (c *FakePostgresqls) List(opts v1.ListOptions) (result *acidzalandov1.Postg } // Watch returns a watch.Interface that watches the requested postgresqls. -func (c *FakePostgresqls) Watch(opts v1.ListOptions) (watch.Interface, error) { +func (c *FakePostgresqls) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { return c.Fake. InvokesWatch(testing.NewWatchAction(postgresqlsResource, c.ns, opts)) } // Create takes the representation of a postgresql and creates it. Returns the server's representation of the postgresql, and an error, if there is any. -func (c *FakePostgresqls) Create(postgresql *acidzalandov1.Postgresql) (result *acidzalandov1.Postgresql, err error) { +func (c *FakePostgresqls) Create(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts v1.CreateOptions) (result *acidzalandov1.Postgresql, err error) { obj, err := c.Fake. Invokes(testing.NewCreateAction(postgresqlsResource, c.ns, postgresql), &acidzalandov1.Postgresql{}) @@ -96,7 +98,7 @@ func (c *FakePostgresqls) Create(postgresql *acidzalandov1.Postgresql) (result * } // Update takes the representation of a postgresql and updates it. Returns the server's representation of the postgresql, and an error, if there is any. -func (c *FakePostgresqls) Update(postgresql *acidzalandov1.Postgresql) (result *acidzalandov1.Postgresql, err error) { +func (c *FakePostgresqls) Update(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts v1.UpdateOptions) (result *acidzalandov1.Postgresql, err error) { obj, err := c.Fake. Invokes(testing.NewUpdateAction(postgresqlsResource, c.ns, postgresql), &acidzalandov1.Postgresql{}) @@ -108,7 +110,7 @@ func (c *FakePostgresqls) Update(postgresql *acidzalandov1.Postgresql) (result * // UpdateStatus was generated because the type contains a Status member. // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakePostgresqls) UpdateStatus(postgresql *acidzalandov1.Postgresql) (*acidzalandov1.Postgresql, error) { +func (c *FakePostgresqls) UpdateStatus(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts v1.UpdateOptions) (*acidzalandov1.Postgresql, error) { obj, err := c.Fake. Invokes(testing.NewUpdateSubresourceAction(postgresqlsResource, "status", c.ns, postgresql), &acidzalandov1.Postgresql{}) @@ -119,7 +121,7 @@ func (c *FakePostgresqls) UpdateStatus(postgresql *acidzalandov1.Postgresql) (*a } // Delete takes name of the postgresql and deletes it. Returns an error if one occurs. -func (c *FakePostgresqls) Delete(name string, options *v1.DeleteOptions) error { +func (c *FakePostgresqls) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. Invokes(testing.NewDeleteAction(postgresqlsResource, c.ns, name), &acidzalandov1.Postgresql{}) @@ -127,15 +129,15 @@ func (c *FakePostgresqls) Delete(name string, options *v1.DeleteOptions) error { } // DeleteCollection deletes a collection of objects. -func (c *FakePostgresqls) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { - action := testing.NewDeleteCollectionAction(postgresqlsResource, c.ns, listOptions) +func (c *FakePostgresqls) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(postgresqlsResource, c.ns, listOpts) _, err := c.Fake.Invokes(action, &acidzalandov1.PostgresqlList{}) return err } // Patch applies the patch and returns the patched postgresql. -func (c *FakePostgresqls) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *acidzalandov1.Postgresql, err error) { +func (c *FakePostgresqls) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *acidzalandov1.Postgresql, err error) { obj, err := c.Fake. Invokes(testing.NewPatchSubresourceAction(postgresqlsResource, c.ns, name, pt, data, subresources...), &acidzalandov1.Postgresql{}) diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go index e9cc0de77..80ef6d6f3 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go @@ -25,6 +25,8 @@ SOFTWARE. package v1 import ( + "context" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,7 +41,7 @@ type OperatorConfigurationsGetter interface { // OperatorConfigurationInterface has methods to work with OperatorConfiguration resources. type OperatorConfigurationInterface interface { - Get(name string, options v1.GetOptions) (*acidzalandov1.OperatorConfiguration, error) + Get(ctx context.Context, name string, opts v1.GetOptions) (*acidzalandov1.OperatorConfiguration, error) OperatorConfigurationExpansion } @@ -58,14 +60,14 @@ func newOperatorConfigurations(c *AcidV1Client, namespace string) *operatorConfi } // Get takes name of the operatorConfiguration, and returns the corresponding operatorConfiguration object, and an error if there is any. -func (c *operatorConfigurations) Get(name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) { +func (c *operatorConfigurations) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) { result = &acidzalandov1.OperatorConfiguration{} err = c.client.Get(). Namespace(c.ns). Resource("operatorconfigurations"). Name(name). VersionedParams(&options, scheme.ParameterCodec). - Do(). + Do(ctx). Into(result) return } diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go index 78c0fc390..ca8c6d7ee 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go @@ -25,6 +25,7 @@ SOFTWARE. package v1 import ( + "context" "time" v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -43,15 +44,15 @@ type PostgresqlsGetter interface { // PostgresqlInterface has methods to work with Postgresql resources. type PostgresqlInterface interface { - Create(*v1.Postgresql) (*v1.Postgresql, error) - Update(*v1.Postgresql) (*v1.Postgresql, error) - UpdateStatus(*v1.Postgresql) (*v1.Postgresql, error) - Delete(name string, options *metav1.DeleteOptions) error - DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error - Get(name string, options metav1.GetOptions) (*v1.Postgresql, error) - List(opts metav1.ListOptions) (*v1.PostgresqlList, error) - Watch(opts metav1.ListOptions) (watch.Interface, error) - Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Postgresql, err error) + Create(ctx context.Context, postgresql *v1.Postgresql, opts metav1.CreateOptions) (*v1.Postgresql, error) + Update(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (*v1.Postgresql, error) + UpdateStatus(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (*v1.Postgresql, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Postgresql, error) + List(ctx context.Context, opts metav1.ListOptions) (*v1.PostgresqlList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.Postgresql, err error) PostgresqlExpansion } @@ -70,20 +71,20 @@ func newPostgresqls(c *AcidV1Client, namespace string) *postgresqls { } // Get takes name of the postgresql, and returns the corresponding postgresql object, and an error if there is any. -func (c *postgresqls) Get(name string, options metav1.GetOptions) (result *v1.Postgresql, err error) { +func (c *postgresqls) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.Postgresql, err error) { result = &v1.Postgresql{} err = c.client.Get(). Namespace(c.ns). Resource("postgresqls"). Name(name). VersionedParams(&options, scheme.ParameterCodec). - Do(). + Do(ctx). Into(result) return } // List takes label and field selectors, and returns the list of Postgresqls that match those selectors. -func (c *postgresqls) List(opts metav1.ListOptions) (result *v1.PostgresqlList, err error) { +func (c *postgresqls) List(ctx context.Context, opts metav1.ListOptions) (result *v1.PostgresqlList, err error) { var timeout time.Duration if opts.TimeoutSeconds != nil { timeout = time.Duration(*opts.TimeoutSeconds) * time.Second @@ -94,13 +95,13 @@ func (c *postgresqls) List(opts metav1.ListOptions) (result *v1.PostgresqlList, Resource("postgresqls"). VersionedParams(&opts, scheme.ParameterCodec). Timeout(timeout). - Do(). + Do(ctx). Into(result) return } // Watch returns a watch.Interface that watches the requested postgresqls. -func (c *postgresqls) Watch(opts metav1.ListOptions) (watch.Interface, error) { +func (c *postgresqls) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { var timeout time.Duration if opts.TimeoutSeconds != nil { timeout = time.Duration(*opts.TimeoutSeconds) * time.Second @@ -111,87 +112,90 @@ func (c *postgresqls) Watch(opts metav1.ListOptions) (watch.Interface, error) { Resource("postgresqls"). VersionedParams(&opts, scheme.ParameterCodec). Timeout(timeout). - Watch() + Watch(ctx) } // Create takes the representation of a postgresql and creates it. Returns the server's representation of the postgresql, and an error, if there is any. -func (c *postgresqls) Create(postgresql *v1.Postgresql) (result *v1.Postgresql, err error) { +func (c *postgresqls) Create(ctx context.Context, postgresql *v1.Postgresql, opts metav1.CreateOptions) (result *v1.Postgresql, err error) { result = &v1.Postgresql{} err = c.client.Post(). Namespace(c.ns). Resource("postgresqls"). + VersionedParams(&opts, scheme.ParameterCodec). Body(postgresql). - Do(). + Do(ctx). Into(result) return } // Update takes the representation of a postgresql and updates it. Returns the server's representation of the postgresql, and an error, if there is any. -func (c *postgresqls) Update(postgresql *v1.Postgresql) (result *v1.Postgresql, err error) { +func (c *postgresqls) Update(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (result *v1.Postgresql, err error) { result = &v1.Postgresql{} err = c.client.Put(). Namespace(c.ns). Resource("postgresqls"). Name(postgresql.Name). + VersionedParams(&opts, scheme.ParameterCodec). Body(postgresql). - Do(). + Do(ctx). Into(result) return } // UpdateStatus was generated because the type contains a Status member. // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). - -func (c *postgresqls) UpdateStatus(postgresql *v1.Postgresql) (result *v1.Postgresql, err error) { +func (c *postgresqls) UpdateStatus(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (result *v1.Postgresql, err error) { result = &v1.Postgresql{} err = c.client.Put(). Namespace(c.ns). Resource("postgresqls"). Name(postgresql.Name). SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). Body(postgresql). - Do(). + Do(ctx). Into(result) return } // Delete takes name of the postgresql and deletes it. Returns an error if one occurs. -func (c *postgresqls) Delete(name string, options *metav1.DeleteOptions) error { +func (c *postgresqls) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { return c.client.Delete(). Namespace(c.ns). Resource("postgresqls"). Name(name). - Body(options). - Do(). + Body(&opts). + Do(ctx). Error() } // DeleteCollection deletes a collection of objects. -func (c *postgresqls) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { +func (c *postgresqls) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { var timeout time.Duration - if listOptions.TimeoutSeconds != nil { - timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second } return c.client.Delete(). Namespace(c.ns). Resource("postgresqls"). - VersionedParams(&listOptions, scheme.ParameterCodec). + VersionedParams(&listOpts, scheme.ParameterCodec). Timeout(timeout). - Body(options). - Do(). + Body(&opts). + Do(ctx). Error() } // Patch applies the patch and returns the patched postgresql. -func (c *postgresqls) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Postgresql, err error) { +func (c *postgresqls) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.Postgresql, err error) { result = &v1.Postgresql{} err = c.client.Patch(pt). Namespace(c.ns). Resource("postgresqls"). - SubResource(subresources...). Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). Body(data). - Do(). + Do(ctx). Into(result) return } diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go index da7f91669..be09839d3 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go @@ -25,6 +25,7 @@ SOFTWARE. package v1 import ( + "context" time "time" acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -67,13 +68,13 @@ func NewFilteredPostgresqlInformer(client versioned.Interface, namespace string, if tweakListOptions != nil { tweakListOptions(&options) } - return client.AcidV1().Postgresqls(namespace).List(options) + return client.AcidV1().Postgresqls(namespace).List(context.TODO(), options) }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.AcidV1().Postgresqls(namespace).Watch(options) + return client.AcidV1().Postgresqls(namespace).Watch(context.TODO(), options) }, }, &acidzalandov1.Postgresql{}, diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 75b99ec7c..3a397af7d 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -1,6 +1,7 @@ package k8sutil import ( + "context" "fmt" "reflect" @@ -237,7 +238,7 @@ func SameLogicalBackupJob(cur, new *batchv1beta1.CronJob) (match bool, reason st return true, "" } -func (c *mockSecret) Get(name string, options metav1.GetOptions) (*v1.Secret, error) { +func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) { if name != "infrastructureroles-test" { return nil, fmt.Errorf("NotFound") } @@ -253,7 +254,7 @@ func (c *mockSecret) Get(name string, options metav1.GetOptions) (*v1.Secret, er } -func (c *mockConfigMap) Get(name string, options metav1.GetOptions) (*v1.ConfigMap, error) { +func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) { if name != "infrastructureroles-test" { return nil, fmt.Errorf("NotFound") } @@ -283,7 +284,7 @@ func (mock *MockDeploymentNotExistGetter) Deployments(namespace string) appsv1.D return &mockDeploymentNotExist{} } -func (mock *mockDeployment) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { +func (mock *mockDeployment) Create(context.Context, *apiappsv1.Deployment, metav1.CreateOptions) (*apiappsv1.Deployment, error) { return &apiappsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment", @@ -294,11 +295,11 @@ func (mock *mockDeployment) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment }, nil } -func (mock *mockDeployment) Delete(name string, opts *metav1.DeleteOptions) error { +func (mock *mockDeployment) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { return nil } -func (mock *mockDeployment) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { +func (mock *mockDeployment) Get(ctx context.Context, name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { return &apiappsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment", @@ -318,7 +319,7 @@ func (mock *mockDeployment) Get(name string, opts metav1.GetOptions) (*apiappsv1 }, nil } -func (mock *mockDeployment) Patch(name string, t types.PatchType, data []byte, subres ...string) (*apiappsv1.Deployment, error) { +func (mock *mockDeployment) Patch(ctx context.Context, name string, t types.PatchType, data []byte, opts metav1.PatchOptions, subres ...string) (*apiappsv1.Deployment, error) { return &apiappsv1.Deployment{ Spec: apiappsv1.DeploymentSpec{ Replicas: Int32ToPointer(2), @@ -329,7 +330,7 @@ func (mock *mockDeployment) Patch(name string, t types.PatchType, data []byte, s }, nil } -func (mock *mockDeploymentNotExist) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { +func (mock *mockDeploymentNotExist) Get(ctx context.Context, name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { return nil, &apierrors.StatusError{ ErrStatus: metav1.Status{ Reason: metav1.StatusReasonNotFound, @@ -337,7 +338,7 @@ func (mock *mockDeploymentNotExist) Get(name string, opts metav1.GetOptions) (*a } } -func (mock *mockDeploymentNotExist) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { +func (mock *mockDeploymentNotExist) Create(context.Context, *apiappsv1.Deployment, metav1.CreateOptions) (*apiappsv1.Deployment, error) { return &apiappsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment", @@ -356,7 +357,7 @@ func (mock *MockServiceNotExistGetter) Services(namespace string) corev1.Service return &mockServiceNotExist{} } -func (mock *mockService) Create(*v1.Service) (*v1.Service, error) { +func (mock *mockService) Create(context.Context, *v1.Service, metav1.CreateOptions) (*v1.Service, error) { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test-service", @@ -364,11 +365,11 @@ func (mock *mockService) Create(*v1.Service) (*v1.Service, error) { }, nil } -func (mock *mockService) Delete(name string, opts *metav1.DeleteOptions) error { +func (mock *mockService) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { return nil } -func (mock *mockService) Get(name string, opts metav1.GetOptions) (*v1.Service, error) { +func (mock *mockService) Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Service, error) { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test-service", @@ -376,7 +377,7 @@ func (mock *mockService) Get(name string, opts metav1.GetOptions) (*v1.Service, }, nil } -func (mock *mockServiceNotExist) Create(*v1.Service) (*v1.Service, error) { +func (mock *mockServiceNotExist) Create(context.Context, *v1.Service, metav1.CreateOptions) (*v1.Service, error) { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test-service", @@ -384,7 +385,7 @@ func (mock *mockServiceNotExist) Create(*v1.Service) (*v1.Service, error) { }, nil } -func (mock *mockServiceNotExist) Get(name string, opts metav1.GetOptions) (*v1.Service, error) { +func (mock *mockServiceNotExist) Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Service, error) { return nil, &apierrors.StatusError{ ErrStatus: metav1.Status{ Reason: metav1.StatusReasonNotFound, diff --git a/pkg/util/teams/teams_test.go b/pkg/util/teams/teams_test.go index 51bbcbc31..33d01b75b 100644 --- a/pkg/util/teams/teams_test.go +++ b/pkg/util/teams/teams_test.go @@ -133,11 +133,11 @@ var requestsURLtc = []struct { }{ { "coffee://localhost/", - fmt.Errorf(`Get coffee://localhost/teams/acid: unsupported protocol scheme "coffee"`), + fmt.Errorf(`Get "coffee://localhost/teams/acid": unsupported protocol scheme "coffee"`), }, { "http://192.168.0.%31/", - fmt.Errorf(`parse http://192.168.0.%%31/teams/acid: invalid URL escape "%%31"`), + fmt.Errorf(`parse "http://192.168.0.%%31/teams/acid": invalid URL escape "%%31"`), }, } From 64d816c55601c45afe8dec7a64f737cb19c19f2c Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 31 Mar 2020 11:47:39 +0200 Subject: [PATCH 071/104] add short sleep before redistributing pods (#891) --- e2e/tests/test_e2e.py | 179 ++++++++++++++++++++++-------------------- 1 file changed, 93 insertions(+), 86 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index cc90aa5e2..f6fc8184c 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -69,6 +69,92 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_enable_disable_connection_pool(self): + ''' + For a database without connection pool, then turns it on, scale up, + turn off and on again. Test with different ways of doing this (via + enableConnectionPool or connectionPool configuration section). At the + end turn the connection pool off to not interfere with other tests. + ''' + k8s = self.k8s + service_labels = { + 'cluster-name': 'acid-minimal-cluster', + } + pod_labels = dict({ + 'connection-pool': 'acid-minimal-cluster-pooler', + }) + + pod_selector = to_selector(pod_labels) + service_selector = to_selector(service_labels) + + try: + # enable connection pool + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': True, + } + }) + k8s.wait_for_pod_start(pod_selector) + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=pod_selector + ).items + + self.assertTrue(pods, 'No connection pool pods') + + k8s.wait_for_service(service_selector) + services = k8s.api.core_v1.list_namespaced_service( + 'default', label_selector=service_selector + ).items + services = [ + s for s in services + if s.metadata.name.endswith('pooler') + ] + + self.assertTrue(services, 'No connection pool service') + + # scale up connection pool deployment + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'connectionPool': { + 'numberOfInstances': 2, + }, + } + }) + + k8s.wait_for_running_pods(pod_selector, 2) + + # turn it off, keeping configuration section + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': False, + } + }) + k8s.wait_for_pods_to_stop(pod_selector) + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': True, + } + }) + k8s.wait_for_pod_start(pod_selector) + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_enable_load_balancer(self): ''' @@ -290,6 +376,10 @@ class EndToEndTestCase(unittest.TestCase): # patch also node where master ran before k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) + + # wait a little before proceeding with the pod distribution test + time.sleep(k8s.RETRY_TIMEOUT_SEC) + # toggle pod anti affinity to move replica away from master node self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) @@ -349,92 +439,6 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(unpatch_custom_service_annotations) - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_enable_disable_connection_pool(self): - ''' - For a database without connection pool, then turns it on, scale up, - turn off and on again. Test with different ways of doing this (via - enableConnectionPool or connectionPool configuration section). At the - end turn the connection pool off to not interfere with other tests. - ''' - k8s = self.k8s - service_labels = { - 'cluster-name': 'acid-minimal-cluster', - } - pod_labels = dict({ - 'connection-pool': 'acid-minimal-cluster-pooler', - }) - - pod_selector = to_selector(pod_labels) - service_selector = to_selector(service_labels) - - try: - # enable connection pool - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPool': True, - } - }) - k8s.wait_for_pod_start(pod_selector) - - pods = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector=pod_selector - ).items - - self.assertTrue(pods, 'No connection pool pods') - - k8s.wait_for_service(service_selector) - services = k8s.api.core_v1.list_namespaced_service( - 'default', label_selector=service_selector - ).items - services = [ - s for s in services - if s.metadata.name.endswith('pooler') - ] - - self.assertTrue(services, 'No connection pool service') - - # scale up connection pool deployment - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'connectionPool': { - 'numberOfInstances': 2, - }, - } - }) - - k8s.wait_for_running_pods(pod_selector, 2) - - # turn it off, keeping configuration section - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPool': False, - } - }) - k8s.wait_for_pods_to_stop(pod_selector) - - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPool': True, - } - }) - k8s.wait_for_pod_start(pod_selector) - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): ''' @@ -473,6 +477,9 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(patch_toleration_config) + # wait a little before proceeding with the pod distribution test + time.sleep(k8s.RETRY_TIMEOUT_SEC) + # toggle pod anti affinity to move replica away from master node self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) From 6ed10308380cbe87e63c92eed0490bde05811229 Mon Sep 17 00:00:00 2001 From: ReSearchITEng Date: Wed, 1 Apr 2020 10:39:54 +0300 Subject: [PATCH 072/104] TLS - add OpenShift compatibility (#885) * solves https://github.com/zalando/postgres-operator/pull/798#issuecomment-605201260 Co-authored-by: Felix Kunde --- docs/user.md | 13 +++++++++---- manifests/complete-postgres-manifest.yaml | 2 ++ pkg/cluster/k8sres.go | 12 ++---------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/user.md b/docs/user.md index 8c79bb485..dba157d90 100644 --- a/docs/user.md +++ b/docs/user.md @@ -572,10 +572,15 @@ However, this certificate cannot be verified and thus doesn't protect from active MITM attacks. In this section we show how to specify a custom TLS certificate which is mounted in the database pods via a K8s Secret. -Before applying these changes, the operator must also be configured with the -`spilo_fsgroup` set to the GID matching the postgres user group. If the value -is not provided, the cluster will default to `103` which is the GID from the -default spilo image. +Before applying these changes, in k8s the operator must also be configured with +the `spilo_fsgroup` set to the GID matching the postgres user group. If you +don't know the value, use `103` which is the GID from the default spilo image +(`spilo_fsgroup=103` in the cluster request spec). + +OpenShift allocates the users and groups dynamically (based on scc), and their +range is different in every namespace. Due to this dynamic behaviour, it's not +trivial to know at deploy time the uid/gid of the user in the cluster. +This way, in OpenShift, you may want to skip the spilo_fsgroup setting. Upload the cert as a kubernetes secret: ```sh diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index c82f1eac5..27dfc5f93 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -109,3 +109,5 @@ spec: certificateFile: "tls.crt" privateKeyFile: "tls.key" caFile: "" # optionally configure Postgres with a CA certificate +# When TLS is enabled, also set spiloFSGroup parameter above to the relevant value. +# if unknown, set it to 103 which is the usual value in the default spilo images. diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index c4919c62d..ee46f81e7 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -37,9 +37,6 @@ const ( localHost = "127.0.0.1/32" connectionPoolContainer = "connection-pool" pgPort = 5432 - - // the gid of the postgres user in the default spilo image - spiloPostgresGID = 103 ) type pgUser struct { @@ -990,13 +987,8 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // configure TLS with a custom secret volume if spec.TLS != nil && spec.TLS.SecretName != "" { - if effectiveFSGroup == nil { - c.logger.Warnf("Setting the default FSGroup to satisfy the TLS configuration") - fsGroup := int64(spiloPostgresGID) - effectiveFSGroup = &fsGroup - } - // this is combined with the FSGroup above to give read access to the - // postgres user + // this is combined with the FSGroup in the section above + // to give read access to the postgres user defaultMode := int32(0640) volumes = append(volumes, v1.Volume{ Name: "tls-secret", From e6eb10d28a8ee843b9389dee2dda7441320f9cd6 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 1 Apr 2020 10:31:31 +0200 Subject: [PATCH 073/104] fix TestTLS (#894) --- pkg/cluster/k8sres_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index e04b281ba..5772f69d1 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -958,6 +958,7 @@ func TestTLS(t *testing.T) { var err error var spec acidv1.PostgresSpec var cluster *Cluster + var spiloFSGroup = int64(103) makeSpec := func(tls acidv1.TLSDescription) acidv1.PostgresSpec { return acidv1.PostgresSpec{ @@ -982,6 +983,9 @@ func TestTLS(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, + Resources: config.Resources{ + SpiloFSGroup: &spiloFSGroup, + }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) spec = makeSpec(acidv1.TLSDescription{SecretName: "my-secret", CAFile: "ca.crt"}) From b43b22dfcced21098308dd0bf3111716772e715c Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 1 Apr 2020 10:34:03 +0200 Subject: [PATCH 074/104] Call me pooler, not pool (#883) * rename pooler parts and add example to manifest * update codegen * fix manifest and add more details to docs * reflect renaming also in e2e tests --- README.md | 3 +- .../crds/operatorconfigurations.yaml | 22 +-- .../postgres-operator/crds/postgresqls.yaml | 4 +- .../templates/configmap.yaml | 2 +- .../templates/operatorconfiguration.yaml | 4 +- charts/postgres-operator/values-crd.yaml | 22 +-- charts/postgres-operator/values.yaml | 22 +-- docs/reference/cluster_manifest.md | 26 ++-- docs/reference/operator_parameters.md | 36 ++--- docs/user.md | 47 ++++--- e2e/tests/test_e2e.py | 26 ++-- manifests/complete-postgres-manifest.yaml | 14 ++ manifests/configmap.yaml | 20 +-- manifests/operatorconfiguration.crd.yaml | 22 +-- ...gresql-operator-default-configuration.yaml | 22 +-- manifests/postgresql.crd.yaml | 4 +- pkg/apis/acid.zalan.do/v1/crds.go | 26 ++-- .../v1/operator_configuration_type.go | 26 ++-- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 8 +- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 28 ++-- pkg/cluster/cluster.go | 106 +++++++------- pkg/cluster/cluster_test.go | 14 +- pkg/cluster/database.go | 42 +++--- pkg/cluster/k8sres.go | 130 +++++++++--------- pkg/cluster/k8sres_test.go | 118 ++++++++-------- pkg/cluster/resources.go | 76 +++++----- pkg/cluster/resources_test.go | 76 +++++----- pkg/cluster/sync.go | 126 ++++++++--------- pkg/cluster/sync_test.go | 58 ++++---- pkg/cluster/util.go | 28 ++-- pkg/controller/operator_config.go | 62 ++++----- pkg/spec/types.go | 6 +- pkg/util/config/config.go | 30 ++-- pkg/util/constants/pooler.go | 26 ++-- pkg/util/constants/roles.go | 2 +- 35 files changed, 651 insertions(+), 633 deletions(-) diff --git a/README.md b/README.md index a2771efa6..564bb68a1 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,9 @@ pipelines with no access to Kubernetes directly. * Rolling updates on Postgres cluster changes * Volume resize without Pod restarts +* Database connection pooler * Cloning Postgres clusters -* Logical Backups to S3 Bucket +* Logical backups to S3 Bucket * Standby cluster from S3 WAL archive * Configurable for non-cloud environments * UI to create and edit Postgres cluster manifests diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 7e3b607c0..63e52fd7a 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -318,44 +318,44 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string - connection_pool: + connection_pooler: type: object properties: - connection_pool_schema: + connection_pooler_schema: type: string #default: "pooler" - connection_pool_user: + connection_pooler_user: type: string #default: "pooler" - connection_pool_image: + connection_pooler_image: type: string #default: "registry.opensource.zalan.do/acid/pgbouncer" - connection_pool_max_db_connections: + connection_pooler_max_db_connections: type: integer #default: 60 - connection_pool_mode: + connection_pooler_mode: type: string enum: - "session" - "transaction" #default: "transaction" - connection_pool_number_of_instances: + connection_pooler_number_of_instances: type: integer minimum: 2 #default: 2 - connection_pool_default_cpu_limit: + connection_pooler_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' #default: "1" - connection_pool_default_cpu_request: + connection_pooler_default_cpu_request: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' #default: "500m" - connection_pool_default_memory_limit: + connection_pooler_default_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' #default: "100Mi" - connection_pool_default_memory_request: + connection_pooler_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' #default: "100Mi" diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index a4c0e4f3a..57875cba7 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -106,7 +106,7 @@ spec: uid: format: uuid type: string - connectionPool: + connectionPooler: type: object properties: dockerImage: @@ -162,7 +162,7 @@ spec: # Note: usernames specified here as database owners must be declared in the users key of the spec key. dockerImage: type: string - enableConnectionPool: + enableConnectionPooler: type: boolean enableLogicalBackup: type: boolean diff --git a/charts/postgres-operator/templates/configmap.yaml b/charts/postgres-operator/templates/configmap.yaml index e8a805db7..64b55e0df 100644 --- a/charts/postgres-operator/templates/configmap.yaml +++ b/charts/postgres-operator/templates/configmap.yaml @@ -20,5 +20,5 @@ data: {{ toYaml .Values.configDebug | indent 2 }} {{ toYaml .Values.configLoggingRestApi | indent 2 }} {{ toYaml .Values.configTeamsApi | indent 2 }} -{{ toYaml .Values.configConnectionPool | indent 2 }} +{{ toYaml .Values.configConnectionPooler | indent 2 }} {{- end }} diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index b52b3d664..5df6e6238 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -34,6 +34,6 @@ configuration: {{ toYaml .Values.configLoggingRestApi | indent 4 }} scalyr: {{ toYaml .Values.configScalyr | indent 4 }} - connection_pool: -{{ toYaml .Values.configConnectionPool | indent 4 }} + connection_pooler: +{{ toYaml .Values.configConnectionPooler | indent 4 }} {{- end }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 79940b236..2490cfec6 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -267,24 +267,24 @@ configScalyr: # Memory request value for the Scalyr sidecar scalyr_memory_request: 50Mi -configConnectionPool: +configConnectionPooler: # db schema to install lookup function into - connection_pool_schema: "pooler" + connection_pooler_schema: "pooler" # db user for pooler to use - connection_pool_user: "pooler" + connection_pooler_user: "pooler" # docker image - connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" # max db connections the pooler should hold - connection_pool_max_db_connections: 60 + connection_pooler_max_db_connections: 60 # default pooling mode - connection_pool_mode: "transaction" + connection_pooler_mode: "transaction" # number of pooler instances - connection_pool_number_of_instances: 2 + connection_pooler_number_of_instances: 2 # default resources - connection_pool_default_cpu_request: 500m - connection_pool_default_memory_request: 100Mi - connection_pool_default_cpu_limit: "1" - connection_pool_default_memory_limit: 100Mi + connection_pooler_default_cpu_request: 500m + connection_pooler_default_memory_request: 100Mi + connection_pooler_default_cpu_limit: "1" + connection_pooler_default_memory_limit: 100Mi rbac: # Specifies whether RBAC resources should be created diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 29f85339d..8d2dd8c5c 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -244,24 +244,24 @@ configTeamsApi: # teams_api_url: http://fake-teams-api.default.svc.cluster.local # configure connection pooler deployment created by the operator -configConnectionPool: +configConnectionPooler: # db schema to install lookup function into - connection_pool_schema: "pooler" + connection_pooler_schema: "pooler" # db user for pooler to use - connection_pool_user: "pooler" + connection_pooler_user: "pooler" # docker image - connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" # max db connections the pooler should hold - connection_pool_max_db_connections: 60 + connection_pooler_max_db_connections: 60 # default pooling mode - connection_pool_mode: "transaction" + connection_pooler_mode: "transaction" # number of pooler instances - connection_pool_number_of_instances: 2 + connection_pooler_number_of_instances: 2 # default resources - connection_pool_default_cpu_request: 500m - connection_pool_default_memory_request: 100Mi - connection_pool_default_cpu_limit: "1" - connection_pool_default_memory_limit: 100Mi + connection_pooler_default_cpu_request: 500m + connection_pooler_default_memory_request: 100Mi + connection_pooler_default_cpu_limit: "1" + connection_pooler_default_memory_limit: 100Mi rbac: # Specifies whether RBAC resources should be created diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 4400cb666..967b2de5d 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -140,10 +140,10 @@ These parameters are grouped directly under the `spec` key in the manifest. is `false`, then no volume will be mounted no matter how operator was configured (so you can override the operator configuration). Optional. -* **enableConnectionPool** - Tells the operator to create a connection pool with a database. If this - field is true, a connection pool deployment will be created even if - `connectionPool` section is empty. Optional, not set by default. +* **enableConnectionPooler** + Tells the operator to create a connection pooler with a database. If this + field is true, a connection pooler deployment will be created even if + `connectionPooler` section is empty. Optional, not set by default. * **enableLogicalBackup** Determines if the logical backup of this cluster should be taken and uploaded @@ -365,34 +365,34 @@ CPU and memory limits for the sidecar container. memory limits for the sidecar container. Optional, overrides the `default_memory_limits` operator configuration parameter. Optional. -## Connection pool +## Connection pooler -Parameters are grouped under the `connectionPool` top-level key and specify -configuration for connection pool. If this section is not empty, a connection -pool will be created for a database even if `enableConnectionPool` is not +Parameters are grouped under the `connectionPooler` top-level key and specify +configuration for connection pooler. If this section is not empty, a connection +pooler will be created for a database even if `enableConnectionPooler` is not present. * **numberOfInstances** - How many instances of connection pool to create. + How many instances of connection pooler to create. * **schema** Schema to create for credentials lookup function. * **user** - User to create for connection pool to be able to connect to a database. + User to create for connection pooler to be able to connect to a database. * **dockerImage** - Which docker image to use for connection pool deployment. + Which docker image to use for connection pooler deployment. * **maxDBConnections** How many connections the pooler can max hold. This value is divided among the pooler pods. * **mode** - In which mode to run connection pool, transaction or session. + In which mode to run connection pooler, transaction or session. * **resources** - Resource configuration for connection pool deployment. + Resource configuration for connection pooler deployment. ## Custom TLS certificates diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 1ab92a287..c25ca3d49 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -597,39 +597,39 @@ scalyr sidecar. In the CRD-based configuration they are grouped under the * **scalyr_memory_limit** Memory limit value for the Scalyr sidecar. The default is `500Mi`. -## Connection pool configuration +## Connection pooler configuration -Parameters are grouped under the `connection_pool` top-level key and specify -default configuration for connection pool, if a postgres manifest requests it +Parameters are grouped under the `connection_pooler` top-level key and specify +default configuration for connection pooler, if a postgres manifest requests it but do not specify some of the parameters. All of them are optional with the operator being able to provide some reasonable defaults. -* **connection_pool_number_of_instances** - How many instances of connection pool to create. Default is 2 which is also +* **connection_pooler_number_of_instances** + How many instances of connection pooler to create. Default is 2 which is also the required minimum. -* **connection_pool_schema** +* **connection_pooler_schema** Schema to create for credentials lookup function. Default is `pooler`. -* **connection_pool_user** - User to create for connection pool to be able to connect to a database. +* **connection_pooler_user** + User to create for connection pooler to be able to connect to a database. Default is `pooler`. -* **connection_pool_image** - Docker image to use for connection pool deployment. +* **connection_pooler_image** + Docker image to use for connection pooler deployment. Default: "registry.opensource.zalan.do/acid/pgbouncer" -* **connection_pool_max_db_connections** +* **connection_pooler_max_db_connections** How many connections the pooler can max hold. This value is divided among the pooler pods. Default is 60 which will make up 30 connections per pod for the default setup with two instances. -* **connection_pool_mode** - Default pool mode, `session` or `transaction`. Default is `transaction`. +* **connection_pooler_mode** + Default pooler mode, `session` or `transaction`. Default is `transaction`. -* **connection_pool_default_cpu_request** - **connection_pool_default_memory_reques** - **connection_pool_default_cpu_limit** - **connection_pool_default_memory_limit** - Default resource configuration for connection pool deployment. The internal +* **connection_pooler_default_cpu_request** + **connection_pooler_default_memory_reques** + **connection_pooler_default_cpu_limit** + **connection_pooler_default_memory_limit** + Default resource configuration for connection pooler deployment. The internal default for memory request and limit is `100Mi`, for CPU it is `500m` and `1`. diff --git a/docs/user.md b/docs/user.md index dba157d90..67ed5971f 100644 --- a/docs/user.md +++ b/docs/user.md @@ -512,39 +512,38 @@ monitoring is outside the scope of operator responsibilities. See [administrator documentation](administrator.md) for details on how backups are executed. -## Connection pool +## Connection pooler -The operator can create a database side connection pool for those applications, -where an application side pool is not feasible, but a number of connections is -high. To create a connection pool together with a database, modify the +The operator can create a database side connection pooler for those applications +where an application side pooler is not feasible, but a number of connections is +high. To create a connection pooler together with a database, modify the manifest: ```yaml spec: - enableConnectionPool: true + enableConnectionPooler: true ``` -This will tell the operator to create a connection pool with default +This will tell the operator to create a connection pooler with default configuration, through which one can access the master via a separate service -`{cluster-name}-pooler`. In most of the cases provided default configuration -should be good enough. - -To configure a new connection pool, specify: +`{cluster-name}-pooler`. In most of the cases the +[default configuration](reference/operator_parameters.md#connection-pool-configuration) +should be good enough. To configure a new connection pooler individually for +each Postgres cluster, specify: ``` spec: - connectionPool: - # how many instances of connection pool to create - number_of_instances: 2 + connectionPooler: + # how many instances of connection pooler to create + numberOfInstances: 2 # in which mode to run, session or transaction mode: "transaction" - # schema, which operator will create to install credentials lookup - # function + # schema, which operator will create to install credentials lookup function schema: "pooler" - # user, which operator will create for connection pool + # user, which operator will create for connection pooler user: "pooler" # resources for each instance @@ -557,13 +556,17 @@ spec: memory: 100Mi ``` -By default `pgbouncer` is used to create a connection pool. To find out about -pool modes see [docs](https://www.pgbouncer.org/config.html#pool_mode) (but it -should be general approach between different implementation). +The `enableConnectionPooler` flag is not required when the `connectionPooler` +section is present in the manifest. But, it can be used to disable/remove the +pooler while keeping its configuration. -Note, that using `pgbouncer` means meaningful resource CPU limit should be less -than 1 core (there is a way to utilize more than one, but in K8S it's easier -just to spin up more instances). +By default, `pgbouncer` is used as connection pooler. To find out about pooler +modes read the `pgbouncer` [docs](https://www.pgbouncer.org/config.html#pooler_mode) +(but it should be the general approach between different implementation). + +Note, that using `pgbouncer` a meaningful resource CPU limit should be 1 core +or less (there is a way to utilize more than one, but in K8s it's easier just to +spin up more instances). ## Custom TLS certificates diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index f6fc8184c..445067d61 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -70,32 +70,32 @@ class EndToEndTestCase(unittest.TestCase): raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_enable_disable_connection_pool(self): + def test_enable_disable_connection_pooler(self): ''' - For a database without connection pool, then turns it on, scale up, + For a database without connection pooler, then turns it on, scale up, turn off and on again. Test with different ways of doing this (via - enableConnectionPool or connectionPool configuration section). At the - end turn the connection pool off to not interfere with other tests. + enableConnectionPooler or connectionPooler configuration section). At + the end turn connection pooler off to not interfere with other tests. ''' k8s = self.k8s service_labels = { 'cluster-name': 'acid-minimal-cluster', } pod_labels = dict({ - 'connection-pool': 'acid-minimal-cluster-pooler', + 'connection-pooler': 'acid-minimal-cluster-pooler', }) pod_selector = to_selector(pod_labels) service_selector = to_selector(service_labels) try: - # enable connection pool + # enable connection pooler k8s.api.custom_objects_api.patch_namespaced_custom_object( 'acid.zalan.do', 'v1', 'default', 'postgresqls', 'acid-minimal-cluster', { 'spec': { - 'enableConnectionPool': True, + 'enableConnectionPooler': True, } }) k8s.wait_for_pod_start(pod_selector) @@ -104,7 +104,7 @@ class EndToEndTestCase(unittest.TestCase): 'default', label_selector=pod_selector ).items - self.assertTrue(pods, 'No connection pool pods') + self.assertTrue(pods, 'No connection pooler pods') k8s.wait_for_service(service_selector) services = k8s.api.core_v1.list_namespaced_service( @@ -115,15 +115,15 @@ class EndToEndTestCase(unittest.TestCase): if s.metadata.name.endswith('pooler') ] - self.assertTrue(services, 'No connection pool service') + self.assertTrue(services, 'No connection pooler service') - # scale up connection pool deployment + # scale up connection pooler deployment k8s.api.custom_objects_api.patch_namespaced_custom_object( 'acid.zalan.do', 'v1', 'default', 'postgresqls', 'acid-minimal-cluster', { 'spec': { - 'connectionPool': { + 'connectionPooler': { 'numberOfInstances': 2, }, } @@ -137,7 +137,7 @@ class EndToEndTestCase(unittest.TestCase): 'postgresqls', 'acid-minimal-cluster', { 'spec': { - 'enableConnectionPool': False, + 'enableConnectionPooler': False, } }) k8s.wait_for_pods_to_stop(pod_selector) @@ -147,7 +147,7 @@ class EndToEndTestCase(unittest.TestCase): 'postgresqls', 'acid-minimal-cluster', { 'spec': { - 'enableConnectionPool': True, + 'enableConnectionPooler': True, } }) k8s.wait_for_pod_start(pod_selector) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 27dfc5f93..a5811504e 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -19,6 +19,7 @@ spec: - createdb enableMasterLoadBalancer: false enableReplicaLoadBalancer: false +# enableConnectionPooler: true # not needed when connectionPooler section is present (see below) allowedSourceRanges: # load balancers' source ranges for both master and replica services - 127.0.0.1/32 databases: @@ -85,6 +86,19 @@ spec: # - 01:00-06:00 #UTC # - Sat:00:00-04:00 + connectionPooler: + numberOfInstances: 2 + mode: "transaction" + schema: "pooler" + user: "pooler" + resources: + requests: + cpu: 300m + memory: 100Mi + limits: + cpu: "1" + memory: 100Mi + initContainers: - name: date image: busybox diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 67c3368f3..c74a906d1 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -11,16 +11,16 @@ data: cluster_history_entries: "1000" cluster_labels: application:spilo cluster_name_label: cluster-name - # connection_pool_default_cpu_limit: "1" - # connection_pool_default_cpu_request: "500m" - # connection_pool_default_memory_limit: 100Mi - # connection_pool_default_memory_request: 100Mi - connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" - # connection_pool_max_db_connections: 60 - # connection_pool_mode: "transaction" - # connection_pool_number_of_instances: 2 - # connection_pool_schema: "pooler" - # connection_pool_user: "pooler" + # connection_pooler_default_cpu_limit: "1" + # connection_pooler_default_cpu_request: "500m" + # connection_pooler_default_memory_limit: 100Mi + # connection_pooler_default_memory_request: 100Mi + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + # connection_pooler_max_db_connections: 60 + # connection_pooler_mode: "transaction" + # connection_pooler_number_of_instances: 2 + # connection_pooler_schema: "pooler" + # connection_pooler_user: "pooler" # custom_service_annotations: "keyx:valuez,keya:valuea" # custom_pod_annotations: "keya:valuea,keyb:valueb" db_hosted_zone: db.example.com diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 4e6858af8..1d5544c7c 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -294,44 +294,44 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string - connection_pool: + connection_pooler: type: object properties: - connection_pool_schema: + connection_pooler_schema: type: string #default: "pooler" - connection_pool_user: + connection_pooler_user: type: string #default: "pooler" - connection_pool_image: + connection_pooler_image: type: string #default: "registry.opensource.zalan.do/acid/pgbouncer" - connection_pool_max_db_connections: + connection_pooler_max_db_connections: type: integer #default: 60 - connection_pool_mode: + connection_pooler_mode: type: string enum: - "session" - "transaction" #default: "transaction" - connection_pool_number_of_instances: + connection_pooler_number_of_instances: type: integer minimum: 2 #default: 2 - connection_pool_default_cpu_limit: + connection_pooler_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' #default: "1" - connection_pool_default_cpu_request: + connection_pooler_default_cpu_request: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' #default: "500m" - connection_pool_default_memory_limit: + connection_pooler_default_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' #default: "100Mi" - connection_pool_default_memory_request: + connection_pooler_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' #default: "100Mi" diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 9d609713c..cd739c817 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -121,14 +121,14 @@ configuration: scalyr_memory_limit: 500Mi scalyr_memory_request: 50Mi # scalyr_server_url: "" - connection_pool: - connection_pool_default_cpu_limit: "1" - connection_pool_default_cpu_request: "500m" - connection_pool_default_memory_limit: 100Mi - connection_pool_default_memory_request: 100Mi - connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" - # connection_pool_max_db_connections: 60 - connection_pool_mode: "transaction" - connection_pool_number_of_instances: 2 - # connection_pool_schema: "pooler" - # connection_pool_user: "pooler" + connection_pooler: + connection_pooler_default_cpu_limit: "1" + connection_pooler_default_cpu_request: "500m" + connection_pooler_default_memory_limit: 100Mi + connection_pooler_default_memory_request: 100Mi + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + # connection_pooler_max_db_connections: 60 + connection_pooler_mode: "transaction" + connection_pooler_number_of_instances: 2 + # connection_pooler_schema: "pooler" + # connection_pooler_user: "pooler" diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 06434da14..48f6c7397 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -70,7 +70,7 @@ spec: uid: format: uuid type: string - connectionPool: + connectionPooler: type: object properties: dockerImage: @@ -126,7 +126,7 @@ spec: # Note: usernames specified here as database owners must be declared in the users key of the spec key. dockerImage: type: string - enableConnectionPool: + enableConnectionPooler: type: boolean enableLogicalBackup: type: boolean diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index dc552d3f4..cfccd1e56 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -177,7 +177,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, - "connectionPool": { + "connectionPooler": { Type: "object", Properties: map[string]apiextv1beta1.JSONSchemaProps{ "dockerImage": { @@ -259,7 +259,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "dockerImage": { Type: "string", }, - "enableConnectionPool": { + "enableConnectionPooler": { Type: "boolean", }, "enableLogicalBackup": { @@ -1129,32 +1129,32 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, - "connection_pool": { + "connection_pooler": { Type: "object", Properties: map[string]apiextv1beta1.JSONSchemaProps{ - "connection_pool_default_cpu_limit": { + "connection_pooler_default_cpu_limit": { Type: "string", Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", }, - "connection_pool_default_cpu_request": { + "connection_pooler_default_cpu_request": { Type: "string", Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", }, - "connection_pool_default_memory_limit": { + "connection_pooler_default_memory_limit": { Type: "string", Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", }, - "connection_pool_default_memory_request": { + "connection_pooler_default_memory_request": { Type: "string", Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", }, - "connection_pool_image": { + "connection_pooler_image": { Type: "string", }, - "connection_pool_max_db_connections": { + "connection_pooler_max_db_connections": { Type: "integer", }, - "connection_pool_mode": { + "connection_pooler_mode": { Type: "string", Enum: []apiextv1beta1.JSON{ { @@ -1165,14 +1165,14 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, - "connection_pool_number_of_instances": { + "connection_pooler_number_of_instances": { Type: "integer", Minimum: &min2, }, - "connection_pool_schema": { + "connection_pooler_schema": { Type: "string", }, - "connection_pool_user": { + "connection_pooler_user": { Type: "string", }, }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 3dbe96b7f..6eb6732a5 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -153,18 +153,18 @@ type ScalyrConfiguration struct { ScalyrMemoryLimit string `json:"scalyr_memory_limit,omitempty"` } -// Defines default configuration for connection pool -type ConnectionPoolConfiguration struct { - NumberOfInstances *int32 `json:"connection_pool_number_of_instances,omitempty"` - Schema string `json:"connection_pool_schema,omitempty"` - User string `json:"connection_pool_user,omitempty"` - Image string `json:"connection_pool_image,omitempty"` - Mode string `json:"connection_pool_mode,omitempty"` - MaxDBConnections *int32 `json:"connection_pool_max_db_connections,omitempty"` - DefaultCPURequest string `json:"connection_pool_default_cpu_request,omitempty"` - DefaultMemoryRequest string `json:"connection_pool_default_memory_request,omitempty"` - DefaultCPULimit string `json:"connection_pool_default_cpu_limit,omitempty"` - DefaultMemoryLimit string `json:"connection_pool_default_memory_limit,omitempty"` +// Defines default configuration for connection pooler +type ConnectionPoolerConfiguration struct { + NumberOfInstances *int32 `json:"connection_pooler_number_of_instances,omitempty"` + Schema string `json:"connection_pooler_schema,omitempty"` + User string `json:"connection_pooler_user,omitempty"` + Image string `json:"connection_pooler_image,omitempty"` + Mode string `json:"connection_pooler_mode,omitempty"` + MaxDBConnections *int32 `json:"connection_pooler_max_db_connections,omitempty"` + DefaultCPURequest string `json:"connection_pooler_default_cpu_request,omitempty"` + DefaultMemoryRequest string `json:"connection_pooler_default_memory_request,omitempty"` + DefaultCPULimit string `json:"connection_pooler_default_cpu_limit,omitempty"` + DefaultMemoryLimit string `json:"connection_pooler_default_memory_limit,omitempty"` } // OperatorLogicalBackupConfiguration defines configuration for logical backup @@ -203,7 +203,7 @@ type OperatorConfigurationData struct { LoggingRESTAPI LoggingRESTAPIConfiguration `json:"logging_rest_api"` Scalyr ScalyrConfiguration `json:"scalyr"` LogicalBackup OperatorLogicalBackupConfiguration `json:"logical_backup"` - ConnectionPool ConnectionPoolConfiguration `json:"connection_pool"` + ConnectionPooler ConnectionPoolerConfiguration `json:"connection_pooler"` } //Duration shortens this frequently used name diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 1784f8235..0695d3f9f 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -29,8 +29,8 @@ type PostgresSpec struct { Patroni `json:"patroni,omitempty"` Resources `json:"resources,omitempty"` - EnableConnectionPool *bool `json:"enableConnectionPool,omitempty"` - ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` + EnableConnectionPooler *bool `json:"enableConnectionPooler,omitempty"` + ConnectionPooler *ConnectionPooler `json:"connectionPooler,omitempty"` TeamID string `json:"teamId"` DockerImage string `json:"dockerImage,omitempty"` @@ -175,10 +175,10 @@ type PostgresStatus struct { // resources) // Type string `json:"type,omitempty"` // -// TODO: figure out what other important parameters of the connection pool it +// TODO: figure out what other important parameters of the connection pooler it // makes sense to expose. E.g. pool size (min/max boundaries), max client // connections etc. -type ConnectionPool struct { +type ConnectionPooler struct { NumberOfInstances *int32 `json:"numberOfInstances,omitempty"` Schema string `json:"schema,omitempty"` User string `json:"user,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 65a19600a..92c8af34b 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -69,7 +69,7 @@ func (in *CloneDescription) DeepCopy() *CloneDescription { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConnectionPool) DeepCopyInto(out *ConnectionPool) { +func (in *ConnectionPooler) DeepCopyInto(out *ConnectionPooler) { *out = *in if in.NumberOfInstances != nil { in, out := &in.NumberOfInstances, &out.NumberOfInstances @@ -85,18 +85,18 @@ func (in *ConnectionPool) DeepCopyInto(out *ConnectionPool) { return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPool. -func (in *ConnectionPool) DeepCopy() *ConnectionPool { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPooler. +func (in *ConnectionPooler) DeepCopy() *ConnectionPooler { if in == nil { return nil } - out := new(ConnectionPool) + out := new(ConnectionPooler) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConnectionPoolConfiguration) DeepCopyInto(out *ConnectionPoolConfiguration) { +func (in *ConnectionPoolerConfiguration) DeepCopyInto(out *ConnectionPoolerConfiguration) { *out = *in if in.NumberOfInstances != nil { in, out := &in.NumberOfInstances, &out.NumberOfInstances @@ -111,12 +111,12 @@ func (in *ConnectionPoolConfiguration) DeepCopyInto(out *ConnectionPoolConfigura return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPoolConfiguration. -func (in *ConnectionPoolConfiguration) DeepCopy() *ConnectionPoolConfiguration { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPoolerConfiguration. +func (in *ConnectionPoolerConfiguration) DeepCopy() *ConnectionPoolerConfiguration { if in == nil { return nil } - out := new(ConnectionPoolConfiguration) + out := new(ConnectionPoolerConfiguration) in.DeepCopyInto(out) return out } @@ -308,7 +308,7 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData out.LoggingRESTAPI = in.LoggingRESTAPI out.Scalyr = in.Scalyr out.LogicalBackup = in.LogicalBackup - in.ConnectionPool.DeepCopyInto(&out.ConnectionPool) + in.ConnectionPooler.DeepCopyInto(&out.ConnectionPooler) return } @@ -471,14 +471,14 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { out.Volume = in.Volume in.Patroni.DeepCopyInto(&out.Patroni) out.Resources = in.Resources - if in.EnableConnectionPool != nil { - in, out := &in.EnableConnectionPool, &out.EnableConnectionPool + if in.EnableConnectionPooler != nil { + in, out := &in.EnableConnectionPooler, &out.EnableConnectionPooler *out = new(bool) **out = **in } - if in.ConnectionPool != nil { - in, out := &in.ConnectionPool, &out.ConnectionPool - *out = new(ConnectionPool) + if in.ConnectionPooler != nil { + in, out := &in.ConnectionPooler, &out.ConnectionPooler + *out = new(ConnectionPooler) (*in).DeepCopyInto(*out) } if in.SpiloFSGroup != nil { diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 45ee55300..cde2dc260 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -50,14 +50,14 @@ type Config struct { PodServiceAccountRoleBinding *rbacv1.RoleBinding } -// K8S objects that are belongs to a connection pool -type ConnectionPoolObjects struct { +// K8S objects that are belongs to a connection pooler +type ConnectionPoolerObjects struct { Deployment *appsv1.Deployment Service *v1.Service - // It could happen that a connection pool was enabled, but the operator was - // not able to properly process a corresponding event or was restarted. In - // this case we will miss missing/require situation and a lookup function + // It could happen that a connection pooler was enabled, but the operator + // was not able to properly process a corresponding event or was restarted. + // In this case we will miss missing/require situation and a lookup function // will not be installed. To avoid synchronizing it all the time to prevent // this, we can remember the result in memory at least until the next // restart. @@ -69,7 +69,7 @@ type kubeResources struct { Endpoints map[PostgresRole]*v1.Endpoints Secrets map[types.UID]*v1.Secret Statefulset *appsv1.StatefulSet - ConnectionPool *ConnectionPoolObjects + ConnectionPooler *ConnectionPoolerObjects PodDisruptionBudget *policybeta1.PodDisruptionBudget //Pods are treated separately //PVCs are treated separately @@ -337,24 +337,24 @@ func (c *Cluster) Create() error { c.logger.Errorf("could not list resources: %v", err) } - // Create connection pool deployment and services if necessary. Since we - // need to peform some operations with the database itself (e.g. install + // Create connection pooler deployment and services if necessary. Since we + // need to perform some operations with the database itself (e.g. install // lookup function), do it as the last step, when everything is available. // - // Do not consider connection pool as a strict requirement, and if + // Do not consider connection pooler as a strict requirement, and if // something fails, report warning - if c.needConnectionPool() { - if c.ConnectionPool != nil { - c.logger.Warning("Connection pool already exists in the cluster") + if c.needConnectionPooler() { + if c.ConnectionPooler != nil { + c.logger.Warning("Connection pooler already exists in the cluster") return nil } - connPool, err := c.createConnectionPool(c.installLookupFunction) + connectionPooler, err := c.createConnectionPooler(c.installLookupFunction) if err != nil { - c.logger.Warningf("could not create connection pool: %v", err) + c.logger.Warningf("could not create connection pooler: %v", err) return nil } - c.logger.Infof("connection pool %q has been successfully created", - util.NameFromMeta(connPool.Deployment.ObjectMeta)) + c.logger.Infof("connection pooler %q has been successfully created", + util.NameFromMeta(connectionPooler.Deployment.ObjectMeta)) } return nil @@ -612,11 +612,11 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } - // connection pool needs one system user created, which is done in + // connection pooler needs one system user created, which is done in // initUsers. Check if it needs to be called. sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) - needConnPool := c.needConnectionPoolWorker(&newSpec.Spec) - if !sameUsers || needConnPool { + needConnectionPooler := c.needConnectionPoolerWorker(&newSpec.Spec) + if !sameUsers || needConnectionPooler { c.logger.Debugf("syncing secrets") if err := c.initUsers(); err != nil { c.logger.Errorf("could not init users: %v", err) @@ -740,9 +740,9 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } - // sync connection pool - if err := c.syncConnectionPool(oldSpec, newSpec, c.installLookupFunction); err != nil { - return fmt.Errorf("could not sync connection pool: %v", err) + // sync connection pooler + if err := c.syncConnectionPooler(oldSpec, newSpec, c.installLookupFunction); err != nil { + return fmt.Errorf("could not sync connection pooler: %v", err) } return nil @@ -796,11 +796,11 @@ func (c *Cluster) Delete() { c.logger.Warningf("could not remove leftover patroni objects; %v", err) } - // Delete connection pool objects anyway, even if it's not mentioned in the + // Delete connection pooler objects anyway, even if it's not mentioned in the // manifest, just to not keep orphaned components in case if something went // wrong - if err := c.deleteConnectionPool(); err != nil { - c.logger.Warningf("could not remove connection pool: %v", err) + if err := c.deleteConnectionPooler(); err != nil { + c.logger.Warningf("could not remove connection pooler: %v", err) } } @@ -869,32 +869,32 @@ func (c *Cluster) initSystemUsers() { Password: util.RandomPassword(constants.PasswordLength), } - // Connection pool user is an exception, if requested it's going to be + // Connection pooler user is an exception, if requested it's going to be // created by operator as a normal pgUser - if c.needConnectionPool() { - // initialize empty connection pool if not done yet - if c.Spec.ConnectionPool == nil { - c.Spec.ConnectionPool = &acidv1.ConnectionPool{} + if c.needConnectionPooler() { + // initialize empty connection pooler if not done yet + if c.Spec.ConnectionPooler == nil { + c.Spec.ConnectionPooler = &acidv1.ConnectionPooler{} } username := util.Coalesce( - c.Spec.ConnectionPool.User, - c.OpConfig.ConnectionPool.User) + c.Spec.ConnectionPooler.User, + c.OpConfig.ConnectionPooler.User) // connection pooler application should be able to login with this role - connPoolUser := spec.PgUser{ - Origin: spec.RoleConnectionPool, + connectionPoolerUser := spec.PgUser{ + Origin: spec.RoleConnectionPooler, Name: username, Flags: []string{constants.RoleFlagLogin}, Password: util.RandomPassword(constants.PasswordLength), } if _, exists := c.pgUsers[username]; !exists { - c.pgUsers[username] = connPoolUser + c.pgUsers[username] = connectionPoolerUser } - if _, exists := c.systemUsers[constants.ConnectionPoolUserKeyName]; !exists { - c.systemUsers[constants.ConnectionPoolUserKeyName] = connPoolUser + if _, exists := c.systemUsers[constants.ConnectionPoolerUserKeyName]; !exists { + c.systemUsers[constants.ConnectionPoolerUserKeyName] = connectionPoolerUser } } } @@ -1224,10 +1224,10 @@ func (c *Cluster) deletePatroniClusterConfigMaps() error { return c.deleteClusterObject(get, deleteConfigMapFn, "configmap") } -// Test if two connection pool configuration needs to be synced. For simplicity +// Test if two connection pooler configuration needs to be synced. For simplicity // compare not the actual K8S objects, but the configuration itself and request // sync if there is any difference. -func (c *Cluster) needSyncConnPoolSpecs(oldSpec, newSpec *acidv1.ConnectionPool) (sync bool, reasons []string) { +func (c *Cluster) needSyncConnectionPoolerSpecs(oldSpec, newSpec *acidv1.ConnectionPooler) (sync bool, reasons []string) { reasons = []string{} sync = false @@ -1264,21 +1264,21 @@ func syncResources(a, b *v1.ResourceRequirements) bool { return false } -// Check if we need to synchronize connection pool deployment due to new +// Check if we need to synchronize connection pooler deployment due to new // defaults, that are different from what we see in the DeploymentSpec -func (c *Cluster) needSyncConnPoolDefaults( - spec *acidv1.ConnectionPool, +func (c *Cluster) needSyncConnectionPoolerDefaults( + spec *acidv1.ConnectionPooler, deployment *appsv1.Deployment) (sync bool, reasons []string) { reasons = []string{} sync = false - config := c.OpConfig.ConnectionPool + config := c.OpConfig.ConnectionPooler podTemplate := deployment.Spec.Template - poolContainer := podTemplate.Spec.Containers[constants.ConnPoolContainer] + poolerContainer := podTemplate.Spec.Containers[constants.ConnectionPoolerContainer] if spec == nil { - spec = &acidv1.ConnectionPool{} + spec = &acidv1.ConnectionPooler{} } if spec.NumberOfInstances == nil && @@ -1291,25 +1291,25 @@ func (c *Cluster) needSyncConnPoolDefaults( } if spec.DockerImage == "" && - poolContainer.Image != config.Image { + poolerContainer.Image != config.Image { sync = true msg := fmt.Sprintf("DockerImage is different (having %s, required %s)", - poolContainer.Image, config.Image) + poolerContainer.Image, config.Image) reasons = append(reasons, msg) } expectedResources, err := generateResourceRequirements(spec.Resources, - c.makeDefaultConnPoolResources()) + c.makeDefaultConnectionPoolerResources()) // An error to generate expected resources means something is not quite // right, but for the purpose of robustness do not panic here, just report // and ignore resources comparison (in the worst case there will be no // updates for new resource values). - if err == nil && syncResources(&poolContainer.Resources, expectedResources) { + if err == nil && syncResources(&poolerContainer.Resources, expectedResources) { sync = true msg := fmt.Sprintf("Resources are different (having %+v, required %+v)", - poolContainer.Resources, expectedResources) + poolerContainer.Resources, expectedResources) reasons = append(reasons, msg) } @@ -1317,13 +1317,13 @@ func (c *Cluster) needSyncConnPoolDefaults( c.logger.Warningf("Cannot generate expected resources, %v", err) } - for _, env := range poolContainer.Env { + for _, env := range poolerContainer.Env { if spec.User == "" && env.Name == "PGUSER" { ref := env.ValueFrom.SecretKeyRef.LocalObjectReference if ref.Name != c.credentialSecretName(config.User) { sync = true - msg := fmt.Sprintf("Pool user is different (having %s, required %s)", + msg := fmt.Sprintf("pooler user is different (having %s, required %s)", ref.Name, config.User) reasons = append(reasons, msg) } @@ -1331,7 +1331,7 @@ func (c *Cluster) needSyncConnPoolDefaults( if spec.Schema == "" && env.Name == "PGSCHEMA" && env.Value != config.Schema { sync = true - msg := fmt.Sprintf("Pool schema is different (having %s, required %s)", + msg := fmt.Sprintf("pooler schema is different (having %s, required %s)", env.Value, config.Schema) reasons = append(reasons, msg) } diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index a1b361642..af3092ad5 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -709,16 +709,16 @@ func TestServiceAnnotations(t *testing.T) { func TestInitSystemUsers(t *testing.T) { testName := "Test system users initialization" - // default cluster without connection pool + // default cluster without connection pooler cl.initSystemUsers() - if _, exist := cl.systemUsers[constants.ConnectionPoolUserKeyName]; exist { - t.Errorf("%s, connection pool user is present", testName) + if _, exist := cl.systemUsers[constants.ConnectionPoolerUserKeyName]; exist { + t.Errorf("%s, connection pooler user is present", testName) } - // cluster with connection pool - cl.Spec.EnableConnectionPool = boolToPointer(true) + // cluster with connection pooler + cl.Spec.EnableConnectionPooler = boolToPointer(true) cl.initSystemUsers() - if _, exist := cl.systemUsers[constants.ConnectionPoolUserKeyName]; !exist { - t.Errorf("%s, connection pool user is not present", testName) + if _, exist := cl.systemUsers[constants.ConnectionPoolerUserKeyName]; !exist { + t.Errorf("%s, connection pooler user is not present", testName) } } diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index bca68c188..28f97b5cc 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -27,13 +27,13 @@ const ( WHERE a.rolname = ANY($1) ORDER BY 1;` - getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` - createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` - alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` - connectionPoolLookup = ` - CREATE SCHEMA IF NOT EXISTS {{.pool_schema}}; + getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` + createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` + alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` + connectionPoolerLookup = ` + CREATE SCHEMA IF NOT EXISTS {{.pooler_schema}}; - CREATE OR REPLACE FUNCTION {{.pool_schema}}.user_lookup( + CREATE OR REPLACE FUNCTION {{.pooler_schema}}.user_lookup( in i_username text, out uname text, out phash text) RETURNS record AS $$ BEGIN @@ -43,11 +43,11 @@ const ( END; $$ LANGUAGE plpgsql SECURITY DEFINER; - REVOKE ALL ON FUNCTION {{.pool_schema}}.user_lookup(text) - FROM public, {{.pool_user}}; - GRANT EXECUTE ON FUNCTION {{.pool_schema}}.user_lookup(text) - TO {{.pool_user}}; - GRANT USAGE ON SCHEMA {{.pool_schema}} TO {{.pool_user}}; + REVOKE ALL ON FUNCTION {{.pooler_schema}}.user_lookup(text) + FROM public, {{.pooler_user}}; + GRANT EXECUTE ON FUNCTION {{.pooler_schema}}.user_lookup(text) + TO {{.pooler_user}}; + GRANT USAGE ON SCHEMA {{.pooler_schema}} TO {{.pooler_user}}; ` ) @@ -278,9 +278,9 @@ func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin return result } -// Creates a connection pool credentials lookup function in every database to -// perform remote authentification. -func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { +// Creates a connection pooler credentials lookup function in every database to +// perform remote authentication. +func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { var stmtBytes bytes.Buffer c.logger.Info("Installing lookup function") @@ -299,11 +299,11 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { currentDatabases, err := c.getDatabases() if err != nil { - msg := "could not get databases to install pool lookup function: %v" + msg := "could not get databases to install pooler lookup function: %v" return fmt.Errorf(msg, err) } - templater := template.Must(template.New("sql").Parse(connectionPoolLookup)) + templater := template.Must(template.New("sql").Parse(connectionPoolerLookup)) for dbname, _ := range currentDatabases { if dbname == "template0" || dbname == "template1" { @@ -314,11 +314,11 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { return fmt.Errorf("could not init database connection to %s", dbname) } - c.logger.Infof("Install pool lookup function into %s", dbname) + c.logger.Infof("Install pooler lookup function into %s", dbname) params := TemplateParams{ - "pool_schema": poolSchema, - "pool_user": poolUser, + "pooler_schema": poolerSchema, + "pooler_user": poolerUser, } if err := templater.Execute(&stmtBytes, params); err != nil { @@ -353,12 +353,12 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { continue } - c.logger.Infof("Pool lookup function installed into %s", dbname) + c.logger.Infof("pooler lookup function installed into %s", dbname) if err := c.closeDbConn(); err != nil { c.logger.Errorf("could not close database connection: %v", err) } } - c.ConnectionPool.LookupFunction = true + c.ConnectionPooler.LookupFunction = true return nil } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index ee46f81e7..d5297ab8e 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -35,7 +35,7 @@ const ( patroniPGParametersParameterName = "parameters" patroniPGHBAConfParameterName = "pg_hba" localHost = "127.0.0.1/32" - connectionPoolContainer = "connection-pool" + connectionPoolerContainer = "connection-pooler" pgPort = 5432 ) @@ -72,7 +72,7 @@ func (c *Cluster) statefulSetName() string { return c.Name } -func (c *Cluster) connPoolName() string { +func (c *Cluster) connectionPoolerName() string { return c.Name + "-pooler" } @@ -139,18 +139,18 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources { } } -// Generate default resource section for connection pool deployment, to be used -// if nothing custom is specified in the manifest -func (c *Cluster) makeDefaultConnPoolResources() acidv1.Resources { +// Generate default resource section for connection pooler deployment, to be +// used if nothing custom is specified in the manifest +func (c *Cluster) makeDefaultConnectionPoolerResources() acidv1.Resources { config := c.OpConfig defaultRequests := acidv1.ResourceDescription{ - CPU: config.ConnectionPool.ConnPoolDefaultCPURequest, - Memory: config.ConnectionPool.ConnPoolDefaultMemoryRequest, + CPU: config.ConnectionPooler.ConnectionPoolerDefaultCPURequest, + Memory: config.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest, } defaultLimits := acidv1.ResourceDescription{ - CPU: config.ConnectionPool.ConnPoolDefaultCPULimit, - Memory: config.ConnectionPool.ConnPoolDefaultMemoryLimit, + CPU: config.ConnectionPooler.ConnectionPoolerDefaultCPULimit, + Memory: config.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit, } return acidv1.Resources{ @@ -1851,33 +1851,33 @@ func (c *Cluster) getLogicalBackupJobName() (jobName string) { // // DEFAULT_SIZE is a pool size per db/user (having in mind the use case when // most of the queries coming through a connection pooler are from the same -// user to the same db). In case if we want to spin up more connection pool +// user to the same db). In case if we want to spin up more connection pooler // instances, take this into account and maintain the same number of // connections. // -// MIN_SIZE is a pool minimal size, to prevent situation when sudden workload +// MIN_SIZE is a pool's minimal size, to prevent situation when sudden workload // have to wait for spinning up a new connections. // -// RESERVE_SIZE is how many additional connections to allow for a pool. -func (c *Cluster) getConnPoolEnvVars(spec *acidv1.PostgresSpec) []v1.EnvVar { +// RESERVE_SIZE is how many additional connections to allow for a pooler. +func (c *Cluster) getConnectionPoolerEnvVars(spec *acidv1.PostgresSpec) []v1.EnvVar { effectiveMode := util.Coalesce( - spec.ConnectionPool.Mode, - c.OpConfig.ConnectionPool.Mode) + spec.ConnectionPooler.Mode, + c.OpConfig.ConnectionPooler.Mode) - numberOfInstances := spec.ConnectionPool.NumberOfInstances + numberOfInstances := spec.ConnectionPooler.NumberOfInstances if numberOfInstances == nil { numberOfInstances = util.CoalesceInt32( - c.OpConfig.ConnectionPool.NumberOfInstances, + c.OpConfig.ConnectionPooler.NumberOfInstances, k8sutil.Int32ToPointer(1)) } effectiveMaxDBConn := util.CoalesceInt32( - spec.ConnectionPool.MaxDBConnections, - c.OpConfig.ConnectionPool.MaxDBConnections) + spec.ConnectionPooler.MaxDBConnections, + c.OpConfig.ConnectionPooler.MaxDBConnections) if effectiveMaxDBConn == nil { effectiveMaxDBConn = k8sutil.Int32ToPointer( - constants.ConnPoolMaxDBConnections) + constants.ConnectionPoolerMaxDBConnections) } maxDBConn := *effectiveMaxDBConn / *numberOfInstances @@ -1888,51 +1888,51 @@ func (c *Cluster) getConnPoolEnvVars(spec *acidv1.PostgresSpec) []v1.EnvVar { return []v1.EnvVar{ { - Name: "CONNECTION_POOL_PORT", + Name: "CONNECTION_POOLER_PORT", Value: fmt.Sprint(pgPort), }, { - Name: "CONNECTION_POOL_MODE", + Name: "CONNECTION_POOLER_MODE", Value: effectiveMode, }, { - Name: "CONNECTION_POOL_DEFAULT_SIZE", + Name: "CONNECTION_POOLER_DEFAULT_SIZE", Value: fmt.Sprint(defaultSize), }, { - Name: "CONNECTION_POOL_MIN_SIZE", + Name: "CONNECTION_POOLER_MIN_SIZE", Value: fmt.Sprint(minSize), }, { - Name: "CONNECTION_POOL_RESERVE_SIZE", + Name: "CONNECTION_POOLER_RESERVE_SIZE", Value: fmt.Sprint(reserveSize), }, { - Name: "CONNECTION_POOL_MAX_CLIENT_CONN", - Value: fmt.Sprint(constants.ConnPoolMaxClientConnections), + Name: "CONNECTION_POOLER_MAX_CLIENT_CONN", + Value: fmt.Sprint(constants.ConnectionPoolerMaxClientConnections), }, { - Name: "CONNECTION_POOL_MAX_DB_CONN", + Name: "CONNECTION_POOLER_MAX_DB_CONN", Value: fmt.Sprint(maxDBConn), }, } } -func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( +func (c *Cluster) generateConnectionPoolerPodTemplate(spec *acidv1.PostgresSpec) ( *v1.PodTemplateSpec, error) { gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) resources, err := generateResourceRequirements( - spec.ConnectionPool.Resources, - c.makeDefaultConnPoolResources()) + spec.ConnectionPooler.Resources, + c.makeDefaultConnectionPoolerResources()) effectiveDockerImage := util.Coalesce( - spec.ConnectionPool.DockerImage, - c.OpConfig.ConnectionPool.Image) + spec.ConnectionPooler.DockerImage, + c.OpConfig.ConnectionPooler.Image) effectiveSchema := util.Coalesce( - spec.ConnectionPool.Schema, - c.OpConfig.ConnectionPool.Schema) + spec.ConnectionPooler.Schema, + c.OpConfig.ConnectionPooler.Schema) if err != nil { return nil, fmt.Errorf("could not generate resource requirements: %v", err) @@ -1940,8 +1940,8 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( secretSelector := func(key string) *v1.SecretKeySelector { effectiveUser := util.Coalesce( - spec.ConnectionPool.User, - c.OpConfig.ConnectionPool.User) + spec.ConnectionPooler.User, + c.OpConfig.ConnectionPooler.User) return &v1.SecretKeySelector{ LocalObjectReference: v1.LocalObjectReference{ @@ -1967,7 +1967,7 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( }, }, // the convention is to use the same schema name as - // connection pool username + // connection pooler username { Name: "PGSCHEMA", Value: effectiveSchema, @@ -1980,10 +1980,10 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( }, } - envVars = append(envVars, c.getConnPoolEnvVars(spec)...) + envVars = append(envVars, c.getConnectionPoolerEnvVars(spec)...) poolerContainer := v1.Container{ - Name: connectionPoolContainer, + Name: connectionPoolerContainer, Image: effectiveDockerImage, ImagePullPolicy: v1.PullIfNotPresent, Resources: *resources, @@ -1998,7 +1998,7 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( podTemplate := &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: c.connPoolLabelsSelector().MatchLabels, + Labels: c.connectionPoolerLabelsSelector().MatchLabels, Namespace: c.Namespace, Annotations: c.generatePodAnnotations(spec), }, @@ -2040,32 +2040,32 @@ func (c *Cluster) ownerReferences() []metav1.OwnerReference { } } -func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( +func (c *Cluster) generateConnectionPoolerDeployment(spec *acidv1.PostgresSpec) ( *appsv1.Deployment, error) { // there are two ways to enable connection pooler, either to specify a - // connectionPool section or enableConnectionPool. In the second case - // spec.connectionPool will be nil, so to make it easier to calculate + // connectionPooler section or enableConnectionPooler. In the second case + // spec.connectionPooler will be nil, so to make it easier to calculate // default values, initialize it to an empty structure. It could be done // anywhere, but here is the earliest common entry point between sync and // create code, so init here. - if spec.ConnectionPool == nil { - spec.ConnectionPool = &acidv1.ConnectionPool{} + if spec.ConnectionPooler == nil { + spec.ConnectionPooler = &acidv1.ConnectionPooler{} } - podTemplate, err := c.generateConnPoolPodTemplate(spec) - numberOfInstances := spec.ConnectionPool.NumberOfInstances + podTemplate, err := c.generateConnectionPoolerPodTemplate(spec) + numberOfInstances := spec.ConnectionPooler.NumberOfInstances if numberOfInstances == nil { numberOfInstances = util.CoalesceInt32( - c.OpConfig.ConnectionPool.NumberOfInstances, + c.OpConfig.ConnectionPooler.NumberOfInstances, k8sutil.Int32ToPointer(1)) } - if *numberOfInstances < constants.ConnPoolMinInstances { - msg := "Adjusted number of connection pool instances from %d to %d" - c.logger.Warningf(msg, numberOfInstances, constants.ConnPoolMinInstances) + if *numberOfInstances < constants.ConnectionPoolerMinInstances { + msg := "Adjusted number of connection pooler instances from %d to %d" + c.logger.Warningf(msg, numberOfInstances, constants.ConnectionPoolerMinInstances) - *numberOfInstances = constants.ConnPoolMinInstances + *numberOfInstances = constants.ConnectionPoolerMinInstances } if err != nil { @@ -2074,9 +2074,9 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: c.connPoolName(), + Name: c.connectionPoolerName(), Namespace: c.Namespace, - Labels: c.connPoolLabelsSelector().MatchLabels, + Labels: c.connectionPoolerLabelsSelector().MatchLabels, Annotations: map[string]string{}, // make StatefulSet object its owner to represent the dependency. // By itself StatefulSet is being deleted with "Orphaned" @@ -2088,7 +2088,7 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( }, Spec: appsv1.DeploymentSpec{ Replicas: numberOfInstances, - Selector: c.connPoolLabelsSelector(), + Selector: c.connectionPoolerLabelsSelector(), Template: *podTemplate, }, } @@ -2096,37 +2096,37 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( return deployment, nil } -func (c *Cluster) generateConnPoolService(spec *acidv1.PostgresSpec) *v1.Service { +func (c *Cluster) generateConnectionPoolerService(spec *acidv1.PostgresSpec) *v1.Service { // there are two ways to enable connection pooler, either to specify a - // connectionPool section or enableConnectionPool. In the second case - // spec.connectionPool will be nil, so to make it easier to calculate + // connectionPooler section or enableConnectionPooler. In the second case + // spec.connectionPooler will be nil, so to make it easier to calculate // default values, initialize it to an empty structure. It could be done // anywhere, but here is the earliest common entry point between sync and // create code, so init here. - if spec.ConnectionPool == nil { - spec.ConnectionPool = &acidv1.ConnectionPool{} + if spec.ConnectionPooler == nil { + spec.ConnectionPooler = &acidv1.ConnectionPooler{} } serviceSpec := v1.ServiceSpec{ Ports: []v1.ServicePort{ { - Name: c.connPoolName(), + Name: c.connectionPoolerName(), Port: pgPort, TargetPort: intstr.IntOrString{StrVal: c.servicePort(Master)}, }, }, Type: v1.ServiceTypeClusterIP, Selector: map[string]string{ - "connection-pool": c.connPoolName(), + "connection-pooler": c.connectionPoolerName(), }, } service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: c.connPoolName(), + Name: c.connectionPoolerName(), Namespace: c.Namespace, - Labels: c.connPoolLabelsSelector().MatchLabels, + Labels: c.connectionPoolerLabelsSelector().MatchLabels, Annotations: map[string]string{}, // make StatefulSet object its owner to represent the dependency. // By itself StatefulSet is being deleted with "Orphaned" diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 5772f69d1..4068811c3 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -587,38 +587,38 @@ func TestSecretVolume(t *testing.T) { func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { cpuReq := podSpec.Spec.Containers[0].Resources.Requests["cpu"] - if cpuReq.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPURequest { + if cpuReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest { return fmt.Errorf("CPU request doesn't match, got %s, expected %s", - cpuReq.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPURequest) + cpuReq.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest) } memReq := podSpec.Spec.Containers[0].Resources.Requests["memory"] - if memReq.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryRequest { + if memReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest { return fmt.Errorf("Memory request doesn't match, got %s, expected %s", - memReq.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryRequest) + memReq.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest) } cpuLim := podSpec.Spec.Containers[0].Resources.Limits["cpu"] - if cpuLim.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPULimit { + if cpuLim.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPULimit { return fmt.Errorf("CPU limit doesn't match, got %s, expected %s", - cpuLim.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPULimit) + cpuLim.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPULimit) } memLim := podSpec.Spec.Containers[0].Resources.Limits["memory"] - if memLim.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryLimit { + if memLim.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit { return fmt.Errorf("Memory limit doesn't match, got %s, expected %s", - memLim.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryLimit) + memLim.String(), cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit) } return nil } func testLabels(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { - poolLabels := podSpec.ObjectMeta.Labels["connection-pool"] + poolerLabels := podSpec.ObjectMeta.Labels["connection-pooler"] - if poolLabels != cluster.connPoolLabelsSelector().MatchLabels["connection-pool"] { + if poolerLabels != cluster.connectionPoolerLabelsSelector().MatchLabels["connection-pooler"] { return fmt.Errorf("Pod labels do not match, got %+v, expected %+v", - podSpec.ObjectMeta.Labels, cluster.connPoolLabelsSelector().MatchLabels) + podSpec.ObjectMeta.Labels, cluster.connectionPoolerLabelsSelector().MatchLabels) } return nil @@ -626,13 +626,13 @@ func testLabels(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { required := map[string]bool{ - "PGHOST": false, - "PGPORT": false, - "PGUSER": false, - "PGSCHEMA": false, - "PGPASSWORD": false, - "CONNECTION_POOL_MODE": false, - "CONNECTION_POOL_PORT": false, + "PGHOST": false, + "PGPORT": false, + "PGUSER": false, + "PGSCHEMA": false, + "PGPASSWORD": false, + "CONNECTION_POOLER_MODE": false, + "CONNECTION_POOLER_PORT": false, } envs := podSpec.Spec.Containers[0].Env @@ -658,8 +658,8 @@ func testCustomPodTemplate(cluster *Cluster, podSpec *v1.PodTemplateSpec) error return nil } -func TestConnPoolPodSpec(t *testing.T) { - testName := "Test connection pool pod template generation" +func TestConnectionPoolerPodSpec(t *testing.T) { + testName := "Test connection pooler pod template generation" var cluster = New( Config{ OpConfig: config.Config{ @@ -668,12 +668,12 @@ func TestConnPoolPodSpec(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{ - MaxDBConnections: int32ToPointer(60), - ConnPoolDefaultCPURequest: "100m", - ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100Mi", - ConnPoolDefaultMemoryLimit: "100Mi", + ConnectionPooler: config.ConnectionPooler{ + MaxDBConnections: int32ToPointer(60), + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) @@ -686,7 +686,7 @@ func TestConnPoolPodSpec(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{}, + ConnectionPooler: config.ConnectionPooler{}, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) @@ -702,7 +702,7 @@ func TestConnPoolPodSpec(t *testing.T) { { subTest: "default configuration", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -711,7 +711,7 @@ func TestConnPoolPodSpec(t *testing.T) { { subTest: "no default resources", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: errors.New(`could not generate resource requirements: could not fill resource requests: could not parse default CPU quantity: quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'`), cluster: clusterNoDefaultRes, @@ -720,7 +720,7 @@ func TestConnPoolPodSpec(t *testing.T) { { subTest: "default resources are set", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -729,7 +729,7 @@ func TestConnPoolPodSpec(t *testing.T) { { subTest: "labels for service", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -738,7 +738,7 @@ func TestConnPoolPodSpec(t *testing.T) { { subTest: "required envs", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -746,7 +746,7 @@ func TestConnPoolPodSpec(t *testing.T) { }, } for _, tt := range tests { - podSpec, err := tt.cluster.generateConnPoolPodTemplate(tt.spec) + podSpec, err := tt.cluster.generateConnectionPoolerPodTemplate(tt.spec) if err != tt.expected && err.Error() != tt.expected.Error() { t.Errorf("%s [%s]: Could not generate pod template,\n %+v, expected\n %+v", @@ -774,9 +774,9 @@ func testDeploymentOwnwerReference(cluster *Cluster, deployment *appsv1.Deployme func testSelector(cluster *Cluster, deployment *appsv1.Deployment) error { labels := deployment.Spec.Selector.MatchLabels - expected := cluster.connPoolLabelsSelector().MatchLabels + expected := cluster.connectionPoolerLabelsSelector().MatchLabels - if labels["connection-pool"] != expected["connection-pool"] { + if labels["connection-pooler"] != expected["connection-pooler"] { return fmt.Errorf("Labels are incorrect, got %+v, expected %+v", labels, expected) } @@ -784,8 +784,8 @@ func testSelector(cluster *Cluster, deployment *appsv1.Deployment) error { return nil } -func TestConnPoolDeploymentSpec(t *testing.T) { - testName := "Test connection pool deployment spec generation" +func TestConnectionPoolerDeploymentSpec(t *testing.T) { + testName := "Test connection pooler deployment spec generation" var cluster = New( Config{ OpConfig: config.Config{ @@ -794,11 +794,11 @@ func TestConnPoolDeploymentSpec(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{ - ConnPoolDefaultCPURequest: "100m", - ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100Mi", - ConnPoolDefaultMemoryLimit: "100Mi", + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) @@ -822,7 +822,7 @@ func TestConnPoolDeploymentSpec(t *testing.T) { { subTest: "default configuration", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -831,7 +831,7 @@ func TestConnPoolDeploymentSpec(t *testing.T) { { subTest: "owner reference", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -840,7 +840,7 @@ func TestConnPoolDeploymentSpec(t *testing.T) { { subTest: "selector", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, expected: nil, cluster: cluster, @@ -848,7 +848,7 @@ func TestConnPoolDeploymentSpec(t *testing.T) { }, } for _, tt := range tests { - deployment, err := tt.cluster.generateConnPoolDeployment(tt.spec) + deployment, err := tt.cluster.generateConnectionPoolerDeployment(tt.spec) if err != tt.expected && err.Error() != tt.expected.Error() { t.Errorf("%s [%s]: Could not generate deployment spec,\n %+v, expected\n %+v", @@ -877,16 +877,16 @@ func testServiceOwnwerReference(cluster *Cluster, service *v1.Service) error { func testServiceSelector(cluster *Cluster, service *v1.Service) error { selector := service.Spec.Selector - if selector["connection-pool"] != cluster.connPoolName() { + if selector["connection-pooler"] != cluster.connectionPoolerName() { return fmt.Errorf("Selector is incorrect, got %s, expected %s", - selector["connection-pool"], cluster.connPoolName()) + selector["connection-pooler"], cluster.connectionPoolerName()) } return nil } -func TestConnPoolServiceSpec(t *testing.T) { - testName := "Test connection pool service spec generation" +func TestConnectionPoolerServiceSpec(t *testing.T) { + testName := "Test connection pooler service spec generation" var cluster = New( Config{ OpConfig: config.Config{ @@ -895,11 +895,11 @@ func TestConnPoolServiceSpec(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{ - ConnPoolDefaultCPURequest: "100m", - ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100Mi", - ConnPoolDefaultMemoryLimit: "100Mi", + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) @@ -922,7 +922,7 @@ func TestConnPoolServiceSpec(t *testing.T) { { subTest: "default configuration", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, cluster: cluster, check: noCheck, @@ -930,7 +930,7 @@ func TestConnPoolServiceSpec(t *testing.T) { { subTest: "owner reference", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, cluster: cluster, check: testServiceOwnwerReference, @@ -938,14 +938,14 @@ func TestConnPoolServiceSpec(t *testing.T) { { subTest: "selector", spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, cluster: cluster, check: testServiceSelector, }, } for _, tt := range tests { - service := tt.cluster.generateConnPoolService(tt.spec) + service := tt.cluster.generateConnectionPoolerService(tt.spec) if err := tt.check(cluster, service); err != nil { t.Errorf("%s [%s]: Service spec is incorrect, %+v", diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index c0c731ed8..4c341aefe 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -94,37 +94,37 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { return statefulSet, nil } -// Prepare the database for connection pool to be used, i.e. install lookup +// Prepare the database for connection pooler to be used, i.e. install lookup // function (do it first, because it should be fast and if it didn't succeed, // it doesn't makes sense to create more K8S objects. At this moment we assume -// that necessary connection pool user exists. +// that necessary connection pooler user exists. // -// After that create all the objects for connection pool, namely a deployment +// After that create all the objects for connection pooler, namely a deployment // with a chosen pooler and a service to expose it. -func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolObjects, error) { +func (c *Cluster) createConnectionPooler(lookup InstallFunction) (*ConnectionPoolerObjects, error) { var msg string - c.setProcessName("creating connection pool") + c.setProcessName("creating connection pooler") - schema := c.Spec.ConnectionPool.Schema + schema := c.Spec.ConnectionPooler.Schema if schema == "" { - schema = c.OpConfig.ConnectionPool.Schema + schema = c.OpConfig.ConnectionPooler.Schema } - user := c.Spec.ConnectionPool.User + user := c.Spec.ConnectionPooler.User if user == "" { - user = c.OpConfig.ConnectionPool.User + user = c.OpConfig.ConnectionPooler.User } err := lookup(schema, user) if err != nil { - msg = "could not prepare database for connection pool: %v" + msg = "could not prepare database for connection pooler: %v" return nil, fmt.Errorf(msg, err) } - deploymentSpec, err := c.generateConnPoolDeployment(&c.Spec) + deploymentSpec, err := c.generateConnectionPoolerDeployment(&c.Spec) if err != nil { - msg = "could not generate deployment for connection pool: %v" + msg = "could not generate deployment for connection pooler: %v" return nil, fmt.Errorf(msg, err) } @@ -139,7 +139,7 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolO return nil, err } - serviceSpec := c.generateConnPoolService(&c.Spec) + serviceSpec := c.generateConnectionPoolerService(&c.Spec) service, err := c.KubeClient. Services(serviceSpec.Namespace). Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) @@ -148,31 +148,31 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolO return nil, err } - c.ConnectionPool = &ConnectionPoolObjects{ + c.ConnectionPooler = &ConnectionPoolerObjects{ Deployment: deployment, Service: service, } - c.logger.Debugf("created new connection pool %q, uid: %q", + c.logger.Debugf("created new connection pooler %q, uid: %q", util.NameFromMeta(deployment.ObjectMeta), deployment.UID) - return c.ConnectionPool, nil + return c.ConnectionPooler, nil } -func (c *Cluster) deleteConnectionPool() (err error) { - c.setProcessName("deleting connection pool") - c.logger.Debugln("deleting connection pool") +func (c *Cluster) deleteConnectionPooler() (err error) { + c.setProcessName("deleting connection pooler") + c.logger.Debugln("deleting connection pooler") // Lack of connection pooler objects is not a fatal error, just log it if // it was present before in the manifest - if c.ConnectionPool == nil { - c.logger.Infof("No connection pool to delete") + if c.ConnectionPooler == nil { + c.logger.Infof("No connection pooler to delete") return nil } // Clean up the deployment object. If deployment resource we've remembered // is somehow empty, try to delete based on what would we generate - deploymentName := c.connPoolName() - deployment := c.ConnectionPool.Deployment + deploymentName := c.connectionPoolerName() + deployment := c.ConnectionPooler.Deployment if deployment != nil { deploymentName = deployment.Name @@ -187,16 +187,16 @@ func (c *Cluster) deleteConnectionPool() (err error) { Delete(context.TODO(), deploymentName, options) if !k8sutil.ResourceNotFound(err) { - c.logger.Debugf("Connection pool deployment was already deleted") + c.logger.Debugf("Connection pooler deployment was already deleted") } else if err != nil { return fmt.Errorf("could not delete deployment: %v", err) } - c.logger.Infof("Connection pool deployment %q has been deleted", deploymentName) + c.logger.Infof("Connection pooler deployment %q has been deleted", deploymentName) // Repeat the same for the service object - service := c.ConnectionPool.Service - serviceName := c.connPoolName() + service := c.ConnectionPooler.Service + serviceName := c.connectionPoolerName() if service != nil { serviceName = service.Name @@ -209,14 +209,14 @@ func (c *Cluster) deleteConnectionPool() (err error) { Delete(context.TODO(), serviceName, options) if !k8sutil.ResourceNotFound(err) { - c.logger.Debugf("Connection pool service was already deleted") + c.logger.Debugf("Connection pooler service was already deleted") } else if err != nil { return fmt.Errorf("could not delete service: %v", err) } - c.logger.Infof("Connection pool service %q has been deleted", serviceName) + c.logger.Infof("Connection pooler service %q has been deleted", serviceName) - c.ConnectionPool = nil + c.ConnectionPooler = nil return nil } @@ -816,12 +816,12 @@ func (c *Cluster) GetPodDisruptionBudget() *policybeta1.PodDisruptionBudget { return c.PodDisruptionBudget } -// Perform actual patching of a connection pool deployment, assuming that all +// Perform actual patching of a connection pooler deployment, assuming that all // the check were already done before. -func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) { - c.setProcessName("updating connection pool") - if c.ConnectionPool == nil || c.ConnectionPool.Deployment == nil { - return nil, fmt.Errorf("there is no connection pool in the cluster") +func (c *Cluster) updateConnectionPoolerDeployment(oldDeploymentSpec, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) { + c.setProcessName("updating connection pooler") + if c.ConnectionPooler == nil || c.ConnectionPooler.Deployment == nil { + return nil, fmt.Errorf("there is no connection pooler in the cluster") } patchData, err := specPatch(newDeployment.Spec) @@ -833,9 +833,9 @@ func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *app // worker at one time will try to update it chances of conflicts are // minimal. deployment, err := c.KubeClient. - Deployments(c.ConnectionPool.Deployment.Namespace).Patch( + Deployments(c.ConnectionPooler.Deployment.Namespace).Patch( context.TODO(), - c.ConnectionPool.Deployment.Name, + c.ConnectionPooler.Deployment.Name, types.MergePatchType, patchData, metav1.PatchOptions{}, @@ -844,7 +844,7 @@ func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *app return nil, fmt.Errorf("could not patch deployment: %v", err) } - c.ConnectionPool.Deployment = deployment + c.ConnectionPooler.Deployment = deployment return deployment, nil } diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go index f06e96e65..2db807b38 100644 --- a/pkg/cluster/resources_test.go +++ b/pkg/cluster/resources_test.go @@ -19,8 +19,8 @@ func boolToPointer(value bool) *bool { return &value } -func TestConnPoolCreationAndDeletion(t *testing.T) { - testName := "Test connection pool creation" +func TestConnectionPoolerCreationAndDeletion(t *testing.T) { + testName := "Test connection pooler creation" var cluster = New( Config{ OpConfig: config.Config{ @@ -29,11 +29,11 @@ func TestConnPoolCreationAndDeletion(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{ - ConnPoolDefaultCPURequest: "100m", - ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100Mi", - ConnPoolDefaultMemoryLimit: "100Mi", + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) @@ -45,31 +45,31 @@ func TestConnPoolCreationAndDeletion(t *testing.T) { } cluster.Spec = acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, } - poolResources, err := cluster.createConnectionPool(mockInstallLookupFunction) + poolerResources, err := cluster.createConnectionPooler(mockInstallLookupFunction) if err != nil { - t.Errorf("%s: Cannot create connection pool, %s, %+v", - testName, err, poolResources) + t.Errorf("%s: Cannot create connection pooler, %s, %+v", + testName, err, poolerResources) } - if poolResources.Deployment == nil { - t.Errorf("%s: Connection pool deployment is empty", testName) + if poolerResources.Deployment == nil { + t.Errorf("%s: Connection pooler deployment is empty", testName) } - if poolResources.Service == nil { - t.Errorf("%s: Connection pool service is empty", testName) + if poolerResources.Service == nil { + t.Errorf("%s: Connection pooler service is empty", testName) } - err = cluster.deleteConnectionPool() + err = cluster.deleteConnectionPooler() if err != nil { - t.Errorf("%s: Cannot delete connection pool, %s", testName, err) + t.Errorf("%s: Cannot delete connection pooler, %s", testName, err) } } -func TestNeedConnPool(t *testing.T) { - testName := "Test how connection pool can be enabled" +func TestNeedConnectionPooler(t *testing.T) { + testName := "Test how connection pooler can be enabled" var cluster = New( Config{ OpConfig: config.Config{ @@ -78,50 +78,50 @@ func TestNeedConnPool(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{ - ConnPoolDefaultCPURequest: "100m", - ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100Mi", - ConnPoolDefaultMemoryLimit: "100Mi", + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) cluster.Spec = acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, } - if !cluster.needConnectionPool() { - t.Errorf("%s: Connection pool is not enabled with full definition", + if !cluster.needConnectionPooler() { + t.Errorf("%s: Connection pooler is not enabled with full definition", testName) } cluster.Spec = acidv1.PostgresSpec{ - EnableConnectionPool: boolToPointer(true), + EnableConnectionPooler: boolToPointer(true), } - if !cluster.needConnectionPool() { - t.Errorf("%s: Connection pool is not enabled with flag", + if !cluster.needConnectionPooler() { + t.Errorf("%s: Connection pooler is not enabled with flag", testName) } cluster.Spec = acidv1.PostgresSpec{ - EnableConnectionPool: boolToPointer(false), - ConnectionPool: &acidv1.ConnectionPool{}, + EnableConnectionPooler: boolToPointer(false), + ConnectionPooler: &acidv1.ConnectionPooler{}, } - if cluster.needConnectionPool() { - t.Errorf("%s: Connection pool is still enabled with flag being false", + if cluster.needConnectionPooler() { + t.Errorf("%s: Connection pooler is still enabled with flag being false", testName) } cluster.Spec = acidv1.PostgresSpec{ - EnableConnectionPool: boolToPointer(true), - ConnectionPool: &acidv1.ConnectionPool{}, + EnableConnectionPooler: boolToPointer(true), + ConnectionPooler: &acidv1.ConnectionPooler{}, } - if !cluster.needConnectionPool() { - t.Errorf("%s: Connection pool is not enabled with flag and full", + if !cluster.needConnectionPooler() { + t.Errorf("%s: Connection pooler is not enabled with flag and full", testName) } } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index eb3835787..a423c2f0e 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -110,9 +110,9 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } - // sync connection pool - if err = c.syncConnectionPool(&oldSpec, newSpec, c.installLookupFunction); err != nil { - return fmt.Errorf("could not sync connection pool: %v", err) + // sync connection pooler + if err = c.syncConnectionPooler(&oldSpec, newSpec, c.installLookupFunction); err != nil { + return fmt.Errorf("could not sync connection pooler: %v", err) } return err @@ -478,12 +478,12 @@ func (c *Cluster) syncRoles() (err error) { userNames = append(userNames, u.Name) } - if c.needConnectionPool() { - connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] - userNames = append(userNames, connPoolUser.Name) + if c.needConnectionPooler() { + connectionPoolerUser := c.systemUsers[constants.ConnectionPoolerUserKeyName] + userNames = append(userNames, connectionPoolerUser.Name) - if _, exists := c.pgUsers[connPoolUser.Name]; !exists { - c.pgUsers[connPoolUser.Name] = connPoolUser + if _, exists := c.pgUsers[connectionPoolerUser.Name]; !exists { + c.pgUsers[connectionPoolerUser.Name] = connectionPoolerUser } } @@ -620,69 +620,69 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } -func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup InstallFunction) error { - if c.ConnectionPool == nil { - c.ConnectionPool = &ConnectionPoolObjects{} +func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, lookup InstallFunction) error { + if c.ConnectionPooler == nil { + c.ConnectionPooler = &ConnectionPoolerObjects{} } - newNeedConnPool := c.needConnectionPoolWorker(&newSpec.Spec) - oldNeedConnPool := c.needConnectionPoolWorker(&oldSpec.Spec) + newNeedConnectionPooler := c.needConnectionPoolerWorker(&newSpec.Spec) + oldNeedConnectionPooler := c.needConnectionPoolerWorker(&oldSpec.Spec) - if newNeedConnPool { - // Try to sync in any case. If we didn't needed connection pool before, + if newNeedConnectionPooler { + // Try to sync in any case. If we didn't needed connection pooler before, // it means we want to create it. If it was already present, still sync // since it could happen that there is no difference in specs, and all - // the resources are remembered, but the deployment was manualy deleted + // the resources are remembered, but the deployment was manually deleted // in between - c.logger.Debug("syncing connection pool") + c.logger.Debug("syncing connection pooler") // in this case also do not forget to install lookup function as for // creating cluster - if !oldNeedConnPool || !c.ConnectionPool.LookupFunction { - newConnPool := newSpec.Spec.ConnectionPool + if !oldNeedConnectionPooler || !c.ConnectionPooler.LookupFunction { + newConnectionPooler := newSpec.Spec.ConnectionPooler specSchema := "" specUser := "" - if newConnPool != nil { - specSchema = newConnPool.Schema - specUser = newConnPool.User + if newConnectionPooler != nil { + specSchema = newConnectionPooler.Schema + specUser = newConnectionPooler.User } schema := util.Coalesce( specSchema, - c.OpConfig.ConnectionPool.Schema) + c.OpConfig.ConnectionPooler.Schema) user := util.Coalesce( specUser, - c.OpConfig.ConnectionPool.User) + c.OpConfig.ConnectionPooler.User) if err := lookup(schema, user); err != nil { return err } } - if err := c.syncConnectionPoolWorker(oldSpec, newSpec); err != nil { - c.logger.Errorf("could not sync connection pool: %v", err) + if err := c.syncConnectionPoolerWorker(oldSpec, newSpec); err != nil { + c.logger.Errorf("could not sync connection pooler: %v", err) return err } } - if oldNeedConnPool && !newNeedConnPool { + if oldNeedConnectionPooler && !newNeedConnectionPooler { // delete and cleanup resources - if err := c.deleteConnectionPool(); err != nil { - c.logger.Warningf("could not remove connection pool: %v", err) + if err := c.deleteConnectionPooler(); err != nil { + c.logger.Warningf("could not remove connection pooler: %v", err) } } - if !oldNeedConnPool && !newNeedConnPool { + if !oldNeedConnectionPooler && !newNeedConnectionPooler { // delete and cleanup resources if not empty - if c.ConnectionPool != nil && - (c.ConnectionPool.Deployment != nil || - c.ConnectionPool.Service != nil) { + if c.ConnectionPooler != nil && + (c.ConnectionPooler.Deployment != nil || + c.ConnectionPooler.Service != nil) { - if err := c.deleteConnectionPool(); err != nil { - c.logger.Warningf("could not remove connection pool: %v", err) + if err := c.deleteConnectionPooler(); err != nil { + c.logger.Warningf("could not remove connection pooler: %v", err) } } } @@ -690,22 +690,22 @@ func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup return nil } -// Synchronize connection pool resources. Effectively we're interested only in +// Synchronize connection pooler resources. Effectively we're interested only in // synchronizing the corresponding deployment, but in case of deployment or // service is missing, create it. After checking, also remember an object for // the future references. -func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) error { +func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql) error { deployment, err := c.KubeClient. Deployments(c.Namespace). - Get(context.TODO(), c.connPoolName(), metav1.GetOptions{}) + Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{}) if err != nil && k8sutil.ResourceNotFound(err) { - msg := "Deployment %s for connection pool synchronization is not found, create it" - c.logger.Warningf(msg, c.connPoolName()) + msg := "Deployment %s for connection pooler synchronization is not found, create it" + c.logger.Warningf(msg, c.connectionPoolerName()) - deploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) + deploymentSpec, err := c.generateConnectionPoolerDeployment(&newSpec.Spec) if err != nil { - msg = "could not generate deployment for connection pool: %v" + msg = "could not generate deployment for connection pooler: %v" return fmt.Errorf(msg, err) } @@ -717,31 +717,31 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) return err } - c.ConnectionPool.Deployment = deployment + c.ConnectionPooler.Deployment = deployment } else if err != nil { - return fmt.Errorf("could not get connection pool deployment to sync: %v", err) + return fmt.Errorf("could not get connection pooler deployment to sync: %v", err) } else { - c.ConnectionPool.Deployment = deployment + c.ConnectionPooler.Deployment = deployment // actual synchronization - oldConnPool := oldSpec.Spec.ConnectionPool - newConnPool := newSpec.Spec.ConnectionPool - specSync, specReason := c.needSyncConnPoolSpecs(oldConnPool, newConnPool) - defaultsSync, defaultsReason := c.needSyncConnPoolDefaults(newConnPool, deployment) + oldConnectionPooler := oldSpec.Spec.ConnectionPooler + newConnectionPooler := newSpec.Spec.ConnectionPooler + specSync, specReason := c.needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler) + defaultsSync, defaultsReason := c.needSyncConnectionPoolerDefaults(newConnectionPooler, deployment) reason := append(specReason, defaultsReason...) if specSync || defaultsSync { - c.logger.Infof("Update connection pool deployment %s, reason: %+v", - c.connPoolName(), reason) + c.logger.Infof("Update connection pooler deployment %s, reason: %+v", + c.connectionPoolerName(), reason) - newDeploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) + newDeploymentSpec, err := c.generateConnectionPoolerDeployment(&newSpec.Spec) if err != nil { - msg := "could not generate deployment for connection pool: %v" + msg := "could not generate deployment for connection pooler: %v" return fmt.Errorf(msg, err) } - oldDeploymentSpec := c.ConnectionPool.Deployment + oldDeploymentSpec := c.ConnectionPooler.Deployment - deployment, err := c.updateConnPoolDeployment( + deployment, err := c.updateConnectionPoolerDeployment( oldDeploymentSpec, newDeploymentSpec) @@ -749,20 +749,20 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) return err } - c.ConnectionPool.Deployment = deployment + c.ConnectionPooler.Deployment = deployment return nil } } service, err := c.KubeClient. Services(c.Namespace). - Get(context.TODO(), c.connPoolName(), metav1.GetOptions{}) + Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{}) if err != nil && k8sutil.ResourceNotFound(err) { - msg := "Service %s for connection pool synchronization is not found, create it" - c.logger.Warningf(msg, c.connPoolName()) + msg := "Service %s for connection pooler synchronization is not found, create it" + c.logger.Warningf(msg, c.connectionPoolerName()) - serviceSpec := c.generateConnPoolService(&newSpec.Spec) + serviceSpec := c.generateConnectionPoolerService(&newSpec.Spec) service, err := c.KubeClient. Services(serviceSpec.Namespace). Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) @@ -771,12 +771,12 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) return err } - c.ConnectionPool.Service = service + c.ConnectionPooler.Service = service } else if err != nil { - return fmt.Errorf("could not get connection pool service to sync: %v", err) + return fmt.Errorf("could not get connection pooler service to sync: %v", err) } else { // Service updates are not supported and probably not that useful anyway - c.ConnectionPool.Service = service + c.ConnectionPooler.Service = service } return nil diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index 483d4ba58..45355cca3 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -18,8 +18,8 @@ func int32ToPointer(value int32) *int32 { } func deploymentUpdated(cluster *Cluster, err error) error { - if cluster.ConnectionPool.Deployment.Spec.Replicas == nil || - *cluster.ConnectionPool.Deployment.Spec.Replicas != 2 { + if cluster.ConnectionPooler.Deployment.Spec.Replicas == nil || + *cluster.ConnectionPooler.Deployment.Spec.Replicas != 2 { return fmt.Errorf("Wrong nubmer of instances") } @@ -27,15 +27,15 @@ func deploymentUpdated(cluster *Cluster, err error) error { } func objectsAreSaved(cluster *Cluster, err error) error { - if cluster.ConnectionPool == nil { - return fmt.Errorf("Connection pool resources are empty") + if cluster.ConnectionPooler == nil { + return fmt.Errorf("Connection pooler resources are empty") } - if cluster.ConnectionPool.Deployment == nil { + if cluster.ConnectionPooler.Deployment == nil { return fmt.Errorf("Deployment was not saved") } - if cluster.ConnectionPool.Service == nil { + if cluster.ConnectionPooler.Service == nil { return fmt.Errorf("Service was not saved") } @@ -43,15 +43,15 @@ func objectsAreSaved(cluster *Cluster, err error) error { } func objectsAreDeleted(cluster *Cluster, err error) error { - if cluster.ConnectionPool != nil { - return fmt.Errorf("Connection pool was not deleted") + if cluster.ConnectionPooler != nil { + return fmt.Errorf("Connection pooler was not deleted") } return nil } -func TestConnPoolSynchronization(t *testing.T) { - testName := "Test connection pool synchronization" +func TestConnectionPoolerSynchronization(t *testing.T) { + testName := "Test connection pooler synchronization" var cluster = New( Config{ OpConfig: config.Config{ @@ -60,12 +60,12 @@ func TestConnPoolSynchronization(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, - ConnectionPool: config.ConnectionPool{ - ConnPoolDefaultCPURequest: "100m", - ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100Mi", - ConnPoolDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: int32ToPointer(1), }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) @@ -84,15 +84,15 @@ func TestConnPoolSynchronization(t *testing.T) { clusterDirtyMock := *cluster clusterDirtyMock.KubeClient = k8sutil.NewMockKubernetesClient() - clusterDirtyMock.ConnectionPool = &ConnectionPoolObjects{ + clusterDirtyMock.ConnectionPooler = &ConnectionPoolerObjects{ Deployment: &appsv1.Deployment{}, Service: &v1.Service{}, } clusterNewDefaultsMock := *cluster clusterNewDefaultsMock.KubeClient = k8sutil.NewMockKubernetesClient() - cluster.OpConfig.ConnectionPool.Image = "pooler:2.0" - cluster.OpConfig.ConnectionPool.NumberOfInstances = int32ToPointer(2) + cluster.OpConfig.ConnectionPooler.Image = "pooler:2.0" + cluster.OpConfig.ConnectionPooler.NumberOfInstances = int32ToPointer(2) tests := []struct { subTest string @@ -105,12 +105,12 @@ func TestConnPoolSynchronization(t *testing.T) { subTest: "create if doesn't exist", oldSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, cluster: &clusterMissingObjects, @@ -123,7 +123,7 @@ func TestConnPoolSynchronization(t *testing.T) { }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - EnableConnectionPool: boolToPointer(true), + EnableConnectionPooler: boolToPointer(true), }, }, cluster: &clusterMissingObjects, @@ -136,7 +136,7 @@ func TestConnPoolSynchronization(t *testing.T) { }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, cluster: &clusterMissingObjects, @@ -146,7 +146,7 @@ func TestConnPoolSynchronization(t *testing.T) { subTest: "delete if not needed", oldSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, newSpec: &acidv1.Postgresql{ @@ -170,14 +170,14 @@ func TestConnPoolSynchronization(t *testing.T) { subTest: "update deployment", oldSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{ + ConnectionPooler: &acidv1.ConnectionPooler{ NumberOfInstances: int32ToPointer(1), }, }, }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{ + ConnectionPooler: &acidv1.ConnectionPooler{ NumberOfInstances: int32ToPointer(2), }, }, @@ -189,12 +189,12 @@ func TestConnPoolSynchronization(t *testing.T) { subTest: "update image from changed defaults", oldSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{}, + ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, cluster: &clusterNewDefaultsMock, @@ -202,7 +202,7 @@ func TestConnPoolSynchronization(t *testing.T) { }, } for _, tt := range tests { - err := tt.cluster.syncConnectionPool(tt.oldSpec, tt.newSpec, mockInstallLookupFunction) + err := tt.cluster.syncConnectionPooler(tt.oldSpec, tt.newSpec, mockInstallLookupFunction) if err := tt.check(tt.cluster, err); err != nil { t.Errorf("%s [%s]: Could not synchronize, %+v", diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 3c3dffcaf..405f48f9f 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -415,24 +415,24 @@ func (c *Cluster) labelsSelector() *metav1.LabelSelector { } } -// Return connection pool labels selector, which should from one point of view +// Return connection pooler labels selector, which should from one point of view // inherit most of the labels from the cluster itself, but at the same time // have e.g. different `application` label, so that recreatePod operation will // not interfere with it (it lists all the pods via labels, and if there would // be no difference, it will recreate also pooler pods). -func (c *Cluster) connPoolLabelsSelector() *metav1.LabelSelector { - connPoolLabels := labels.Set(map[string]string{}) +func (c *Cluster) connectionPoolerLabelsSelector() *metav1.LabelSelector { + connectionPoolerLabels := labels.Set(map[string]string{}) extraLabels := labels.Set(map[string]string{ - "connection-pool": c.connPoolName(), - "application": "db-connection-pool", + "connection-pooler": c.connectionPoolerName(), + "application": "db-connection-pooler", }) - connPoolLabels = labels.Merge(connPoolLabels, c.labelsSet(false)) - connPoolLabels = labels.Merge(connPoolLabels, extraLabels) + connectionPoolerLabels = labels.Merge(connectionPoolerLabels, c.labelsSet(false)) + connectionPoolerLabels = labels.Merge(connectionPoolerLabels, extraLabels) return &metav1.LabelSelector{ - MatchLabels: connPoolLabels, + MatchLabels: connectionPoolerLabels, MatchExpressions: nil, } } @@ -510,14 +510,14 @@ func (c *Cluster) patroniUsesKubernetes() bool { return c.OpConfig.EtcdHost == "" } -func (c *Cluster) needConnectionPoolWorker(spec *acidv1.PostgresSpec) bool { - if spec.EnableConnectionPool == nil { - return spec.ConnectionPool != nil +func (c *Cluster) needConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool { + if spec.EnableConnectionPooler == nil { + return spec.ConnectionPooler != nil } else { - return *spec.EnableConnectionPool + return *spec.EnableConnectionPooler } } -func (c *Cluster) needConnectionPool() bool { - return c.needConnectionPoolWorker(&c.Spec) +func (c *Cluster) needConnectionPooler() bool { + return c.needConnectionPoolerWorker(&c.Spec) } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index c9b3c5ea4..e9fe2f379 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -150,51 +150,51 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ScalyrCPULimit = fromCRD.Scalyr.ScalyrCPULimit result.ScalyrMemoryLimit = fromCRD.Scalyr.ScalyrMemoryLimit - // Connection pool. Looks like we can't use defaulting in CRD before 1.17, + // Connection pooler. Looks like we can't use defaulting in CRD before 1.17, // so ensure default values here. - result.ConnectionPool.NumberOfInstances = util.CoalesceInt32( - fromCRD.ConnectionPool.NumberOfInstances, + result.ConnectionPooler.NumberOfInstances = util.CoalesceInt32( + fromCRD.ConnectionPooler.NumberOfInstances, int32ToPointer(2)) - result.ConnectionPool.NumberOfInstances = util.MaxInt32( - result.ConnectionPool.NumberOfInstances, + result.ConnectionPooler.NumberOfInstances = util.MaxInt32( + result.ConnectionPooler.NumberOfInstances, int32ToPointer(2)) - result.ConnectionPool.Schema = util.Coalesce( - fromCRD.ConnectionPool.Schema, - constants.ConnectionPoolSchemaName) + result.ConnectionPooler.Schema = util.Coalesce( + fromCRD.ConnectionPooler.Schema, + constants.ConnectionPoolerSchemaName) - result.ConnectionPool.User = util.Coalesce( - fromCRD.ConnectionPool.User, - constants.ConnectionPoolUserName) + result.ConnectionPooler.User = util.Coalesce( + fromCRD.ConnectionPooler.User, + constants.ConnectionPoolerUserName) - result.ConnectionPool.Image = util.Coalesce( - fromCRD.ConnectionPool.Image, + result.ConnectionPooler.Image = util.Coalesce( + fromCRD.ConnectionPooler.Image, "registry.opensource.zalan.do/acid/pgbouncer") - result.ConnectionPool.Mode = util.Coalesce( - fromCRD.ConnectionPool.Mode, - constants.ConnectionPoolDefaultMode) + result.ConnectionPooler.Mode = util.Coalesce( + fromCRD.ConnectionPooler.Mode, + constants.ConnectionPoolerDefaultMode) - result.ConnectionPool.ConnPoolDefaultCPURequest = util.Coalesce( - fromCRD.ConnectionPool.DefaultCPURequest, - constants.ConnectionPoolDefaultCpuRequest) + result.ConnectionPooler.ConnectionPoolerDefaultCPURequest = util.Coalesce( + fromCRD.ConnectionPooler.DefaultCPURequest, + constants.ConnectionPoolerDefaultCpuRequest) - result.ConnectionPool.ConnPoolDefaultMemoryRequest = util.Coalesce( - fromCRD.ConnectionPool.DefaultMemoryRequest, - constants.ConnectionPoolDefaultMemoryRequest) + result.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest = util.Coalesce( + fromCRD.ConnectionPooler.DefaultMemoryRequest, + constants.ConnectionPoolerDefaultMemoryRequest) - result.ConnectionPool.ConnPoolDefaultCPULimit = util.Coalesce( - fromCRD.ConnectionPool.DefaultCPULimit, - constants.ConnectionPoolDefaultCpuLimit) + result.ConnectionPooler.ConnectionPoolerDefaultCPULimit = util.Coalesce( + fromCRD.ConnectionPooler.DefaultCPULimit, + constants.ConnectionPoolerDefaultCpuLimit) - result.ConnectionPool.ConnPoolDefaultMemoryLimit = util.Coalesce( - fromCRD.ConnectionPool.DefaultMemoryLimit, - constants.ConnectionPoolDefaultMemoryLimit) + result.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit = util.Coalesce( + fromCRD.ConnectionPooler.DefaultMemoryLimit, + constants.ConnectionPoolerDefaultMemoryLimit) - result.ConnectionPool.MaxDBConnections = util.CoalesceInt32( - fromCRD.ConnectionPool.MaxDBConnections, - int32ToPointer(constants.ConnPoolMaxDBConnections)) + result.ConnectionPooler.MaxDBConnections = util.CoalesceInt32( + fromCRD.ConnectionPooler.MaxDBConnections, + int32ToPointer(constants.ConnectionPoolerMaxDBConnections)) return result } diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 36783204d..e1c49a1fd 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -31,7 +31,7 @@ const ( RoleOriginInfrastructure RoleOriginTeamsAPI RoleOriginSystem - RoleConnectionPool + RoleConnectionPooler ) type syncUserOperation int @@ -180,8 +180,8 @@ func (r RoleOrigin) String() string { return "teams API role" case RoleOriginSystem: return "system role" - case RoleConnectionPool: - return "connection pool role" + case RoleConnectionPooler: + return "connection pooler role" default: panic(fmt.Sprintf("bogus role origin value %d", r)) } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 403615f06..d5464ca7c 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -85,17 +85,17 @@ type LogicalBackup struct { } // Operator options for connection pooler -type ConnectionPool struct { - NumberOfInstances *int32 `name:"connection_pool_number_of_instances" default:"2"` - Schema string `name:"connection_pool_schema" default:"pooler"` - User string `name:"connection_pool_user" default:"pooler"` - Image string `name:"connection_pool_image" default:"registry.opensource.zalan.do/acid/pgbouncer"` - Mode string `name:"connection_pool_mode" default:"transaction"` - MaxDBConnections *int32 `name:"connection_pool_max_db_connections" default:"60"` - ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"500m"` - ConnPoolDefaultMemoryRequest string `name:"connection_pool_default_memory_request" default:"100Mi"` - ConnPoolDefaultCPULimit string `name:"connection_pool_default_cpu_limit" default:"1"` - ConnPoolDefaultMemoryLimit string `name:"connection_pool_default_memory_limit" default:"100Mi"` +type ConnectionPooler struct { + NumberOfInstances *int32 `name:"connection_pooler_number_of_instances" default:"2"` + Schema string `name:"connection_pooler_schema" default:"pooler"` + User string `name:"connection_pooler_user" default:"pooler"` + Image string `name:"connection_pooler_image" default:"registry.opensource.zalan.do/acid/pgbouncer"` + Mode string `name:"connection_pooler_mode" default:"transaction"` + MaxDBConnections *int32 `name:"connection_pooler_max_db_connections" default:"60"` + ConnectionPoolerDefaultCPURequest string `name:"connection_pooler_default_cpu_request" default:"500m"` + ConnectionPoolerDefaultMemoryRequest string `name:"connection_pooler_default_memory_request" default:"100Mi"` + ConnectionPoolerDefaultCPULimit string `name:"connection_pooler_default_cpu_limit" default:"1"` + ConnectionPoolerDefaultMemoryLimit string `name:"connection_pooler_default_memory_limit" default:"100Mi"` } // Config describes operator config @@ -105,7 +105,7 @@ type Config struct { Auth Scalyr LogicalBackup - ConnectionPool + ConnectionPooler WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS @@ -213,9 +213,9 @@ func validate(cfg *Config) (err error) { err = fmt.Errorf("number of workers should be higher than 0") } - if *cfg.ConnectionPool.NumberOfInstances < constants.ConnPoolMinInstances { - msg := "number of connection pool instances should be higher than %d" - err = fmt.Errorf(msg, constants.ConnPoolMinInstances) + if *cfg.ConnectionPooler.NumberOfInstances < constants.ConnectionPoolerMinInstances { + msg := "number of connection pooler instances should be higher than %d" + err = fmt.Errorf(msg, constants.ConnectionPoolerMinInstances) } return } diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go index 540d64e2c..52e47c9cd 100644 --- a/pkg/util/constants/pooler.go +++ b/pkg/util/constants/pooler.go @@ -1,18 +1,18 @@ package constants -// Connection pool specific constants +// Connection pooler specific constants const ( - ConnectionPoolUserName = "pooler" - ConnectionPoolSchemaName = "pooler" - ConnectionPoolDefaultType = "pgbouncer" - ConnectionPoolDefaultMode = "transaction" - ConnectionPoolDefaultCpuRequest = "500m" - ConnectionPoolDefaultCpuLimit = "1" - ConnectionPoolDefaultMemoryRequest = "100Mi" - ConnectionPoolDefaultMemoryLimit = "100Mi" + ConnectionPoolerUserName = "pooler" + ConnectionPoolerSchemaName = "pooler" + ConnectionPoolerDefaultType = "pgbouncer" + ConnectionPoolerDefaultMode = "transaction" + ConnectionPoolerDefaultCpuRequest = "500m" + ConnectionPoolerDefaultCpuLimit = "1" + ConnectionPoolerDefaultMemoryRequest = "100Mi" + ConnectionPoolerDefaultMemoryLimit = "100Mi" - ConnPoolContainer = 0 - ConnPoolMaxDBConnections = 60 - ConnPoolMaxClientConnections = 10000 - ConnPoolMinInstances = 2 + ConnectionPoolerContainer = 0 + ConnectionPoolerMaxDBConnections = 60 + ConnectionPoolerMaxClientConnections = 10000 + ConnectionPoolerMinInstances = 2 ) diff --git a/pkg/util/constants/roles.go b/pkg/util/constants/roles.go index 3d201142c..c2c287472 100644 --- a/pkg/util/constants/roles.go +++ b/pkg/util/constants/roles.go @@ -4,7 +4,7 @@ package constants const ( PasswordLength = 64 SuperuserKeyName = "superuser" - ConnectionPoolUserKeyName = "pooler" + ConnectionPoolerUserKeyName = "pooler" ReplicationUserKeyName = "replication" RoleFlagSuperuser = "SUPERUSER" RoleFlagInherit = "INHERIT" From 1249626a6023d9ec9c4e52f4d47971b860a1e91e Mon Sep 17 00:00:00 2001 From: ReSearchITEng Date: Thu, 2 Apr 2020 14:20:45 +0300 Subject: [PATCH 075/104] kubernetes_use_configmap (#887) * kubernetes_use_configmap * Update manifests/postgresql-operator-default-configuration.yaml Co-Authored-By: Felix Kunde * Update manifests/configmap.yaml Co-Authored-By: Felix Kunde * Update charts/postgres-operator/values.yaml Co-Authored-By: Felix Kunde * go.fmt Co-authored-by: Felix Kunde --- .../crds/operatorconfigurations.yaml | 2 ++ charts/postgres-operator/values-crd.yaml | 2 ++ charts/postgres-operator/values.yaml | 2 ++ docs/reference/operator_parameters.md | 6 ++++++ manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 2 ++ .../postgresql-operator-default-configuration.yaml | 1 + pkg/apis/acid.zalan.do/v1/crds.go | 3 +++ .../acid.zalan.do/v1/operator_configuration_type.go | 1 + pkg/cluster/k8sres.go | 6 +++++- pkg/cluster/util.go | 9 +++++++++ pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 11 ++++++----- 13 files changed, 41 insertions(+), 6 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 63e52fd7a..7e20c8fea 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -66,6 +66,8 @@ spec: type: boolean etcd_host: type: string + kubernetes_use_configmaps: + type: boolean max_instances: type: integer minimum: -1 # -1 = disabled diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 2490cfec6..37ad6e45e 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -23,6 +23,8 @@ configGeneral: enable_shm_volume: true # etcd connection string for Patroni. Empty uses K8s-native DCS. etcd_host: "" + # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) + # kubernetes_use_configmaps: false # Spilo docker image docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 # max number of instances in Postgres cluster. -1 = no limit diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 8d2dd8c5c..3a701bb64 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -23,6 +23,8 @@ configGeneral: enable_shm_volume: "true" # etcd connection string for Patroni. Empty uses K8s-native DCS. etcd_host: "" + # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) + # kubernetes_use_configmaps: "false" # Spilo docker image docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 # max number of instances in Postgres cluster. -1 = no limit diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index c25ca3d49..9f0e0b67a 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -80,6 +80,12 @@ Those are top-level keys, containing both leaf keys and groups. Patroni native Kubernetes support is used. The default is empty (use Kubernetes-native DCS). +* **kubernetes_use_configmaps** + Select if setup uses endpoints (default), or configmaps to manage leader when + DCS is kubernetes (not etcd or similar). In OpenShift it is not possible to + use endpoints option, and configmaps is required. By default, + `kubernetes_use_configmaps: false`, meaning endpoints will be used. + * **docker_image** Spilo Docker image for Postgres instances. For production, don't rely on the default image, as it might be not the most up-to-date one. Instead, build diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index c74a906d1..fdb11f2fc 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -43,6 +43,7 @@ data: # enable_team_superuser: "false" enable_teams_api: "false" # etcd_host: "" + # kubernetes_use_configmaps: "false" # infrastructure_roles_secret_name: postgresql-infrastructure-roles # inherited_labels: application,environment # kube_iam_role: "" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 1d5544c7c..86051e43b 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -42,6 +42,8 @@ spec: type: boolean etcd_host: type: string + kubernetes_use_configmaps: + type: boolean max_instances: type: integer minimum: -1 # -1 = disabled diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index cd739c817..1f061ef7a 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -5,6 +5,7 @@ metadata: configuration: # enable_crd_validation: true etcd_host: "" + # kubernetes_use_configmaps: false docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 # enable_shm_volume: true max_instances: -1 diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index cfccd1e56..4869e9288 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -727,6 +727,9 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "etcd_host": { Type: "string", }, + "kubernetes_use_configmaps": { + Type: "boolean", + }, "max_instances": { Type: "integer", Description: "-1 = disabled", diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 6eb6732a5..8ed4281f4 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -183,6 +183,7 @@ type OperatorLogicalBackupConfiguration struct { type OperatorConfigurationData struct { EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` EtcdHost string `json:"etcd_host,omitempty"` + KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` DockerImage string `json:"docker_image,omitempty"` Workers uint32 `json:"workers,omitempty"` MinInstances int32 `json:"min_instances,omitempty"` diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index d5297ab8e..1baa4455c 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -672,6 +672,10 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri envVars = append(envVars, v1.EnvVar{Name: "ETCD_HOST", Value: c.OpConfig.EtcdHost}) } + if c.patroniKubernetesUseConfigMaps() { + envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_USE_CONFIGMAPS", Value: "true"}) + } + if cloneDescription.ClusterName != "" { envVars = append(envVars, c.generateCloneEnvironment(cloneDescription)...) } @@ -1406,7 +1410,7 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) Type: v1.ServiceTypeClusterIP, } - if role == Replica { + if role == Replica || c.patroniKubernetesUseConfigMaps() { serviceSpec.Selector = c.roleLabelsSet(false, role) } diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 405f48f9f..4dcdfb28a 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -510,6 +510,15 @@ func (c *Cluster) patroniUsesKubernetes() bool { return c.OpConfig.EtcdHost == "" } +func (c *Cluster) patroniKubernetesUseConfigMaps() bool { + if !c.patroniUsesKubernetes() { + return false + } + + // otherwise, follow the operator configuration + return c.OpConfig.KubernetesUseConfigMaps +} + func (c *Cluster) needConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool { if spec.EnableConnectionPooler == nil { return spec.ConnectionPooler != nil diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index e9fe2f379..a9c36f995 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -35,6 +35,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // general config result.EnableCRDValidation = fromCRD.EnableCRDValidation result.EtcdHost = fromCRD.EtcdHost + result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps result.DockerImage = fromCRD.DockerImage result.Workers = fromCRD.Workers result.MinInstances = fromCRD.MinInstances diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index d5464ca7c..0a7bac835 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -107,11 +107,12 @@ type Config struct { LogicalBackup ConnectionPooler - WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' - EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` - Sidecars map[string]string `name:"sidecar_docker_images"` - PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` + WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' + KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` + EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` + Sidecars map[string]string `name:"sidecar_docker_images"` + PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` // value of this string must be valid JSON or YAML; see initPodServiceAccount PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""` PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""` From 64389b8bad2940e829047799850645b02c55da5e Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 3 Apr 2020 16:28:36 +0200 Subject: [PATCH 076/104] update image and docs for connection pooler (#898) --- docs/reference/cluster_manifest.md | 3 ++- docs/reference/operator_parameters.md | 9 ++++++--- docs/user.md | 11 ++++++----- manifests/configmap.yaml | 2 +- .../postgresql-operator-default-configuration.yaml | 2 +- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 967b2de5d..f482cc218 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -376,10 +376,11 @@ present. How many instances of connection pooler to create. * **schema** - Schema to create for credentials lookup function. + Database schema to create for credentials lookup function. * **user** User to create for connection pooler to be able to connect to a database. + You can also choose a role from the `users` section or a system user role. * **dockerImage** Which docker image to use for connection pooler deployment. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 9f0e0b67a..3d31abab4 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -83,7 +83,7 @@ Those are top-level keys, containing both leaf keys and groups. * **kubernetes_use_configmaps** Select if setup uses endpoints (default), or configmaps to manage leader when DCS is kubernetes (not etcd or similar). In OpenShift it is not possible to - use endpoints option, and configmaps is required. By default, + use endpoints option, and configmaps is required. By default, `kubernetes_use_configmaps: false`, meaning endpoints will be used. * **docker_image** @@ -615,11 +615,14 @@ operator being able to provide some reasonable defaults. the required minimum. * **connection_pooler_schema** - Schema to create for credentials lookup function. Default is `pooler`. + Database schema to create for credentials lookup function to be used by the + connection pooler. Is is created in every database of the Postgres cluster. + You can also choose an existing schema. Default schema is `pooler`. * **connection_pooler_user** User to create for connection pooler to be able to connect to a database. - Default is `pooler`. + You can also choose an existing role, but make sure it has the `LOGIN` + privilege. Default role is `pooler`. * **connection_pooler_image** Docker image to use for connection pooler deployment. diff --git a/docs/user.md b/docs/user.md index 67ed5971f..1be50a01a 100644 --- a/docs/user.md +++ b/docs/user.md @@ -527,7 +527,7 @@ spec: This will tell the operator to create a connection pooler with default configuration, through which one can access the master via a separate service `{cluster-name}-pooler`. In most of the cases the -[default configuration](reference/operator_parameters.md#connection-pool-configuration) +[default configuration](reference/operator_parameters.md#connection-pooler-configuration) should be good enough. To configure a new connection pooler individually for each Postgres cluster, specify: @@ -540,7 +540,8 @@ spec: # in which mode to run, session or transaction mode: "transaction" - # schema, which operator will create to install credentials lookup function + # schema, which operator will create in each database + # to install credentials lookup function for connection pooler schema: "pooler" # user, which operator will create for connection pooler @@ -560,11 +561,11 @@ The `enableConnectionPooler` flag is not required when the `connectionPooler` section is present in the manifest. But, it can be used to disable/remove the pooler while keeping its configuration. -By default, `pgbouncer` is used as connection pooler. To find out about pooler -modes read the `pgbouncer` [docs](https://www.pgbouncer.org/config.html#pooler_mode) +By default, [`PgBouncer`](https://www.pgbouncer.org/) is used as connection pooler. +To find out about pool modes read the `PgBouncer` [docs](https://www.pgbouncer.org/config.html#pooler_mode) (but it should be the general approach between different implementation). -Note, that using `pgbouncer` a meaningful resource CPU limit should be 1 core +Note, that using `PgBouncer` a meaningful resource CPU limit should be 1 core or less (there is a way to utilize more than one, but in K8s it's easier just to spin up more instances). diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index fdb11f2fc..954881ed3 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,7 +15,7 @@ data: # connection_pooler_default_cpu_request: "500m" # connection_pooler_default_memory_limit: 100Mi # connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-6" # connection_pooler_max_db_connections: 60 # connection_pooler_mode: "transaction" # connection_pooler_number_of_instances: 2 diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 1f061ef7a..209e2684b 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -127,7 +127,7 @@ configuration: connection_pooler_default_cpu_request: "500m" connection_pooler_default_memory_limit: 100Mi connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-6" # connection_pooler_max_db_connections: 60 connection_pooler_mode: "transaction" connection_pooler_number_of_instances: 2 From 4dee8918bd45e4356e10acf15b9f71bf0a43f9bd Mon Sep 17 00:00:00 2001 From: Leon Albers <5120677+lalbers@users.noreply.github.com> Date: Mon, 6 Apr 2020 14:27:17 +0200 Subject: [PATCH 077/104] Allow configuration of patroni's replication mode (#869) * Add patroni parameters for `synchronous_mode` * Update complete-postgres-manifest.yaml, removed quotation marks * Update k8sres_test.go, adjust result for `Patroni configured` * Update k8sres_test.go, adjust result for `Patroni configured` * Update complete-postgres-manifest.yaml, set synchronous mode to false in this example * Update pkg/cluster/k8sres.go Does the same but is shorter. So we fix that it if you like. Co-Authored-By: Felix Kunde * Update docs/reference/cluster_manifest.md Co-Authored-By: Felix Kunde * Add patroni's `synchronous_mode_strict` * Extend `TestGenerateSpiloConfig` with `SynchronousModeStrict` Co-authored-by: Felix Kunde --- charts/postgres-operator/crds/postgresqls.yaml | 4 ++++ docs/reference/cluster_manifest.md | 6 ++++++ manifests/complete-postgres-manifest.yaml | 2 ++ manifests/postgresql.crd.yaml | 4 ++++ pkg/apis/acid.zalan.do/v1/crds.go | 6 ++++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 16 +++++++++------- pkg/cluster/k8sres.go | 8 ++++++++ pkg/cluster/k8sres_test.go | 16 +++++++++------- 8 files changed, 48 insertions(+), 14 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 57875cba7..f64080ed5 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -218,6 +218,10 @@ spec: type: integer retry_timeout: type: integer + synchronous_mode: + type: boolean + synchronous_mode_strict: + type: boolean maximum_lag_on_failover: type: integer podAnnotations: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index f482cc218..83dedabd3 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -217,6 +217,12 @@ explanation of `ttl` and `loop_wait` parameters. automatically created by Patroni for cluster members and permanent replication slots. Optional. +* **synchronous_mode** + Patroni `synchronous_mode` parameter value. The default is set to `false`. Optional. + +* **synchronous_mode_strict** + Patroni `synchronous_mode_strict` parameter value. Can be used in addition to `synchronous_mode`. The default is set to `false`. Optional. + ## Postgres container resources Those parameters define [CPU and memory requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index a5811504e..3a87254cf 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -67,6 +67,8 @@ spec: ttl: 30 loop_wait: &loop_wait 10 retry_timeout: 10 + synchronous_mode: false + synchronous_mode_strict: false maximum_lag_on_failover: 33554432 # restore a Postgres DB with point-in-time-recovery diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 48f6c7397..f8631f0b7 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -184,6 +184,10 @@ spec: type: integer maximum_lag_on_failover: type: integer + synchronous_mode: + type: boolean + synchronous_mode_strict: + type: boolean podAnnotations: type: object additionalProperties: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 4869e9288..60dd66ba8 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -358,6 +358,12 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "maximum_lag_on_failover": { Type: "integer", }, + "synchronous_mode": { + Type: "boolean", + }, + "synchronous_mode_strict": { + Type: "boolean", + }, }, }, "podAnnotations": { diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 0695d3f9f..b1b6a36a6 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -118,13 +118,15 @@ type Resources struct { // Patroni contains Patroni-specific configuration type Patroni struct { - InitDB map[string]string `json:"initdb"` - PgHba []string `json:"pg_hba"` - TTL uint32 `json:"ttl"` - LoopWait uint32 `json:"loop_wait"` - RetryTimeout uint32 `json:"retry_timeout"` - MaximumLagOnFailover float32 `json:"maximum_lag_on_failover"` // float32 because https://github.com/kubernetes/kubernetes/issues/30213 - Slots map[string]map[string]string `json:"slots"` + InitDB map[string]string `json:"initdb"` + PgHba []string `json:"pg_hba"` + TTL uint32 `json:"ttl"` + LoopWait uint32 `json:"loop_wait"` + RetryTimeout uint32 `json:"retry_timeout"` + MaximumLagOnFailover float32 `json:"maximum_lag_on_failover"` // float32 because https://github.com/kubernetes/kubernetes/issues/30213 + Slots map[string]map[string]string `json:"slots"` + SynchronousMode bool `json:"synchronous_mode"` + SynchronousModeStrict bool `json:"synchronous_mode_strict"` } //StandbyCluster diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 1baa4455c..8de61bdea 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -49,6 +49,8 @@ type patroniDCS struct { LoopWait uint32 `json:"loop_wait,omitempty"` RetryTimeout uint32 `json:"retry_timeout,omitempty"` MaximumLagOnFailover float32 `json:"maximum_lag_on_failover,omitempty"` + SynchronousMode bool `json:"synchronous_mode,omitempty"` + SynchronousModeStrict bool `json:"synchronous_mode_strict,omitempty"` PGBootstrapConfiguration map[string]interface{} `json:"postgresql,omitempty"` Slots map[string]map[string]string `json:"slots,omitempty"` } @@ -283,6 +285,12 @@ PatroniInitDBParams: if patroni.Slots != nil { config.Bootstrap.DCS.Slots = patroni.Slots } + if patroni.SynchronousMode { + config.Bootstrap.DCS.SynchronousMode = patroni.SynchronousMode + } + if patroni.SynchronousModeStrict != false { + config.Bootstrap.DCS.SynchronousModeStrict = patroni.SynchronousModeStrict + } config.PgLocalConfiguration = make(map[string]interface{}) config.PgLocalConfiguration[patroniPGBinariesParameterName] = fmt.Sprintf(pgBinariesLocationTemplate, pg.PgVersion) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 4068811c3..0930279d2 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -65,16 +65,18 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { "locale": "en_US.UTF-8", "data-checksums": "true", }, - PgHba: []string{"hostssl all all 0.0.0.0/0 md5", "host all all 0.0.0.0/0 md5"}, - TTL: 30, - LoopWait: 10, - RetryTimeout: 10, - MaximumLagOnFailover: 33554432, - Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, + PgHba: []string{"hostssl all all 0.0.0.0/0 md5", "host all all 0.0.0.0/0 md5"}, + TTL: 30, + LoopWait: 10, + RetryTimeout: 10, + MaximumLagOnFailover: 33554432, + SynchronousMode: true, + SynchronousModeStrict: true, + Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, }, role: "zalandos", opConfig: config.Config{}, - result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}}}}`, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}}}}`, }, } for _, tt := range tests { From 7232326159a9f77766260bc3e7b49f79dd101bd8 Mon Sep 17 00:00:00 2001 From: ReSearchITEng Date: Thu, 9 Apr 2020 10:16:45 +0300 Subject: [PATCH 078/104] Fix val docs (#901) * missing quotes in pooler configmap in values.yaml * missing quotes in pooler configmap in values-crd.yaml * docs clarifications * helm3 --skip-crds * Update docs/user.md Co-Authored-By: Felix Kunde * details moved in docs Co-authored-by: Felix Kunde --- charts/postgres-operator/values-crd.yaml | 5 +++-- charts/postgres-operator/values.yaml | 5 +++-- docs/reference/cluster_manifest.md | 8 ++++++-- docs/user.md | 3 ++- manifests/complete-postgres-manifest.yaml | 3 +++ 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 37ad6e45e..caa4dda4d 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -277,11 +277,11 @@ configConnectionPooler: # docker image connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" # max db connections the pooler should hold - connection_pooler_max_db_connections: 60 + connection_pooler_max_db_connections: "60" # default pooling mode connection_pooler_mode: "transaction" # number of pooler instances - connection_pooler_number_of_instances: 2 + connection_pooler_number_of_instances: "2" # default resources connection_pooler_default_cpu_request: 500m connection_pooler_default_memory_request: 100Mi @@ -294,6 +294,7 @@ rbac: crd: # Specifies whether custom resource definitions should be created + # When using helm3, this is ignored; instead use "--skip-crds" to skip. create: true serviceAccount: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 3a701bb64..e7db249f0 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -254,11 +254,11 @@ configConnectionPooler: # docker image connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" # max db connections the pooler should hold - connection_pooler_max_db_connections: 60 + connection_pooler_max_db_connections: "60" # default pooling mode connection_pooler_mode: "transaction" # number of pooler instances - connection_pooler_number_of_instances: 2 + connection_pooler_number_of_instances: "2" # default resources connection_pooler_default_cpu_request: 500m connection_pooler_default_memory_request: 100Mi @@ -271,6 +271,7 @@ rbac: crd: # Specifies whether custom resource definitions should be created + # When using helm3, this is ignored; instead use "--skip-crds" to skip. create: true serviceAccount: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 83dedabd3..0cbcdc08e 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -419,5 +419,9 @@ Those parameters are grouped under the `tls` top-level key. Filename of the private key. Defaults to "tls.key". * **caFile** - Optional filename to the CA certificate. Useful when the client connects - with `sslmode=verify-ca` or `sslmode=verify-full`. Default is empty. + Optional filename to the CA certificate (e.g. "ca.crt"). Useful when the + client connects with `sslmode=verify-ca` or `sslmode=verify-full`. + Default is empty. + + Optionally one can provide full path for any of them. By default it is + relative to the "/tls/", which is mount path of the tls secret. diff --git a/docs/user.md b/docs/user.md index 1be50a01a..bb12fd2e1 100644 --- a/docs/user.md +++ b/docs/user.md @@ -584,7 +584,8 @@ don't know the value, use `103` which is the GID from the default spilo image OpenShift allocates the users and groups dynamically (based on scc), and their range is different in every namespace. Due to this dynamic behaviour, it's not trivial to know at deploy time the uid/gid of the user in the cluster. -This way, in OpenShift, you may want to skip the spilo_fsgroup setting. +Therefore, instead of using a global `spilo_fsgroup` setting, use the `spiloFSGroup` field +per Postgres cluster.``` Upload the cert as a kubernetes secret: ```sh diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 3a87254cf..6de6db11e 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -125,5 +125,8 @@ spec: certificateFile: "tls.crt" privateKeyFile: "tls.key" caFile: "" # optionally configure Postgres with a CA certificate +# file names can be also defined with absolute path, and will no longer be relative +# to the "/tls/" path where the secret is being mounted by default. # When TLS is enabled, also set spiloFSGroup parameter above to the relevant value. # if unknown, set it to 103 which is the usual value in the default spilo images. +# In Openshift, there is no need to set spiloFSGroup/spilo_fsgroup. From a1f2bd05b978b4dac384eb14a06a39f590cf5f57 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Thu, 9 Apr 2020 09:21:45 +0200 Subject: [PATCH 079/104] Prevent superuser from being a connection pool user (#906) * Protected and system users can't be a connection pool user It's not supported, neither it's a best practice. Also fix potential null pointer access. For protected users it makes sense by intent of protecting this users (e.g. from being overriden or used as something else than supposed). For system users the reason is the same as for superuser, it's about replicastion user and it's under patroni control. This is implemented on both levels, operator config and postgresql manifest. For the latter we just use default name in this case, assuming that operator config is always correct. For the former, since it's a serious misconfiguration, operator panics. --- pkg/cluster/cluster.go | 17 +++++++++++++--- pkg/cluster/cluster_test.go | 34 +++++++++++++++++++++++++++++++ pkg/cluster/resources.go | 5 +++++ pkg/controller/operator_config.go | 5 +++++ pkg/util/config/config.go | 6 ++++++ 5 files changed, 64 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index cde2dc260..51dd368fb 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -877,9 +877,20 @@ func (c *Cluster) initSystemUsers() { c.Spec.ConnectionPooler = &acidv1.ConnectionPooler{} } - username := util.Coalesce( - c.Spec.ConnectionPooler.User, - c.OpConfig.ConnectionPooler.User) + // Using superuser as pooler user is not a good idea. First of all it's + // not going to be synced correctly with the current implementation, + // and second it's a bad practice. + username := c.OpConfig.ConnectionPooler.User + + isSuperUser := c.Spec.ConnectionPooler.User == c.OpConfig.SuperUsername + isProtectedUser := c.shouldAvoidProtectedOrSystemRole( + c.Spec.ConnectionPooler.User, "connection pool role") + + if !isSuperUser && !isProtectedUser { + username = util.Coalesce( + c.Spec.ConnectionPooler.User, + c.OpConfig.ConnectionPooler.User) + } // connection pooler application should be able to login with this role connectionPoolerUser := spec.PgUser{ diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index af3092ad5..432f53132 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -721,4 +721,38 @@ func TestInitSystemUsers(t *testing.T) { if _, exist := cl.systemUsers[constants.ConnectionPoolerUserKeyName]; !exist { t.Errorf("%s, connection pooler user is not present", testName) } + + // superuser is not allowed as connection pool user + cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{ + User: "postgres", + } + cl.OpConfig.SuperUsername = "postgres" + cl.OpConfig.ConnectionPooler.User = "pooler" + + cl.initSystemUsers() + if _, exist := cl.pgUsers["pooler"]; !exist { + t.Errorf("%s, Superuser is not allowed to be a connection pool user", testName) + } + + // neither protected users are + delete(cl.pgUsers, "pooler") + cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{ + User: "admin", + } + cl.OpConfig.ProtectedRoles = []string{"admin"} + + cl.initSystemUsers() + if _, exist := cl.pgUsers["pooler"]; !exist { + t.Errorf("%s, Protected user are not allowed to be a connection pool user", testName) + } + + delete(cl.pgUsers, "pooler") + cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{ + User: "standby", + } + + cl.initSystemUsers() + if _, exist := cl.pgUsers["pooler"]; !exist { + t.Errorf("%s, System users are not allowed to be a connection pool user", testName) + } } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 4c341aefe..b38458af8 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -105,7 +105,12 @@ func (c *Cluster) createConnectionPooler(lookup InstallFunction) (*ConnectionPoo var msg string c.setProcessName("creating connection pooler") + if c.ConnectionPooler == nil { + c.ConnectionPooler = &ConnectionPoolerObjects{} + } + schema := c.Spec.ConnectionPooler.Schema + if schema == "" { schema = c.OpConfig.ConnectionPooler.Schema } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index a9c36f995..07be90f22 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -169,6 +169,11 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur fromCRD.ConnectionPooler.User, constants.ConnectionPoolerUserName) + if result.ConnectionPooler.User == result.SuperUsername { + msg := "Connection pool user is not allowed to be the same as super user, username: %s" + panic(fmt.Errorf(msg, result.ConnectionPooler.User)) + } + result.ConnectionPooler.Image = util.Coalesce( fromCRD.ConnectionPooler.Image, "registry.opensource.zalan.do/acid/pgbouncer") diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 0a7bac835..84a62c0fd 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -218,5 +218,11 @@ func validate(cfg *Config) (err error) { msg := "number of connection pooler instances should be higher than %d" err = fmt.Errorf(msg, constants.ConnectionPoolerMinInstances) } + + if cfg.ConnectionPooler.User == cfg.SuperUsername { + msg := "Connection pool user is not allowed to be the same as super user, username: %s" + err = fmt.Errorf(msg, cfg.ConnectionPooler.User) + } + return } From ea3eef45d95c62ff1f4a4c825744835031a7b54f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thierry=20Sall=C3=A9?= Date: Wed, 15 Apr 2020 09:13:35 +0200 Subject: [PATCH 080/104] Additional volumes capability (#736) * Allow additional Volumes to be mounted * added TargetContainers option to determine if additional volume need to be mounter or not * fixed dependencies * updated manifest additional volume example * More validation Check that there are no volume mount path clashes or "all" vs ["a", "b"] mixtures. Also change the default behaviour to mount to "postgres" container. * More documentation / example about additional volumes * Revert go.sum and go.mod from origin/master * Declare addictionalVolume specs in CRDs * fixed k8sres after rebase * resolv conflict Co-authored-by: Dmitrii Dolgov <9erthalion6@gmail.com> Co-authored-by: Thierry --- .../postgres-operator/crds/postgresqls.yaml | 22 +++ docs/reference/cluster_manifest.md | 12 ++ manifests/complete-postgres-manifest.yaml | 23 +++ manifests/postgresql.crd.yaml | 22 +++ pkg/apis/acid.zalan.do/v1/crds.go | 31 ++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 9 + pkg/cluster/k8sres.go | 82 ++++++++- pkg/cluster/k8sres_test.go | 169 ++++++++++++++++++ 8 files changed, 364 insertions(+), 6 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index f64080ed5..3c666b9ab 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -74,6 +74,28 @@ spec: - teamId - postgresql properties: + additionalVolumes: + type: array + items: + type: object + required: + - name + - mountPath + - volumeSource + properties: + name: + type: string + mountPath: + type: string + targetContainers: + type: array + nullable: true + items: + type: string + volumeSource: + type: object + subPath: + type: string allowedSourceRanges: type: array nullable: true diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 0cbcdc08e..361e32780 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -154,6 +154,18 @@ These parameters are grouped directly under the `spec` key in the manifest. [the reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule) into account. Optional. Default is: "30 00 \* \* \*" +* **additionalVolumes** + List of additional volumes to mount in each container of the statefulset pod. + Each item must contain a `name`, `mountPath`, and `volumeSource` which is a + [kubernetes volumeSource](https://godoc.org/k8s.io/api/core/v1#VolumeSource). + It allows you to mount existing PersistentVolumeClaims, ConfigMaps and Secrets inside the StatefulSet. + Also an `emptyDir` volume can be shared between initContainer and statefulSet. + Additionaly, you can provide a `SubPath` for volume mount (a file in a configMap source volume, for example). + You can also specify in which container the additional Volumes will be mounted with the `targetContainers` array option. + If `targetContainers` is empty, additional volumes will be mounted only in the `postgres` container. + If you set the `all` special item, it will be mounted in all containers (postgres + sidecars). + Else you can set the list of target containers in which the additional volumes will be mounted (eg : postgres, telegraf) + ## Postgres parameters Those parameters are grouped under the `postgresql` top-level key, which is diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 6de6db11e..23b0b6096 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -12,6 +12,29 @@ spec: volume: size: 1Gi # storageClass: my-sc + additionalVolumes: + - name: data + mountPath: /home/postgres/pgdata/partitions + targetContainers: + - postgres + volumeSource: + PersistentVolumeClaim: + claimName: pvc-postgresql-data-partitions + readyOnly: false + - name: conf + mountPath: /etc/telegraf + subPath: telegraf.conf + targetContainers: + - telegraf-sidecar + volumeSource: + configMap: + name: my-config-map + - name: empty + mountPath: /opt/empty + targetContainers: + - all + volumeSource: + emptyDir: {} numberOfInstances: 2 users: # Application/Robot users zalando: diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index f8631f0b7..c9d60d60a 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -38,6 +38,28 @@ spec: - teamId - postgresql properties: + additionalVolumes: + type: array + items: + type: object + required: + - name + - mountPath + - volumeSource + properties: + name: + type: string + mountPath: + type: string + targetContainers: + type: array + nullable: true + items: + type: string + volumeSource: + type: object + subPath: + type: string allowedSourceRanges: type: array nullable: true diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 60dd66ba8..d693d0e15 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -682,6 +682,37 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "additionalVolumes": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + Required: []string{"name", "mountPath", "volumeSource"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "name": { + Type: "string", + }, + "mountPath": { + Type: "string", + }, + "targetContainers": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "volumeSource": { + Type: "object", + }, + "subPath": { + Type: "string", + }, + }, + }, + }, + }, }, }, "status": { diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index b1b6a36a6..04b70cba8 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -67,6 +67,7 @@ type PostgresSpec struct { PodAnnotations map[string]string `json:"podAnnotations"` ServiceAnnotations map[string]string `json:"serviceAnnotations"` TLS *TLSDescription `json:"tls"` + AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` // deprecated json tags InitContainersOld []v1.Container `json:"init_containers,omitempty"` @@ -98,6 +99,14 @@ type Volume struct { SubPath string `json:"subPath,omitempty"` } +type AdditionalVolume struct { + Name string `json:"name"` + MountPath string `json:"mountPath"` + SubPath string `json:"subPath"` + TargetContainers []string `json:"targetContainers"` + VolumeSource v1.VolumeSource `json:"volume"` +} + // PostgresqlParam describes PostgreSQL version and pairs of configuration parameter name - values. type PostgresqlParam struct { PgVersion string `json:"version"` diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 8de61bdea..e6f53af9d 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -500,7 +500,7 @@ func mountShmVolumeNeeded(opConfig config.Config, spec *acidv1.PostgresSpec) *bo return opConfig.ShmVolume } -func generatePodTemplate( +func (c *Cluster) generatePodTemplate( namespace string, labels labels.Set, annotations map[string]string, @@ -520,6 +520,7 @@ func generatePodTemplate( additionalSecretMount string, additionalSecretMountPath string, volumes []v1.Volume, + additionalVolumes []acidv1.AdditionalVolume, ) (*v1.PodTemplateSpec, error) { terminateGracePeriodSeconds := terminateGracePeriod @@ -559,6 +560,10 @@ func generatePodTemplate( addSecretVolume(&podSpec, additionalSecretMount, additionalSecretMountPath) } + if additionalVolumes != nil { + c.addAdditionalVolumes(&podSpec, additionalVolumes) + } + template := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, @@ -1084,7 +1089,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef annotations := c.generatePodAnnotations(spec) // generate pod template for the statefulset, based on the spilo container and sidecars - podTemplate, err = generatePodTemplate( + podTemplate, err = c.generatePodTemplate( c.Namespace, c.labelsSet(true), annotations, @@ -1104,7 +1109,8 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.OpConfig.AdditionalSecretMount, c.OpConfig.AdditionalSecretMountPath, volumes, - ) + spec.AdditionalVolumes) + if err != nil { return nil, fmt.Errorf("could not generate pod template: %v", err) } @@ -1298,6 +1304,69 @@ func addSecretVolume(podSpec *v1.PodSpec, additionalSecretMount string, addition podSpec.Volumes = volumes } +func (c *Cluster) addAdditionalVolumes(podSpec *v1.PodSpec, + additionalVolumes []acidv1.AdditionalVolume) { + + volumes := podSpec.Volumes + mountPaths := map[string]acidv1.AdditionalVolume{} + for i, v := range additionalVolumes { + if previousVolume, exist := mountPaths[v.MountPath]; exist { + msg := "Volume %+v cannot be mounted to the same path as %+v" + c.logger.Warningf(msg, v, previousVolume) + continue + } + + if v.MountPath == constants.PostgresDataMount { + msg := "Cannot mount volume on postgresql data directory, %+v" + c.logger.Warningf(msg, v) + continue + } + + if v.TargetContainers == nil { + spiloContainer := podSpec.Containers[0] + additionalVolumes[i].TargetContainers = []string{spiloContainer.Name} + } + + for _, target := range v.TargetContainers { + if target == "all" && len(v.TargetContainers) != 1 { + msg := `Target containers could be either "all" or a list + of containers, mixing those is not allowed, %+v` + c.logger.Warningf(msg, v) + continue + } + } + + volumes = append(volumes, + v1.Volume{ + Name: v.Name, + VolumeSource: v.VolumeSource, + }, + ) + + mountPaths[v.MountPath] = v + } + + c.logger.Infof("Mount additional volumes: %+v", additionalVolumes) + + for i := range podSpec.Containers { + mounts := podSpec.Containers[i].VolumeMounts + for _, v := range additionalVolumes { + for _, target := range v.TargetContainers { + if podSpec.Containers[i].Name == target || target == "all" { + mounts = append(mounts, v1.VolumeMount{ + Name: v.Name, + MountPath: v.MountPath, + SubPath: v.SubPath, + }) + } + } + } + podSpec.Containers[i].VolumeMounts = mounts + } + + podSpec.Volumes = volumes +} + func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string) (*v1.PersistentVolumeClaim, error) { var storageClassName *string @@ -1702,7 +1771,7 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { annotations := c.generatePodAnnotations(&c.Spec) // re-use the method that generates DB pod templates - if podTemplate, err = generatePodTemplate( + if podTemplate, err = c.generatePodTemplate( c.Namespace, labels, annotations, @@ -1721,8 +1790,9 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { "", c.OpConfig.AdditionalSecretMount, c.OpConfig.AdditionalSecretMountPath, - nil); err != nil { - return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) + nil, + []acidv1.AdditionalVolume{}); err != nil { + return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) } // overwrite specific params of logical backups pods diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 0930279d2..5b55e988c 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1021,3 +1021,172 @@ func TestTLS(t *testing.T) { assert.Contains(t, s.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "SSL_PRIVATE_KEY_FILE", Value: "/tls/tls.key"}) assert.Contains(t, s.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "SSL_CA_FILE", Value: "/tls/ca.crt"}) } + +func TestAdditionalVolume(t *testing.T) { + testName := "TestAdditionalVolume" + tests := []struct { + subTest string + podSpec *v1.PodSpec + volumePos int + }{ + { + subTest: "empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{}, + }, + }, + }, + volumePos: 0, + }, + { + subTest: "non empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{{}}, + Containers: []v1.Container{ + { + Name: "postgres", + VolumeMounts: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + }, + }, + }, + }, + }, + volumePos: 1, + }, + { + subTest: "non empty PodSpec with sidecar", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{{}}, + Containers: []v1.Container{ + { + Name: "postgres", + VolumeMounts: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + }, + }, + }, + { + Name: "sidecar", + VolumeMounts: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + }, + }, + }, + }, + }, + volumePos: 1, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + for _, tt := range tests { + // Test with additional volume mounted in all containers + additionalVolumeMount := []acidv1.AdditionalVolume{ + { + Name: "test", + MountPath: "/test", + TargetContainers: []string{"all"}, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + } + + numMounts := len(tt.podSpec.Containers[0].VolumeMounts) + + cluster.addAdditionalVolumes(tt.podSpec, additionalVolumeMount) + volumeName := tt.podSpec.Volumes[tt.volumePos].Name + + if volumeName != additionalVolumeMount[0].Name { + t.Errorf("%s %s: Expected volume %v was not created, have %s instead", + testName, tt.subTest, additionalVolumeMount, volumeName) + } + + for i := range tt.podSpec.Containers { + volumeMountName := tt.podSpec.Containers[i].VolumeMounts[tt.volumePos].Name + + if volumeMountName != additionalVolumeMount[0].Name { + t.Errorf("%s %s: Expected mount %v was not created, have %s instead", + testName, tt.subTest, additionalVolumeMount, volumeMountName) + } + + } + + numMountsCheck := len(tt.podSpec.Containers[0].VolumeMounts) + + if numMountsCheck != numMounts+1 { + t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v", + numMountsCheck, numMounts+1) + } + } + + for _, tt := range tests { + // Test with additional volume mounted only in first container + additionalVolumeMount := []acidv1.AdditionalVolume{ + { + Name: "test", + MountPath: "/test", + TargetContainers: []string{"postgres"}, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + } + + numMounts := len(tt.podSpec.Containers[0].VolumeMounts) + + cluster.addAdditionalVolumes(tt.podSpec, additionalVolumeMount) + volumeName := tt.podSpec.Volumes[tt.volumePos].Name + + if volumeName != additionalVolumeMount[0].Name { + t.Errorf("%s %s: Expected volume %v was not created, have %s instead", + testName, tt.subTest, additionalVolumeMount, volumeName) + } + + for _, container := range tt.podSpec.Containers { + if container.Name == "postgres" { + volumeMountName := container.VolumeMounts[tt.volumePos].Name + + if volumeMountName != additionalVolumeMount[0].Name { + t.Errorf("%s %s: Expected mount %v was not created, have %s instead", + testName, tt.subTest, additionalVolumeMount, volumeMountName) + } + + numMountsCheck := len(container.VolumeMounts) + if numMountsCheck != numMounts+1 { + t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v", + numMountsCheck, numMounts+1) + } + } else { + numMountsCheck := len(container.VolumeMounts) + if numMountsCheck == numMounts+1 { + t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v", + numMountsCheck, numMounts) + } + } + } + } +} From 7e8f6687ebbb4a168a98dbb7d69311bd0d393f30 Mon Sep 17 00:00:00 2001 From: ReSearchITEng Date: Wed, 15 Apr 2020 16:24:55 +0300 Subject: [PATCH 081/104] make tls pr798 use additionalVolumes capability from pr736 (#920) * make tls pr798 use additionalVolumes capability from pr736 * move the volume* sections lower * update helm chart crds and docs * fix user.md typos --- .gitignore | 2 + .../postgres-operator/crds/postgresqls.yaml | 15 ++++++ docs/reference/cluster_manifest.md | 7 +++ docs/user.md | 31 ++++++++++-- manifests/complete-postgres-manifest.yaml | 40 ++++++++-------- manifests/postgresql.crd.yaml | 2 + pkg/apis/acid.zalan.do/v1/crds.go | 3 ++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 1 + .../acid.zalan.do/v1/zz_generated.deepcopy.go | 29 ++++++++++++ pkg/cluster/k8sres.go | 47 ++++++++++++------- pkg/cluster/k8sres_test.go | 18 +++++-- 11 files changed, 150 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index 991fe754f..b407c62f1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ _obj _test _manifests +_tmp +github.com # Architecture specific extensions/prefixes *.[568vq] diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 3c666b9ab..78850ee3b 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -364,6 +364,21 @@ spec: type: string teamId: type: string + tls: + type: object + required: + - secretName + properties: + secretName: + type: string + certificateFile: + type: string + privateKeyFile: + type: string + caFile: + type: string + caSecretName: + type: string tolerations: type: array items: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 361e32780..c87728812 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -435,5 +435,12 @@ Those parameters are grouped under the `tls` top-level key. client connects with `sslmode=verify-ca` or `sslmode=verify-full`. Default is empty. +* **caSecretName** + By setting the `caSecretName` value, the ca certificate file defined by the + `caFile` will be fetched from this secret instead of `secretName` above. + This secret has to hold a file with that name in its root. + Optionally one can provide full path for any of them. By default it is relative to the "/tls/", which is mount path of the tls secret. + If `caSecretName` is defined, the ca.crt path is relative to "/tlsca/", + otherwise to the same "/tls/". diff --git a/docs/user.md b/docs/user.md index bb12fd2e1..2c1c4fd1f 100644 --- a/docs/user.md +++ b/docs/user.md @@ -585,7 +585,7 @@ OpenShift allocates the users and groups dynamically (based on scc), and their range is different in every namespace. Due to this dynamic behaviour, it's not trivial to know at deploy time the uid/gid of the user in the cluster. Therefore, instead of using a global `spilo_fsgroup` setting, use the `spiloFSGroup` field -per Postgres cluster.``` +per Postgres cluster. Upload the cert as a kubernetes secret: ```sh @@ -594,7 +594,7 @@ kubectl create secret tls pg-tls \ --cert pg-tls.crt ``` -Or with a CA: +When doing client auth, CA can come optionally from the same secret: ```sh kubectl create secret generic pg-tls \ --from-file=tls.crt=server.crt \ @@ -602,9 +602,6 @@ kubectl create secret generic pg-tls \ --from-file=ca.crt=ca.crt ``` -Alternatively it is also possible to use -[cert-manager](https://cert-manager.io/docs/) to generate these secrets. - Then configure the postgres resource with the TLS secret: ```yaml @@ -619,5 +616,29 @@ spec: caFile: "ca.crt" # add this if the secret is configured with a CA ``` +Optionally, the CA can be provided by a different secret: +```sh +kubectl create secret generic pg-tls-ca \ + --from-file=ca.crt=ca.crt +``` + +Then configure the postgres resource with the TLS secret: + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql + +metadata: + name: acid-test-cluster +spec: + tls: + secretName: "pg-tls" # this should hold tls.key and tls.crt + caSecretName: "pg-tls-ca" # this should hold ca.crt + caFile: "ca.crt" # add this if the secret is configured with a CA +``` + +Alternatively, it is also possible to use +[cert-manager](https://cert-manager.io/docs/) to generate these secrets. + Certificate rotation is handled in the spilo image which checks every 5 minutes if the certificates have changed and reloads postgres accordingly. diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 23b0b6096..b469a7564 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -9,6 +9,24 @@ metadata: spec: dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 teamId: "acid" + numberOfInstances: 2 + users: # Application/Robot users + zalando: + - superuser + - createdb + enableMasterLoadBalancer: false + enableReplicaLoadBalancer: false +# enableConnectionPooler: true # not needed when connectionPooler section is present (see below) + allowedSourceRanges: # load balancers' source ranges for both master and replica services + - 127.0.0.1/32 + databases: + foo: zalando + postgresql: + version: "12" + parameters: # Expert section + shared_buffers: "32MB" + max_connections: "10" + log_statement: "all" volume: size: 1Gi # storageClass: my-sc @@ -35,24 +53,6 @@ spec: - all volumeSource: emptyDir: {} - numberOfInstances: 2 - users: # Application/Robot users - zalando: - - superuser - - createdb - enableMasterLoadBalancer: false - enableReplicaLoadBalancer: false -# enableConnectionPooler: true # not needed when connectionPooler section is present (see below) - allowedSourceRanges: # load balancers' source ranges for both master and replica services - - 127.0.0.1/32 - databases: - foo: zalando - postgresql: - version: "12" - parameters: # Expert section - shared_buffers: "32MB" - max_connections: "10" - log_statement: "all" enableShmVolume: true # spiloFSGroup: 103 @@ -148,8 +148,10 @@ spec: certificateFile: "tls.crt" privateKeyFile: "tls.key" caFile: "" # optionally configure Postgres with a CA certificate + caSecretName: "" # optionally the ca.crt can come from this secret instead. # file names can be also defined with absolute path, and will no longer be relative -# to the "/tls/" path where the secret is being mounted by default. +# to the "/tls/" path where the secret is being mounted by default, and "/tlsca/" +# where the caSecret is mounted by default. # When TLS is enabled, also set spiloFSGroup parameter above to the relevant value. # if unknown, set it to 103 which is the usual value in the default spilo images. # In Openshift, there is no need to set spiloFSGroup/spilo_fsgroup. diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index c9d60d60a..1ee6a1ae5 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -341,6 +341,8 @@ spec: type: string caFile: type: string + caSecretName: + type: string tolerations: type: array items: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index d693d0e15..3f4314240 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -513,6 +513,9 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "caFile": { Type: "string", }, + "caSecretName": { + Type: "string", + }, }, }, "tolerations": { diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 04b70cba8..961051c8d 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -148,6 +148,7 @@ type TLSDescription struct { CertificateFile string `json:"certificateFile,omitempty"` PrivateKeyFile string `json:"privateKeyFile,omitempty"` CAFile string `json:"caFile,omitempty"` + CASecretName string `json:"caSecretName,omitempty"` } // CloneDescription describes which cluster the new should clone and up to which point in time diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 92c8af34b..e6b387ec4 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -47,6 +47,28 @@ func (in *AWSGCPConfiguration) DeepCopy() *AWSGCPConfiguration { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AdditionalVolume) DeepCopyInto(out *AdditionalVolume) { + *out = *in + if in.TargetContainers != nil { + in, out := &in.TargetContainers, &out.TargetContainers + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.VolumeSource.DeepCopyInto(&out.VolumeSource) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalVolume. +func (in *AdditionalVolume) DeepCopy() *AdditionalVolume { + if in == nil { + return nil + } + out := new(AdditionalVolume) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloneDescription) DeepCopyInto(out *CloneDescription) { *out = *in @@ -591,6 +613,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = new(TLSDescription) **out = **in } + if in.AdditionalVolumes != nil { + in, out := &in.AdditionalVolumes, &out.AdditionalVolumes + *out = make([]AdditionalVolume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.InitContainersOld != nil { in, out := &in.InitContainersOld, &out.InitContainersOld *out = make([]corev1.Container, len(*in)) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index e6f53af9d..9fb33eab2 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -519,7 +519,6 @@ func (c *Cluster) generatePodTemplate( podAntiAffinityTopologyKey string, additionalSecretMount string, additionalSecretMountPath string, - volumes []v1.Volume, additionalVolumes []acidv1.AdditionalVolume, ) (*v1.PodTemplateSpec, error) { @@ -539,7 +538,6 @@ func (c *Cluster) generatePodTemplate( InitContainers: initContainers, Tolerations: *tolerationsSpec, SecurityContext: &securityContext, - Volumes: volumes, } if shmVolume != nil && *shmVolume { @@ -854,7 +852,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef sidecarContainers []v1.Container podTemplate *v1.PodTemplateSpec volumeClaimTemplate *v1.PersistentVolumeClaim - volumes []v1.Volume + additionalVolumes = spec.AdditionalVolumes ) // Improve me. Please. @@ -1007,8 +1005,10 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // this is combined with the FSGroup in the section above // to give read access to the postgres user defaultMode := int32(0640) - volumes = append(volumes, v1.Volume{ - Name: "tls-secret", + mountPath := "/tls" + additionalVolumes = append(additionalVolumes, acidv1.AdditionalVolume{ + Name: spec.TLS.SecretName, + MountPath: mountPath, VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ SecretName: spec.TLS.SecretName, @@ -1017,13 +1017,6 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef }, }) - mountPath := "/tls" - volumeMounts = append(volumeMounts, v1.VolumeMount{ - MountPath: mountPath, - Name: "tls-secret", - ReadOnly: true, - }) - // use the same filenames as Secret resources by default certFile := ensurePath(spec.TLS.CertificateFile, mountPath, "tls.crt") privateKeyFile := ensurePath(spec.TLS.PrivateKeyFile, mountPath, "tls.key") @@ -1034,11 +1027,31 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef ) if spec.TLS.CAFile != "" { - caFile := ensurePath(spec.TLS.CAFile, mountPath, "") + // support scenario when the ca.crt resides in a different secret, diff path + mountPathCA := mountPath + if spec.TLS.CASecretName != "" { + mountPathCA = mountPath + "ca" + } + + caFile := ensurePath(spec.TLS.CAFile, mountPathCA, "") spiloEnvVars = append( spiloEnvVars, v1.EnvVar{Name: "SSL_CA_FILE", Value: caFile}, ) + + // the ca file from CASecretName secret takes priority + if spec.TLS.CASecretName != "" { + additionalVolumes = append(additionalVolumes, acidv1.AdditionalVolume{ + Name: spec.TLS.CASecretName, + MountPath: mountPathCA, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: spec.TLS.CASecretName, + DefaultMode: &defaultMode, + }, + }, + }) + } } } @@ -1108,8 +1121,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.OpConfig.PodAntiAffinityTopologyKey, c.OpConfig.AdditionalSecretMount, c.OpConfig.AdditionalSecretMountPath, - volumes, - spec.AdditionalVolumes) + additionalVolumes) if err != nil { return nil, fmt.Errorf("could not generate pod template: %v", err) @@ -1614,11 +1626,11 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) c.logger.Info(msg, description.S3WalPath) envs := []v1.EnvVar{ - v1.EnvVar{ + { Name: "CLONE_WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket, }, - v1.EnvVar{ + { Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(description.UID), }, @@ -1790,7 +1802,6 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { "", c.OpConfig.AdditionalSecretMount, c.OpConfig.AdditionalSecretMountPath, - nil, []acidv1.AdditionalVolume{}); err != nil { return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 5b55e988c..6e4587627 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -961,6 +961,7 @@ func TestTLS(t *testing.T) { var spec acidv1.PostgresSpec var cluster *Cluster var spiloFSGroup = int64(103) + var additionalVolumes = spec.AdditionalVolumes makeSpec := func(tls acidv1.TLSDescription) acidv1.PostgresSpec { return acidv1.PostgresSpec{ @@ -1000,8 +1001,20 @@ func TestTLS(t *testing.T) { assert.Equal(t, &fsGroup, s.Spec.Template.Spec.SecurityContext.FSGroup, "has a default FSGroup assigned") defaultMode := int32(0640) + mountPath := "/tls" + additionalVolumes = append(additionalVolumes, acidv1.AdditionalVolume{ + Name: spec.TLS.SecretName, + MountPath: mountPath, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: spec.TLS.SecretName, + DefaultMode: &defaultMode, + }, + }, + }) + volume := v1.Volume{ - Name: "tls-secret", + Name: "my-secret", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ SecretName: "my-secret", @@ -1013,8 +1026,7 @@ func TestTLS(t *testing.T) { assert.Contains(t, s.Spec.Template.Spec.Containers[0].VolumeMounts, v1.VolumeMount{ MountPath: "/tls", - Name: "tls-secret", - ReadOnly: true, + Name: "my-secret", }, "the volume gets mounted in /tls") assert.Contains(t, s.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "SSL_CERTIFICATE_FILE", Value: "/tls/tls.crt"}) From 6a689cdc1c15bef7498cd29c0ad3e79a5f840996 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Thu, 16 Apr 2020 15:14:31 +0200 Subject: [PATCH 082/104] Prevent empty syncs (#922) There is a possibility to pass nil as one of the specs and an empty spec into syncConnectionPooler. In this case it will perfom a syncronization because nil != empty struct. Avoid such cases and make it testable by returning list of syncronization reasons on top together with the final error. --- pkg/cluster/cluster.go | 3 +- pkg/cluster/sync.go | 64 +++++++++++++++++-------- pkg/cluster/sync_test.go | 101 +++++++++++++++++++++++++++++---------- pkg/cluster/types.go | 5 ++ 4 files changed, 127 insertions(+), 46 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 51dd368fb..83c1bc157 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -741,7 +741,8 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } // sync connection pooler - if err := c.syncConnectionPooler(oldSpec, newSpec, c.installLookupFunction); err != nil { + if _, err := c.syncConnectionPooler(oldSpec, newSpec, + c.installLookupFunction); err != nil { return fmt.Errorf("could not sync connection pooler: %v", err) } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index a423c2f0e..361673891 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -111,7 +111,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } // sync connection pooler - if err = c.syncConnectionPooler(&oldSpec, newSpec, c.installLookupFunction); err != nil { + if _, err = c.syncConnectionPooler(&oldSpec, newSpec, c.installLookupFunction); err != nil { return fmt.Errorf("could not sync connection pooler: %v", err) } @@ -620,7 +620,13 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } -func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, lookup InstallFunction) error { +func (c *Cluster) syncConnectionPooler(oldSpec, + newSpec *acidv1.Postgresql, + lookup InstallFunction) (SyncReason, error) { + + var reason SyncReason + var err error + if c.ConnectionPooler == nil { c.ConnectionPooler = &ConnectionPoolerObjects{} } @@ -657,20 +663,20 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, look specUser, c.OpConfig.ConnectionPooler.User) - if err := lookup(schema, user); err != nil { - return err + if err = lookup(schema, user); err != nil { + return NoSync, err } } - if err := c.syncConnectionPoolerWorker(oldSpec, newSpec); err != nil { + if reason, err = c.syncConnectionPoolerWorker(oldSpec, newSpec); err != nil { c.logger.Errorf("could not sync connection pooler: %v", err) - return err + return reason, err } } if oldNeedConnectionPooler && !newNeedConnectionPooler { // delete and cleanup resources - if err := c.deleteConnectionPooler(); err != nil { + if err = c.deleteConnectionPooler(); err != nil { c.logger.Warningf("could not remove connection pooler: %v", err) } } @@ -681,20 +687,22 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, look (c.ConnectionPooler.Deployment != nil || c.ConnectionPooler.Service != nil) { - if err := c.deleteConnectionPooler(); err != nil { + if err = c.deleteConnectionPooler(); err != nil { c.logger.Warningf("could not remove connection pooler: %v", err) } } } - return nil + return reason, nil } // Synchronize connection pooler resources. Effectively we're interested only in // synchronizing the corresponding deployment, but in case of deployment or // service is missing, create it. After checking, also remember an object for // the future references. -func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql) error { +func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql) ( + SyncReason, error) { + deployment, err := c.KubeClient. Deployments(c.Namespace). Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{}) @@ -706,7 +714,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql deploymentSpec, err := c.generateConnectionPoolerDeployment(&newSpec.Spec) if err != nil { msg = "could not generate deployment for connection pooler: %v" - return fmt.Errorf(msg, err) + return NoSync, fmt.Errorf(msg, err) } deployment, err := c.KubeClient. @@ -714,18 +722,35 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql Create(context.TODO(), deploymentSpec, metav1.CreateOptions{}) if err != nil { - return err + return NoSync, err } c.ConnectionPooler.Deployment = deployment } else if err != nil { - return fmt.Errorf("could not get connection pooler deployment to sync: %v", err) + msg := "could not get connection pooler deployment to sync: %v" + return NoSync, fmt.Errorf(msg, err) } else { c.ConnectionPooler.Deployment = deployment // actual synchronization oldConnectionPooler := oldSpec.Spec.ConnectionPooler newConnectionPooler := newSpec.Spec.ConnectionPooler + + // sync implementation below assumes that both old and new specs are + // not nil, but it can happen. To avoid any confusion like updating a + // deployment because the specification changed from nil to an empty + // struct (that was initialized somewhere before) replace any nil with + // an empty spec. + if oldConnectionPooler == nil { + oldConnectionPooler = &acidv1.ConnectionPooler{} + } + + if newConnectionPooler == nil { + newConnectionPooler = &acidv1.ConnectionPooler{} + } + + c.logger.Infof("Old: %+v, New %+v", oldConnectionPooler, newConnectionPooler) + specSync, specReason := c.needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler) defaultsSync, defaultsReason := c.needSyncConnectionPoolerDefaults(newConnectionPooler, deployment) reason := append(specReason, defaultsReason...) @@ -736,7 +761,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql newDeploymentSpec, err := c.generateConnectionPoolerDeployment(&newSpec.Spec) if err != nil { msg := "could not generate deployment for connection pooler: %v" - return fmt.Errorf(msg, err) + return reason, fmt.Errorf(msg, err) } oldDeploymentSpec := c.ConnectionPooler.Deployment @@ -746,11 +771,11 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql newDeploymentSpec) if err != nil { - return err + return reason, err } c.ConnectionPooler.Deployment = deployment - return nil + return reason, nil } } @@ -768,16 +793,17 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) if err != nil { - return err + return NoSync, err } c.ConnectionPooler.Service = service } else if err != nil { - return fmt.Errorf("could not get connection pooler service to sync: %v", err) + msg := "could not get connection pooler service to sync: %v" + return NoSync, fmt.Errorf(msg, err) } else { // Service updates are not supported and probably not that useful anyway c.ConnectionPooler.Service = service } - return nil + return NoSync, nil } diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index 45355cca3..50b5cfaa8 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -2,6 +2,7 @@ package cluster import ( "fmt" + "strings" "testing" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -17,7 +18,7 @@ func int32ToPointer(value int32) *int32 { return &value } -func deploymentUpdated(cluster *Cluster, err error) error { +func deploymentUpdated(cluster *Cluster, err error, reason SyncReason) error { if cluster.ConnectionPooler.Deployment.Spec.Replicas == nil || *cluster.ConnectionPooler.Deployment.Spec.Replicas != 2 { return fmt.Errorf("Wrong nubmer of instances") @@ -26,7 +27,7 @@ func deploymentUpdated(cluster *Cluster, err error) error { return nil } -func objectsAreSaved(cluster *Cluster, err error) error { +func objectsAreSaved(cluster *Cluster, err error, reason SyncReason) error { if cluster.ConnectionPooler == nil { return fmt.Errorf("Connection pooler resources are empty") } @@ -42,7 +43,7 @@ func objectsAreSaved(cluster *Cluster, err error) error { return nil } -func objectsAreDeleted(cluster *Cluster, err error) error { +func objectsAreDeleted(cluster *Cluster, err error, reason SyncReason) error { if cluster.ConnectionPooler != nil { return fmt.Errorf("Connection pooler was not deleted") } @@ -50,6 +51,16 @@ func objectsAreDeleted(cluster *Cluster, err error) error { return nil } +func noEmptySync(cluster *Cluster, err error, reason SyncReason) error { + for _, msg := range reason { + if strings.HasPrefix(msg, "update [] from '' to '") { + return fmt.Errorf("There is an empty reason, %s", msg) + } + } + + return nil +} + func TestConnectionPoolerSynchronization(t *testing.T) { testName := "Test connection pooler synchronization" var cluster = New( @@ -91,15 +102,15 @@ func TestConnectionPoolerSynchronization(t *testing.T) { clusterNewDefaultsMock := *cluster clusterNewDefaultsMock.KubeClient = k8sutil.NewMockKubernetesClient() - cluster.OpConfig.ConnectionPooler.Image = "pooler:2.0" - cluster.OpConfig.ConnectionPooler.NumberOfInstances = int32ToPointer(2) tests := []struct { - subTest string - oldSpec *acidv1.Postgresql - newSpec *acidv1.Postgresql - cluster *Cluster - check func(cluster *Cluster, err error) error + subTest string + oldSpec *acidv1.Postgresql + newSpec *acidv1.Postgresql + cluster *Cluster + defaultImage string + defaultInstances int32 + check func(cluster *Cluster, err error, reason SyncReason) error }{ { subTest: "create if doesn't exist", @@ -113,8 +124,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterMissingObjects, - check: objectsAreSaved, + cluster: &clusterMissingObjects, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreSaved, }, { subTest: "create if doesn't exist with a flag", @@ -126,8 +139,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { EnableConnectionPooler: boolToPointer(true), }, }, - cluster: &clusterMissingObjects, - check: objectsAreSaved, + cluster: &clusterMissingObjects, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreSaved, }, { subTest: "create from scratch", @@ -139,8 +154,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterMissingObjects, - check: objectsAreSaved, + cluster: &clusterMissingObjects, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreSaved, }, { subTest: "delete if not needed", @@ -152,8 +169,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{}, }, - cluster: &clusterMock, - check: objectsAreDeleted, + cluster: &clusterMock, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreDeleted, }, { subTest: "cleanup if still there", @@ -163,8 +182,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{}, }, - cluster: &clusterDirtyMock, - check: objectsAreDeleted, + cluster: &clusterDirtyMock, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: objectsAreDeleted, }, { subTest: "update deployment", @@ -182,8 +203,10 @@ func TestConnectionPoolerSynchronization(t *testing.T) { }, }, }, - cluster: &clusterMock, - check: deploymentUpdated, + cluster: &clusterMock, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: deploymentUpdated, }, { subTest: "update image from changed defaults", @@ -197,14 +220,40 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterNewDefaultsMock, - check: deploymentUpdated, + cluster: &clusterNewDefaultsMock, + defaultImage: "pooler:2.0", + defaultInstances: 2, + check: deploymentUpdated, + }, + { + subTest: "there is no sync from nil to an empty spec", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + ConnectionPooler: nil, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + }, + cluster: &clusterMock, + defaultImage: "pooler:1.0", + defaultInstances: 1, + check: noEmptySync, }, } for _, tt := range tests { - err := tt.cluster.syncConnectionPooler(tt.oldSpec, tt.newSpec, mockInstallLookupFunction) + tt.cluster.OpConfig.ConnectionPooler.Image = tt.defaultImage + tt.cluster.OpConfig.ConnectionPooler.NumberOfInstances = + int32ToPointer(tt.defaultInstances) - if err := tt.check(tt.cluster, err); err != nil { + reason, err := tt.cluster.syncConnectionPooler(tt.oldSpec, + tt.newSpec, mockInstallLookupFunction) + + if err := tt.check(tt.cluster, err, reason); err != nil { t.Errorf("%s [%s]: Could not synchronize, %+v", testName, tt.subTest, err) } diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 04d00cb58..199914ccc 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -73,3 +73,8 @@ type ClusterStatus struct { type TemplateParams map[string]interface{} type InstallFunction func(schema string, user string) error + +type SyncReason []string + +// no sync happened, empty value +var NoSync SyncReason = []string{} From 5014eebfb2261d778ac1ed8f8da103e474d85d61 Mon Sep 17 00:00:00 2001 From: ReSearchITEng Date: Thu, 16 Apr 2020 17:47:59 +0300 Subject: [PATCH 083/104] when kubernetes_use_configmaps -> skip further endpoints actions even delete (#921) * further compatibility with k8sUseConfigMaps - skip further endpoints related actions * Update pkg/cluster/cluster.go thanks! Co-Authored-By: Felix Kunde * Update pkg/cluster/cluster.go Co-Authored-By: Felix Kunde * Update pkg/cluster/cluster.go Co-authored-by: Felix Kunde --- pkg/cluster/cluster.go | 18 ++++++++++++++---- pkg/cluster/sync.go | 7 ++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 83c1bc157..74cf6e61d 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -784,8 +784,10 @@ func (c *Cluster) Delete() { for _, role := range []PostgresRole{Master, Replica} { - if err := c.deleteEndpoint(role); err != nil { - c.logger.Warningf("could not delete %s endpoint: %v", role, err) + if !c.patroniKubernetesUseConfigMaps() { + if err := c.deleteEndpoint(role); err != nil { + c.logger.Warningf("could not delete %s endpoint: %v", role, err) + } } if err := c.deleteService(role); err != nil { @@ -1161,11 +1163,19 @@ type clusterObjectDelete func(name string) error func (c *Cluster) deletePatroniClusterObjects() error { // TODO: figure out how to remove leftover patroni objects in other cases + var actionsList []simpleActionWithResult + if !c.patroniUsesKubernetes() { c.logger.Infof("not cleaning up Etcd Patroni objects on cluster delete") } - c.logger.Debugf("removing leftover Patroni objects (endpoints, services and configmaps)") - for _, deleter := range []simpleActionWithResult{c.deletePatroniClusterEndpoints, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps} { + + if !c.patroniKubernetesUseConfigMaps() { + actionsList = append(actionsList, c.deletePatroniClusterEndpoints) + } + actionsList = append(actionsList, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps) + + c.logger.Debugf("removing leftover Patroni objects (endpoints / services and configmaps)") + for _, deleter := range actionsList { if err := deleter(); err != nil { return err } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 361673891..43173102b 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -122,10 +122,11 @@ func (c *Cluster) syncServices() error { for _, role := range []PostgresRole{Master, Replica} { c.logger.Debugf("syncing %s service", role) - if err := c.syncEndpoint(role); err != nil { - return fmt.Errorf("could not sync %s endpoint: %v", role, err) + if !c.patroniKubernetesUseConfigMaps() { + if err := c.syncEndpoint(role); err != nil { + return fmt.Errorf("could not sync %s endpoint: %v", role, err) + } } - if err := c.syncService(role); err != nil { return fmt.Errorf("could not sync %s service: %v", role, err) } From 3c91bdeffadb5ec736a63548b5ffa08517c59de8 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 20 Apr 2020 15:14:11 +0200 Subject: [PATCH 084/104] Re-create pods only if all replicas are running (#903) * adds a Get call to Patroni interface to fetch state of a Patroni member * postpones re-creating pods if at least one replica is currently being created Co-authored-by: Sergey Dudoladov Co-authored-by: Felix Kunde --- .gitignore | 1 + e2e/tests/test_e2e.py | 9 +++++---- pkg/cluster/pod.go | 25 +++++++++++++++++++++++++ pkg/util/patroni/patroni.go | 37 ++++++++++++++++++++++++++++++++++++- 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index b407c62f1..0fdb50756 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ _testmain.go /vendor/ /build/ /docker/build/ +/github.com/ .idea scm-source.json diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 445067d61..f46c0577e 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -344,7 +344,6 @@ class EndToEndTestCase(unittest.TestCase): ''' k8s = self.k8s cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - labels = 'spilo-role=master,' + cluster_label readiness_label = 'lifecycle-status' readiness_value = 'ready' @@ -709,14 +708,16 @@ class K8s: def wait_for_logical_backup_job_creation(self): self.wait_for_logical_backup_job(expected_num_of_jobs=1) - def update_config(self, config_map_patch): - self.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) - + def delete_operator_pod(self): operator_pod = self.api.core_v1.list_namespaced_pod( 'default', label_selector="name=postgres-operator").items[0].metadata.name self.api.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf self.wait_for_operator_pod_start() + def update_config(self, config_map_patch): + self.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) + self.delete_operator_pod() + def create_with_kubectl(self, path): return subprocess.run( ["kubectl", "create", "-f", path], diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index 9991621cc..a734e4835 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -294,6 +294,27 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) { return pod, nil } +func (c *Cluster) isSafeToRecreatePods(pods *v1.PodList) bool { + + /* + Operator should not re-create pods if there is at least one replica being bootstrapped + because Patroni might use other replicas to take basebackup from (see Patroni's "clonefrom" tag). + + XXX operator cannot forbid replica re-init, so we might still fail if re-init is started + after this check succeeds but before a pod is re-created + */ + + for _, pod := range pods.Items { + state, err := c.patroni.GetPatroniMemberState(&pod) + if err != nil || state == "creating replica" { + c.logger.Warningf("cannot re-create replica %s: it is currently being initialized", pod.Name) + return false + } + + } + return true +} + func (c *Cluster) recreatePods() error { c.setProcessName("starting to recreate pods") ls := c.labelsSet(false) @@ -309,6 +330,10 @@ func (c *Cluster) recreatePods() error { } c.logger.Infof("there are %d pods in the cluster to recreate", len(pods.Items)) + if !c.isSafeToRecreatePods(pods) { + return fmt.Errorf("postpone pod recreation until next Sync: recreation is unsafe because pods are being initilalized") + } + var ( masterPod, newMasterPod, newPod *v1.Pod ) diff --git a/pkg/util/patroni/patroni.go b/pkg/util/patroni/patroni.go index bdd96f048..53065e599 100644 --- a/pkg/util/patroni/patroni.go +++ b/pkg/util/patroni/patroni.go @@ -3,6 +3,7 @@ package patroni import ( "bytes" "encoding/json" + "errors" "fmt" "io/ioutil" "net" @@ -11,7 +12,7 @@ import ( "time" "github.com/sirupsen/logrus" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) const ( @@ -25,6 +26,7 @@ const ( type Interface interface { Switchover(master *v1.Pod, candidate string) error SetPostgresParameters(server *v1.Pod, options map[string]string) error + GetPatroniMemberState(pod *v1.Pod) (string, error) } // Patroni API client @@ -123,3 +125,36 @@ func (p *Patroni) SetPostgresParameters(server *v1.Pod, parameters map[string]st } return p.httpPostOrPatch(http.MethodPatch, apiURLString+configPath, buf) } + +//GetPatroniMemberState returns a state of member of a Patroni cluster +func (p *Patroni) GetPatroniMemberState(server *v1.Pod) (string, error) { + + apiURLString, err := apiURL(server) + if err != nil { + return "", err + } + response, err := p.httpClient.Get(apiURLString) + if err != nil { + return "", fmt.Errorf("could not perform Get request: %v", err) + } + defer response.Body.Close() + + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return "", fmt.Errorf("could not read response: %v", err) + } + + data := make(map[string]interface{}) + err = json.Unmarshal(body, &data) + if err != nil { + return "", err + } + + state, ok := data["state"].(string) + if !ok { + return "", errors.New("Patroni Get call response contains wrong type for 'state' field") + } + + return state, nil + +} From 21b9b6fcbe5dc03eb52d76f302cfdb4cd36c80de Mon Sep 17 00:00:00 2001 From: Christian Rohmann Date: Mon, 27 Apr 2020 08:22:07 +0200 Subject: [PATCH 085/104] Emit K8S events to the postgresql CR as feedback to the requestor / user (#896) * Add EventsGetter to KubeClient to enable to sending K8S events * Add eventRecorder to the controller, initialize it and hand it down to cluster via its constructor to enable it to emit events this way * Add first set of events which then go to the postgresql custom resource the user interacts with to provide some feedback * Add right to "create" events to operator cluster role * Adapt cluster tests to new function sigurature with eventRecord (via NewFakeRecorder) * Get a proper reference before sending events to a resource Co-authored-by: Christian Rohmann --- .../templates/clusterrole.yaml | 7 ++++ manifests/operator-service-account-rbac.yaml | 7 ++++ pkg/cluster/cluster.go | 32 +++++++++++++++- pkg/cluster/cluster_test.go | 4 ++ pkg/cluster/k8sres_test.go | 32 +++++++++------- pkg/cluster/resources_test.go | 4 +- pkg/cluster/sync.go | 2 + pkg/cluster/sync_test.go | 2 +- pkg/controller/controller.go | 37 ++++++++++++++----- pkg/controller/postgresql.go | 7 +++- pkg/util/k8sutil/k8sutil.go | 2 + 11 files changed, 107 insertions(+), 29 deletions(-) diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 38ce85e7a..0defcab41 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -42,6 +42,13 @@ rules: - configmaps verbs: - get +# to send events to the CRs +- apiGroups: + - "" + resources: + - events + verbs: + - create # to manage endpoints which are also used by Patroni - apiGroups: - "" diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 83cd721e7..667941a24 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -43,6 +43,13 @@ rules: - configmaps verbs: - get +# to send events to the CRs +- apiGroups: + - "" + resources: + - events + verbs: + - create # to manage endpoints which are also used by Patroni - apiGroups: - "" diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 74cf6e61d..244916074 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -21,8 +21,11 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/reference" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" @@ -81,6 +84,7 @@ type Cluster struct { acidv1.Postgresql Config logger *logrus.Entry + eventRecorder record.EventRecorder patroni patroni.Interface pgUsers map[string]spec.PgUser systemUsers map[string]spec.PgUser @@ -109,7 +113,7 @@ type compareStatefulsetResult struct { } // New creates a new cluster. This function should be called from a controller. -func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgresql, logger *logrus.Entry) *Cluster { +func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgresql, logger *logrus.Entry, eventRecorder record.EventRecorder) *Cluster { deletePropagationPolicy := metav1.DeletePropagationOrphan podEventsQueue := cache.NewFIFO(func(obj interface{}) (string, error) { @@ -140,7 +144,7 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres cluster.teamsAPIClient = teams.NewTeamsAPI(cfg.OpConfig.TeamsAPIUrl, logger) cluster.oauthTokenGetter = newSecretOauthTokenGetter(&kubeClient, cfg.OpConfig.OAuthTokenSecretName) cluster.patroni = patroni.New(cluster.logger) - + cluster.eventRecorder = eventRecorder return cluster } @@ -166,6 +170,16 @@ func (c *Cluster) setProcessName(procName string, args ...interface{}) { } } +// GetReference of Postgres CR object +// i.e. required to emit events to this resource +func (c *Cluster) GetReference() *v1.ObjectReference { + ref, err := reference.GetReference(scheme.Scheme, &c.Postgresql) + if err != nil { + c.logger.Errorf("could not get reference for Postgresql CR %v/%v: %v", c.Postgresql.Namespace, c.Postgresql.Name, err) + } + return ref +} + // SetStatus of Postgres cluster // TODO: eventually switch to updateStatus() for kubernetes 1.11 and above func (c *Cluster) setStatus(status string) { @@ -245,6 +259,7 @@ func (c *Cluster) Create() error { }() c.setStatus(acidv1.ClusterStatusCreating) + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Create", "Started creation of new cluster resources") if err = c.enforceMinResourceLimits(&c.Spec); err != nil { return fmt.Errorf("could not enforce minimum resource limits: %v", err) @@ -263,6 +278,7 @@ func (c *Cluster) Create() error { return fmt.Errorf("could not create %s endpoint: %v", role, err) } c.logger.Infof("endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta)) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Endpoints", "Endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta)) } if c.Services[role] != nil { @@ -273,6 +289,7 @@ func (c *Cluster) Create() error { return fmt.Errorf("could not create %s service: %v", role, err) } c.logger.Infof("%s service %q has been successfully created", role, util.NameFromMeta(service.ObjectMeta)) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Services", "The service %q for role %s has been successfully created", util.NameFromMeta(service.ObjectMeta), role) } if err = c.initUsers(); err != nil { @@ -284,6 +301,7 @@ func (c *Cluster) Create() error { return fmt.Errorf("could not create secrets: %v", err) } c.logger.Infof("secrets have been successfully created") + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Secrets", "The secrets have been successfully created") if c.PodDisruptionBudget != nil { return fmt.Errorf("pod disruption budget already exists in the cluster") @@ -302,6 +320,7 @@ func (c *Cluster) Create() error { return fmt.Errorf("could not create statefulset: %v", err) } c.logger.Infof("statefulset %q has been successfully created", util.NameFromMeta(ss.ObjectMeta)) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "StatefulSet", "Statefulset %q has been successfully created", util.NameFromMeta(ss.ObjectMeta)) c.logger.Info("waiting for the cluster being ready") @@ -310,6 +329,7 @@ func (c *Cluster) Create() error { return err } c.logger.Infof("pods are ready") + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "StatefulSet", "Pods are ready") // create database objects unless we are running without pods or disabled // that feature explicitly @@ -555,6 +575,7 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error { } if isSmaller { c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit) spec.Resources.ResourceLimits.CPU = minCPULimit } } @@ -567,6 +588,7 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error { } if isSmaller { c.logger.Warningf("defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit) spec.Resources.ResourceLimits.Memory = minMemoryLimit } } @@ -598,6 +620,8 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { if oldSpec.Spec.PostgresqlParam.PgVersion != newSpec.Spec.PostgresqlParam.PgVersion { // PG versions comparison c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "PostgreSQL", "postgresql version change(%q -> %q) has no effect", + oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) //we need that hack to generate statefulset with the old version newSpec.Spec.PostgresqlParam.PgVersion = oldSpec.Spec.PostgresqlParam.PgVersion } @@ -757,6 +781,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { func (c *Cluster) Delete() { c.mu.Lock() defer c.mu.Unlock() + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Delete", "Started deletion of new cluster resources") // delete the backup job before the stateful set of the cluster to prevent connections to non-existing pods // deleting the cron job also removes pods and batch jobs it created @@ -1095,6 +1120,7 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e var err error c.logger.Debugf("switching over from %q to %q", curMaster.Name, candidate) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switching over from %q to %q", curMaster.Name, candidate) var wg sync.WaitGroup @@ -1121,6 +1147,7 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e if err = c.patroni.Switchover(curMaster, candidate.Name); err == nil { c.logger.Debugf("successfully switched over from %q to %q", curMaster.Name, candidate) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Successfully switched over from %q to %q", curMaster.Name, candidate) if err = <-podLabelErr; err != nil { err = fmt.Errorf("could not get master pod label: %v", err) } @@ -1136,6 +1163,7 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e // close the label waiting channel no sooner than the waiting goroutine terminates. close(podLabelErr) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switchover from %q to %q FAILED: %v", curMaster.Name, candidate, err) return err } diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 432f53132..84ec04e3e 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -13,6 +13,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/teams" v1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/record" ) const ( @@ -21,6 +22,8 @@ const ( ) var logger = logrus.New().WithField("test", "cluster") +var eventRecorder = record.NewFakeRecorder(1) + var cl = New( Config{ OpConfig: config.Config{ @@ -34,6 +37,7 @@ var cl = New( k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, + eventRecorder, ) func TestInitRobotUsers(t *testing.T) { diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 6e4587627..1291d4f47 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -37,7 +37,7 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { ReplicationUsername: replicationUserName, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) testName := "TestGenerateSpiloConfig" tests := []struct { @@ -102,7 +102,7 @@ func TestCreateLoadBalancerLogic(t *testing.T) { ReplicationUsername: replicationUserName, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) testName := "TestCreateLoadBalancerLogic" tests := []struct { @@ -164,7 +164,8 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { acidv1.Postgresql{ ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, - logger), + logger, + eventRecorder), policyv1beta1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: "postgres-myapp-database-pdb", @@ -187,7 +188,8 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { acidv1.Postgresql{ ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 0}}, - logger), + logger, + eventRecorder), policyv1beta1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: "postgres-myapp-database-pdb", @@ -210,7 +212,8 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { acidv1.Postgresql{ ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, - logger), + logger, + eventRecorder), policyv1beta1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: "postgres-myapp-database-pdb", @@ -233,7 +236,8 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { acidv1.Postgresql{ ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, - logger), + logger, + eventRecorder), policyv1beta1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: "postgres-myapp-database-databass-budget", @@ -368,7 +372,7 @@ func TestCloneEnv(t *testing.T) { ReplicationUsername: replicationUserName, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) for _, tt := range tests { envs := cluster.generateCloneEnvironment(tt.cloneOpts) @@ -502,7 +506,7 @@ func TestGetPgVersion(t *testing.T) { ReplicationUsername: replicationUserName, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) for _, tt := range tests { pgVersion, err := cluster.getNewPgVersion(tt.pgContainer, tt.newPgVersion) @@ -678,7 +682,7 @@ func TestConnectionPoolerPodSpec(t *testing.T) { ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) var clusterNoDefaultRes = New( Config{ @@ -690,7 +694,7 @@ func TestConnectionPoolerPodSpec(t *testing.T) { }, ConnectionPooler: config.ConnectionPooler{}, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) noCheck := func(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { return nil } @@ -803,7 +807,7 @@ func TestConnectionPoolerDeploymentSpec(t *testing.T) { ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) cluster.Statefulset = &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: "test-sts", @@ -904,7 +908,7 @@ func TestConnectionPoolerServiceSpec(t *testing.T) { ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) cluster.Statefulset = &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: "test-sts", @@ -990,7 +994,7 @@ func TestTLS(t *testing.T) { SpiloFSGroup: &spiloFSGroup, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) spec = makeSpec(acidv1.TLSDescription{SecretName: "my-secret", CAFile: "ca.crt"}) s, err := cluster.generateStatefulSet(&spec) if err != nil { @@ -1112,7 +1116,7 @@ func TestAdditionalVolume(t *testing.T) { ReplicationUsername: replicationUserName, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) for _, tt := range tests { // Test with additional volume mounted in all containers diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go index 2db807b38..9739cc354 100644 --- a/pkg/cluster/resources_test.go +++ b/pkg/cluster/resources_test.go @@ -36,7 +36,7 @@ func TestConnectionPoolerCreationAndDeletion(t *testing.T) { ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, - }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder) cluster.Statefulset = &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -85,7 +85,7 @@ func TestNeedConnectionPooler(t *testing.T) { ConnectionPoolerDefaultMemoryLimit: "100Mi", }, }, - }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder) cluster.Spec = acidv1.PostgresSpec{ ConnectionPooler: &acidv1.ConnectionPooler{}, diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 43173102b..a9a2b5177 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -343,10 +343,12 @@ func (c *Cluster) syncStatefulSet() error { // statefulset or those that got their configuration from the outdated statefulset) if podsRollingUpdateRequired { c.logger.Debugln("performing rolling update") + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Performing rolling update") if err := c.recreatePods(); err != nil { return fmt.Errorf("could not recreate pods: %v", err) } c.logger.Infof("pods have been recreated") + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Rolling update done - pods have been recreated") if err := c.applyRollingUpdateFlagforStatefulSet(false); err != nil { c.logger.Warningf("could not clear rolling update for the statefulset: %v", err) } diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index 50b5cfaa8..3a7317938 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -79,7 +79,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { NumberOfInstances: int32ToPointer(1), }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) cluster.Statefulset = &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 9c48b7ef2..4e4685379 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -7,24 +7,24 @@ import ( "sync" "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/tools/cache" - acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/apiserver" "github.com/zalando/postgres-operator/pkg/cluster" + acidv1informer "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/ringlog" - - acidv1informer "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do/v1" + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" ) // Controller represents operator controller @@ -36,6 +36,9 @@ type Controller struct { KubeClient k8sutil.KubernetesClient apiserver *apiserver.Server + eventRecorder record.EventRecorder + eventBroadcaster record.EventBroadcaster + stopCh chan struct{} controllerID string @@ -67,10 +70,21 @@ type Controller struct { func NewController(controllerConfig *spec.ControllerConfig, controllerId string) *Controller { logger := logrus.New() + var myComponentName = "postgres-operator" + if controllerId != "" { + myComponentName += "/" + controllerId + } + + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(logger.Debugf) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: myComponentName}) + c := &Controller{ config: *controllerConfig, opConfig: &config.Config{}, logger: logger.WithField("pkg", "controller"), + eventRecorder: recorder, + eventBroadcaster: eventBroadcaster, controllerID: controllerId, curWorkerCluster: sync.Map{}, clusterWorkers: make(map[spec.NamespacedName]uint32), @@ -93,6 +107,11 @@ func (c *Controller) initClients() { if err != nil { c.logger.Fatalf("could not create kubernetes clients: %v", err) } + c.eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: c.KubeClient.EventsGetter.Events("")}) + if err != nil { + c.logger.Fatalf("could not setup kubernetes event sink: %v", err) + } + } func (c *Controller) initOperatorConfig() { diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index e81671c7d..2a9e1b650 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -11,6 +11,7 @@ import ( "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/cache" @@ -157,7 +158,7 @@ func (c *Controller) acquireInitialListOfClusters() error { } func (c *Controller) addCluster(lg *logrus.Entry, clusterName spec.NamespacedName, pgSpec *acidv1.Postgresql) *cluster.Cluster { - cl := cluster.New(c.makeClusterConfig(), c.KubeClient, *pgSpec, lg) + cl := cluster.New(c.makeClusterConfig(), c.KubeClient, *pgSpec, lg, c.eventRecorder) cl.Run(c.stopCh) teamName := strings.ToLower(cl.Spec.TeamID) @@ -236,6 +237,7 @@ func (c *Controller) processEvent(event ClusterEvent) { if err := cl.Create(); err != nil { cl.Error = fmt.Sprintf("could not create cluster: %v", err) lg.Error(cl.Error) + c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Create", "%v", cl.Error) return } @@ -274,6 +276,8 @@ func (c *Controller) processEvent(event ClusterEvent) { c.curWorkerCluster.Store(event.WorkerID, cl) cl.Delete() + // Fixme - no error handling for delete ? + // c.eventRecorder.Eventf(cl.GetReference, v1.EventTypeWarning, "Delete", "%v", cl.Error) func() { defer c.clustersMu.Unlock() @@ -304,6 +308,7 @@ func (c *Controller) processEvent(event ClusterEvent) { c.curWorkerCluster.Store(event.WorkerID, cl) if err := cl.Sync(event.NewSpec); err != nil { cl.Error = fmt.Sprintf("could not sync cluster: %v", err) + c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Sync", "%v", cl.Error) lg.Error(cl.Error) return } diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 3a397af7d..d7be2f48a 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -45,6 +45,7 @@ type KubernetesClient struct { corev1.NodesGetter corev1.NamespacesGetter corev1.ServiceAccountsGetter + corev1.EventsGetter appsv1.StatefulSetsGetter appsv1.DeploymentsGetter rbacv1.RoleBindingsGetter @@ -142,6 +143,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.RESTClient = client.CoreV1().RESTClient() kubeClient.RoleBindingsGetter = client.RbacV1() kubeClient.CronJobsGetter = client.BatchV1beta1() + kubeClient.EventsGetter = client.CoreV1() apiextClient, err := apiextclient.NewForConfig(cfg) if err != nil { From f32c615a531426804d031b16967d69a4ec364c16 Mon Sep 17 00:00:00 2001 From: siku4 <44839490+siku4@users.noreply.github.com> Date: Mon, 27 Apr 2020 12:22:42 +0200 Subject: [PATCH 086/104] fix typo in additionalVolume struct (#933) * fix typo in additionalVolume struct Co-authored-by: siku4 --- manifests/complete-postgres-manifest.yaml | 32 ++++++++++---------- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index b469a7564..e701fdfaa 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -31,28 +31,28 @@ spec: size: 1Gi # storageClass: my-sc additionalVolumes: - - name: data - mountPath: /home/postgres/pgdata/partitions - targetContainers: - - postgres - volumeSource: - PersistentVolumeClaim: - claimName: pvc-postgresql-data-partitions - readyOnly: false - - name: conf - mountPath: /etc/telegraf - subPath: telegraf.conf - targetContainers: - - telegraf-sidecar - volumeSource: - configMap: - name: my-config-map - name: empty mountPath: /opt/empty targetContainers: - all volumeSource: emptyDir: {} +# - name: data +# mountPath: /home/postgres/pgdata/partitions +# targetContainers: +# - postgres +# volumeSource: +# PersistentVolumeClaim: +# claimName: pvc-postgresql-data-partitions +# readyOnly: false +# - name: conf +# mountPath: /etc/telegraf +# subPath: telegraf.conf +# targetContainers: +# - telegraf-sidecar +# volumeSource: +# configMap: +# name: my-config-map enableShmVolume: true # spiloFSGroup: 103 diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 961051c8d..e36009208 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -104,7 +104,7 @@ type AdditionalVolume struct { MountPath string `json:"mountPath"` SubPath string `json:"subPath"` TargetContainers []string `json:"targetContainers"` - VolumeSource v1.VolumeSource `json:"volume"` + VolumeSource v1.VolumeSource `json:"volumeSource"` } // PostgresqlParam describes PostgreSQL version and pairs of configuration parameter name - values. From 168abfe37b3e90cf1e3e4cb2cfd15f3c288d9235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Fischer?= Date: Mon, 27 Apr 2020 17:40:22 +0200 Subject: [PATCH 087/104] Fully speced global sidecars (#890) * implement fully speced global sidecars * fix issue #924 --- .../crds/operatorconfigurations.yaml | 6 + docs/administrator.md | 27 ++ docs/reference/operator_parameters.md | 27 +- docs/user.md | 2 + manifests/operatorconfiguration.crd.yaml | 6 + ...gresql-operator-default-configuration.yaml | 7 +- pkg/apis/acid.zalan.do/v1/crds.go | 11 + .../v1/operator_configuration_type.go | 27 +- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 11 +- pkg/cluster/k8sres.go | 237 ++++++++++-------- pkg/cluster/k8sres_test.go | 199 +++++++++++++++ pkg/cluster/util.go | 19 ++ pkg/controller/controller.go | 5 + pkg/controller/operator_config.go | 3 +- pkg/util/config/config.go | 15 +- 15 files changed, 462 insertions(+), 140 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 7e20c8fea..285d99b40 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -84,6 +84,12 @@ spec: type: object additionalProperties: type: string + sidecars: + type: array + nullable: true + items: + type: object + additionalProperties: true workers: type: integer minimum: 1 diff --git a/docs/administrator.md b/docs/administrator.md index 93adf2eb1..158b733ad 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -507,6 +507,33 @@ A secret can be pre-provisioned in different ways: * Automatically provisioned via a custom K8s controller like [kube-aws-iam-controller](https://github.com/mikkeloscar/kube-aws-iam-controller) +## Sidecars for Postgres clusters + +A list of sidecars is added to each cluster created by the +operator. The default is empty list. + + +```yaml +kind: OperatorConfiguration +configuration: + sidecars: + - image: image:123 + name: global-sidecar + ports: + - containerPort: 80 + volumeMounts: + - mountPath: /custom-pgdata-mountpoint + name: pgdata + - ... +``` + +In addition to any environment variables you specify, the following environment variables are always passed to sidecars: + + - `POD_NAME` - field reference to `metadata.name` + - `POD_NAMESPACE` - field reference to `metadata.namespace` + - `POSTGRES_USER` - the superuser that can be used to connect to the database + - `POSTGRES_PASSWORD` - the password for the superuser + ## Setting up the Postgres Operator UI Since the v1.2 release the Postgres Operator is shipped with a browser-based diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 3d31abab4..259f04527 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -93,9 +93,17 @@ Those are top-level keys, containing both leaf keys and groups. repository](https://github.com/zalando/spilo). * **sidecar_docker_images** - a map of sidecar names to Docker images to run with Spilo. In case of the name - conflict with the definition in the cluster manifest the cluster-specific one - is preferred. + *deprecated*: use **sidecars** instead. A map of sidecar names to Docker images to + run with Spilo. In case of the name conflict with the definition in the cluster + manifest the cluster-specific one is preferred. + +* **sidecars** + a list of sidecars to run with Spilo, for any cluster (i.e. globally defined sidecars). + Each item in the list is of type + [Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core). + Globally defined sidecars can be overwritten by specifying a sidecar in the custom resource with + the same name. Note: This field is not part of the schema validation. If the container specification + is invalid, then the operator fails to create the statefulset. * **enable_shm_volume** Instruct operator to start any new database pod without limitations on shm @@ -133,8 +141,9 @@ Those are top-level keys, containing both leaf keys and groups. at the cost of overprovisioning memory and potential scheduling problems for containers with high memory limits due to the lack of memory on Kubernetes cluster nodes. This affects all containers created by the operator (Postgres, - Scalyr sidecar, and other sidecars); to set resources for the operator's own - container, change the [operator deployment manually](../../manifests/postgres-operator.yaml#L20). + Scalyr sidecar, and other sidecars except **sidecars** defined in the operator + configuration); to set resources for the operator's own container, change the + [operator deployment manually](../../manifests/postgres-operator.yaml#L20). The default is `false`. ## Postgres users @@ -206,12 +215,12 @@ configuration they are grouped under the `kubernetes` key. Default is true. * **enable_init_containers** - global option to allow for creating init containers to run actions before - Spilo is started. Default is true. + global option to allow for creating init containers in the cluster manifest to + run actions before Spilo is started. Default is true. * **enable_sidecars** - global option to allow for creating sidecar containers to run alongside Spilo - on the same pod. Default is true. + global option to allow for creating sidecar containers in the cluster manifest + to run alongside Spilo on the same pod. Globally defined sidecars are always enabled. Default is true. * **secret_name_template** a template for the name of the database user secrets generated by the diff --git a/docs/user.md b/docs/user.md index 2c1c4fd1f..d7e6add0a 100644 --- a/docs/user.md +++ b/docs/user.md @@ -442,6 +442,8 @@ The PostgreSQL volume is shared with sidecars and is mounted at specified but globally disabled in the configuration. The `enable_sidecars` option must be set to `true`. +If you want to add a sidecar to every cluster managed by the operator, you can specify it in the [operator configuration](administrator.md#sidecars-for-postgres-clusters) instead. + ## InitContainers Support Each cluster can specify arbitrary init containers to run. These containers can diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 86051e43b..b2496c9c9 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -60,6 +60,12 @@ spec: type: object additionalProperties: type: string + sidecars: + type: array + nullable: true + items: + type: object + additionalProperties: true workers: type: integer minimum: 1 diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 209e2684b..e80bfa846 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -13,8 +13,11 @@ configuration: resync_period: 30m repair_period: 5m # set_memory_request_to_limit: false - # sidecar_docker_images: - # example: "exampleimage:exampletag" + # sidecars: + # - image: image:123 + # name: global-sidecar-1 + # ports: + # - containerPort: 80 workers: 4 users: replication_username: standby diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 3f4314240..bcb35c56c 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -797,6 +797,17 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "sidecars": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Allows: true, + }, + }, + }, + }, "workers": { Type: "integer", Minimum: &min1, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 8ed4281f4..c377a294b 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -8,6 +8,7 @@ import ( "time" "github.com/zalando/postgres-operator/pkg/spec" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -181,18 +182,20 @@ type OperatorLogicalBackupConfiguration struct { // OperatorConfigurationData defines the operation config type OperatorConfigurationData struct { - EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` - EtcdHost string `json:"etcd_host,omitempty"` - KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` - DockerImage string `json:"docker_image,omitempty"` - Workers uint32 `json:"workers,omitempty"` - MinInstances int32 `json:"min_instances,omitempty"` - MaxInstances int32 `json:"max_instances,omitempty"` - ResyncPeriod Duration `json:"resync_period,omitempty"` - RepairPeriod Duration `json:"repair_period,omitempty"` - SetMemoryRequestToLimit bool `json:"set_memory_request_to_limit,omitempty"` - ShmVolume *bool `json:"enable_shm_volume,omitempty"` - Sidecars map[string]string `json:"sidecar_docker_images,omitempty"` + EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` + EtcdHost string `json:"etcd_host,omitempty"` + KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` + DockerImage string `json:"docker_image,omitempty"` + Workers uint32 `json:"workers,omitempty"` + MinInstances int32 `json:"min_instances,omitempty"` + MaxInstances int32 `json:"max_instances,omitempty"` + ResyncPeriod Duration `json:"resync_period,omitempty"` + RepairPeriod Duration `json:"repair_period,omitempty"` + SetMemoryRequestToLimit bool `json:"set_memory_request_to_limit,omitempty"` + ShmVolume *bool `json:"enable_shm_volume,omitempty"` + // deprecated in favour of SidecarContainers + SidecarImages map[string]string `json:"sidecar_docker_images,omitempty"` + SidecarContainers []v1.Container `json:"sidecars,omitempty"` PostgresUsersConfiguration PostgresUsersConfiguration `json:"users"` Kubernetes KubernetesMetaConfiguration `json:"kubernetes"` PostgresPodResources PostgresPodResourcesDefaults `json:"postgres_pod_resources"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index e6b387ec4..e2e1d5bd1 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -312,13 +312,20 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData *out = new(bool) **out = **in } - if in.Sidecars != nil { - in, out := &in.Sidecars, &out.Sidecars + if in.SidecarImages != nil { + in, out := &in.SidecarImages, &out.SidecarImages *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } + if in.SidecarContainers != nil { + in, out := &in.SidecarContainers, &out.SidecarContainers + *out = make([]corev1.Container, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } out.PostgresUsersConfiguration = in.PostgresUsersConfiguration in.Kubernetes.DeepCopyInto(&out.Kubernetes) out.PostgresPodResources = in.PostgresPodResources diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 9fb33eab2..43190491b 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -462,8 +462,7 @@ func generateContainer( } func generateSidecarContainers(sidecars []acidv1.Sidecar, - volumeMounts []v1.VolumeMount, defaultResources acidv1.Resources, - superUserName string, credentialsSecretName string, logger *logrus.Entry) ([]v1.Container, error) { + defaultResources acidv1.Resources, startIndex int, logger *logrus.Entry) ([]v1.Container, error) { if len(sidecars) > 0 { result := make([]v1.Container, 0) @@ -482,7 +481,7 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar, return nil, err } - sc := getSidecarContainer(sidecar, index, volumeMounts, resources, superUserName, credentialsSecretName, logger) + sc := getSidecarContainer(sidecar, startIndex+index, resources) result = append(result, *sc) } return result, nil @@ -490,6 +489,55 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar, return nil, nil } +// adds common fields to sidecars +func patchSidecarContainers(in []v1.Container, volumeMounts []v1.VolumeMount, superUserName string, credentialsSecretName string, logger *logrus.Entry) []v1.Container { + result := []v1.Container{} + + for _, container := range in { + container.VolumeMounts = append(container.VolumeMounts, volumeMounts...) + env := []v1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "POSTGRES_USER", + Value: superUserName, + }, + { + Name: "POSTGRES_PASSWORD", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: credentialsSecretName, + }, + Key: "password", + }, + }, + }, + } + mergedEnv := append(container.Env, env...) + container.Env = deduplicateEnvVars(mergedEnv, container.Name, logger) + result = append(result, container) + } + + return result +} + // Check whether or not we're requested to mount an shm volume, // taking into account that PostgreSQL manifest has precedence. func mountShmVolumeNeeded(opConfig config.Config, spec *acidv1.PostgresSpec) *bool { @@ -724,58 +772,18 @@ func deduplicateEnvVars(input []v1.EnvVar, containerName string, logger *logrus. return result } -func getSidecarContainer(sidecar acidv1.Sidecar, index int, volumeMounts []v1.VolumeMount, - resources *v1.ResourceRequirements, superUserName string, credentialsSecretName string, logger *logrus.Entry) *v1.Container { +func getSidecarContainer(sidecar acidv1.Sidecar, index int, resources *v1.ResourceRequirements) *v1.Container { name := sidecar.Name if name == "" { name = fmt.Sprintf("sidecar-%d", index) } - env := []v1.EnvVar{ - { - Name: "POD_NAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.name", - }, - }, - }, - { - Name: "POD_NAMESPACE", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.namespace", - }, - }, - }, - { - Name: "POSTGRES_USER", - Value: superUserName, - }, - { - Name: "POSTGRES_PASSWORD", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: credentialsSecretName, - }, - Key: "password", - }, - }, - }, - } - if len(sidecar.Env) > 0 { - env = append(env, sidecar.Env...) - } return &v1.Container{ Name: name, Image: sidecar.DockerImage, ImagePullPolicy: v1.PullIfNotPresent, Resources: *resources, - VolumeMounts: volumeMounts, - Env: deduplicateEnvVars(env, name, logger), + Env: sidecar.Env, Ports: sidecar.Ports, } } @@ -1065,37 +1073,63 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.OpConfig.Resources.SpiloPrivileged, ) - // resolve conflicts between operator-global and per-cluster sidecars - sideCars := c.mergeSidecars(spec.Sidecars) + // generate container specs for sidecars specified in the cluster manifest + clusterSpecificSidecars := []v1.Container{} + if spec.Sidecars != nil && len(spec.Sidecars) > 0 { + // warn if sidecars are defined, but globally disabled (does not apply to globally defined sidecars) + if c.OpConfig.EnableSidecars != nil && !(*c.OpConfig.EnableSidecars) { + c.logger.Warningf("sidecars specified but disabled in configuration - next statefulset creation would fail") + } - resourceRequirementsScalyrSidecar := makeResources( - c.OpConfig.ScalyrCPURequest, - c.OpConfig.ScalyrMemoryRequest, - c.OpConfig.ScalyrCPULimit, - c.OpConfig.ScalyrMemoryLimit, - ) + if clusterSpecificSidecars, err = generateSidecarContainers(spec.Sidecars, defaultResources, 0, c.logger); err != nil { + return nil, fmt.Errorf("could not generate sidecar containers: %v", err) + } + } + + // decrapted way of providing global sidecars + var globalSidecarContainersByDockerImage []v1.Container + var globalSidecarsByDockerImage []acidv1.Sidecar + for name, dockerImage := range c.OpConfig.SidecarImages { + globalSidecarsByDockerImage = append(globalSidecarsByDockerImage, acidv1.Sidecar{Name: name, DockerImage: dockerImage}) + } + if globalSidecarContainersByDockerImage, err = generateSidecarContainers(globalSidecarsByDockerImage, defaultResources, len(clusterSpecificSidecars), c.logger); err != nil { + return nil, fmt.Errorf("could not generate sidecar containers: %v", err) + } + // make the resulting list reproducible + // c.OpConfig.SidecarImages is unsorted by Golang definition + // .Name is unique + sort.Slice(globalSidecarContainersByDockerImage, func(i, j int) bool { + return globalSidecarContainersByDockerImage[i].Name < globalSidecarContainersByDockerImage[j].Name + }) // generate scalyr sidecar container - if scalyrSidecar := + var scalyrSidecars []v1.Container + if scalyrSidecar, err := generateScalyrSidecarSpec(c.Name, c.OpConfig.ScalyrAPIKey, c.OpConfig.ScalyrServerURL, c.OpConfig.ScalyrImage, - &resourceRequirementsScalyrSidecar, c.logger); scalyrSidecar != nil { - sideCars = append(sideCars, *scalyrSidecar) + c.OpConfig.ScalyrCPURequest, + c.OpConfig.ScalyrMemoryRequest, + c.OpConfig.ScalyrCPULimit, + c.OpConfig.ScalyrMemoryLimit, + defaultResources, + c.logger); err != nil { + return nil, fmt.Errorf("could not generate Scalyr sidecar: %v", err) + } else { + if scalyrSidecar != nil { + scalyrSidecars = append(scalyrSidecars, *scalyrSidecar) + } } - // generate sidecar containers - if sideCars != nil && len(sideCars) > 0 { - if c.OpConfig.EnableSidecars != nil && !(*c.OpConfig.EnableSidecars) { - c.logger.Warningf("sidecars specified but disabled in configuration - next statefulset creation would fail") - } - if sidecarContainers, err = generateSidecarContainers(sideCars, volumeMounts, defaultResources, - c.OpConfig.SuperUsername, c.credentialSecretName(c.OpConfig.SuperUsername), c.logger); err != nil { - return nil, fmt.Errorf("could not generate sidecar containers: %v", err) - } + sidecarContainers, conflicts := mergeContainers(clusterSpecificSidecars, c.Config.OpConfig.SidecarContainers, globalSidecarContainersByDockerImage, scalyrSidecars) + for containerName := range conflicts { + c.logger.Warningf("a sidecar is specified twice. Ignoring sidecar %q in favor of %q with high a precendence", + containerName, containerName) } + sidecarContainers = patchSidecarContainers(sidecarContainers, volumeMounts, c.OpConfig.SuperUsername, c.credentialSecretName(c.OpConfig.SuperUsername), c.logger) + tolerationSpec := tolerations(&spec.Tolerations, c.OpConfig.PodToleration) effectivePodPriorityClassName := util.Coalesce(spec.PodPriorityClassName, c.OpConfig.PodPriorityClassName) @@ -1188,57 +1222,44 @@ func (c *Cluster) generatePodAnnotations(spec *acidv1.PostgresSpec) map[string]s } func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage string, - containerResources *acidv1.Resources, logger *logrus.Entry) *acidv1.Sidecar { + scalyrCPURequest string, scalyrMemoryRequest string, scalyrCPULimit string, scalyrMemoryLimit string, + defaultResources acidv1.Resources, logger *logrus.Entry) (*v1.Container, error) { if APIKey == "" || dockerImage == "" { if APIKey == "" && dockerImage != "" { logger.Warning("Not running Scalyr sidecar: SCALYR_API_KEY must be defined") } - return nil + return nil, nil } - scalarSpec := &acidv1.Sidecar{ - Name: "scalyr-sidecar", - DockerImage: dockerImage, - Env: []v1.EnvVar{ - { - Name: "SCALYR_API_KEY", - Value: APIKey, - }, - { - Name: "SCALYR_SERVER_HOST", - Value: clusterName, - }, + resourcesScalyrSidecar := makeResources( + scalyrCPURequest, + scalyrMemoryRequest, + scalyrCPULimit, + scalyrMemoryLimit, + ) + resourceRequirementsScalyrSidecar, err := generateResourceRequirements(resourcesScalyrSidecar, defaultResources) + if err != nil { + return nil, fmt.Errorf("invalid resources for Scalyr sidecar: %v", err) + } + env := []v1.EnvVar{ + { + Name: "SCALYR_API_KEY", + Value: APIKey, + }, + { + Name: "SCALYR_SERVER_HOST", + Value: clusterName, }, - Resources: *containerResources, } if serverURL != "" { - scalarSpec.Env = append(scalarSpec.Env, v1.EnvVar{Name: "SCALYR_SERVER_URL", Value: serverURL}) + env = append(env, v1.EnvVar{Name: "SCALYR_SERVER_URL", Value: serverURL}) } - return scalarSpec -} - -// mergeSidecar merges globally-defined sidecars with those defined in the cluster manifest -func (c *Cluster) mergeSidecars(sidecars []acidv1.Sidecar) []acidv1.Sidecar { - globalSidecarsToSkip := map[string]bool{} - result := make([]acidv1.Sidecar, 0) - - for i, sidecar := range sidecars { - dockerImage, ok := c.OpConfig.Sidecars[sidecar.Name] - if ok { - if dockerImage != sidecar.DockerImage { - c.logger.Warningf("merging definitions for sidecar %q: "+ - "ignoring %q in the global scope in favor of %q defined in the cluster", - sidecar.Name, dockerImage, sidecar.DockerImage) - } - globalSidecarsToSkip[sidecar.Name] = true - } - result = append(result, sidecars[i]) - } - for name, dockerImage := range c.OpConfig.Sidecars { - if !globalSidecarsToSkip[name] { - result = append(result, acidv1.Sidecar{Name: name, DockerImage: dockerImage}) - } - } - return result + return &v1.Container{ + Name: "scalyr-sidecar", + Image: dockerImage, + Env: env, + ImagePullPolicy: v1.PullIfNotPresent, + Resources: *resourceRequirementsScalyrSidecar, + }, nil } func (c *Cluster) getNumberOfInstances(spec *acidv1.PostgresSpec) int32 { @@ -1803,7 +1824,7 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { c.OpConfig.AdditionalSecretMount, c.OpConfig.AdditionalSecretMountPath, []acidv1.AdditionalVolume{}); err != nil { - return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) + return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) } // overwrite specific params of logical backups pods diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 1291d4f47..f9e95feef 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -18,6 +18,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -1206,3 +1207,201 @@ func TestAdditionalVolume(t *testing.T) { } } } + +// inject sidecars through all available mechanisms and check the resulting container specs +func TestSidecars(t *testing.T) { + var err error + var spec acidv1.PostgresSpec + var cluster *Cluster + + generateKubernetesResources := func(cpuRequest string, cpuLimit string, memoryRequest string, memoryLimit string) v1.ResourceRequirements { + parsedCPURequest, err := resource.ParseQuantity(cpuRequest) + assert.NoError(t, err) + parsedCPULimit, err := resource.ParseQuantity(cpuLimit) + assert.NoError(t, err) + parsedMemoryRequest, err := resource.ParseQuantity(memoryRequest) + assert.NoError(t, err) + parsedMemoryLimit, err := resource.ParseQuantity(memoryLimit) + assert.NoError(t, err) + return v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: parsedCPURequest, + v1.ResourceMemory: parsedMemoryRequest, + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: parsedCPULimit, + v1.ResourceMemory: parsedMemoryLimit, + }, + } + } + + spec = acidv1.PostgresSpec{ + TeamID: "myapp", NumberOfInstances: 1, + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + Sidecars: []acidv1.Sidecar{ + acidv1.Sidecar{ + Name: "cluster-specific-sidecar", + }, + acidv1.Sidecar{ + Name: "cluster-specific-sidecar-with-resources", + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "210m", Memory: "0.8Gi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "510m", Memory: "1.4Gi"}, + }, + }, + acidv1.Sidecar{ + Name: "replace-sidecar", + DockerImage: "overwrite-image", + }, + }, + } + + cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + Resources: config.Resources{ + DefaultCPURequest: "200m", + DefaultCPULimit: "500m", + DefaultMemoryRequest: "0.7Gi", + DefaultMemoryLimit: "1.3Gi", + }, + SidecarImages: map[string]string{ + "deprecated-global-sidecar": "image:123", + }, + SidecarContainers: []v1.Container{ + v1.Container{ + Name: "global-sidecar", + }, + // will be replaced by a cluster specific sidecar with the same name + v1.Container{ + Name: "replace-sidecar", + Image: "replaced-image", + }, + }, + Scalyr: config.Scalyr{ + ScalyrAPIKey: "abc", + ScalyrImage: "scalyr-image", + ScalyrCPURequest: "220m", + ScalyrCPULimit: "520m", + ScalyrMemoryRequest: "0.9Gi", + // ise default memory limit + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + s, err := cluster.generateStatefulSet(&spec) + assert.NoError(t, err) + + env := []v1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "POSTGRES_USER", + Value: superUserName, + }, + { + Name: "POSTGRES_PASSWORD", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "", + }, + Key: "password", + }, + }, + }, + } + mounts := []v1.VolumeMount{ + v1.VolumeMount{ + Name: "pgdata", + MountPath: "/home/postgres/pgdata", + }, + } + + // deduplicated sidecars and Patroni + assert.Equal(t, 7, len(s.Spec.Template.Spec.Containers), "wrong number of containers") + + // cluster specific sidecar + assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ + Name: "cluster-specific-sidecar", + Env: env, + Resources: generateKubernetesResources("200m", "500m", "0.7Gi", "1.3Gi"), + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: mounts, + }) + + // container specific resources + expectedResources := generateKubernetesResources("210m", "510m", "0.8Gi", "1.4Gi") + assert.Equal(t, expectedResources.Requests[v1.ResourceCPU], s.Spec.Template.Spec.Containers[2].Resources.Requests[v1.ResourceCPU]) + assert.Equal(t, expectedResources.Limits[v1.ResourceCPU], s.Spec.Template.Spec.Containers[2].Resources.Limits[v1.ResourceCPU]) + assert.Equal(t, expectedResources.Requests[v1.ResourceMemory], s.Spec.Template.Spec.Containers[2].Resources.Requests[v1.ResourceMemory]) + assert.Equal(t, expectedResources.Limits[v1.ResourceMemory], s.Spec.Template.Spec.Containers[2].Resources.Limits[v1.ResourceMemory]) + + // deprecated global sidecar + assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ + Name: "deprecated-global-sidecar", + Image: "image:123", + Env: env, + Resources: generateKubernetesResources("200m", "500m", "0.7Gi", "1.3Gi"), + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: mounts, + }) + + // global sidecar + assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ + Name: "global-sidecar", + Env: env, + VolumeMounts: mounts, + }) + + // replaced sidecar + assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ + Name: "replace-sidecar", + Image: "overwrite-image", + Resources: generateKubernetesResources("200m", "500m", "0.7Gi", "1.3Gi"), + ImagePullPolicy: v1.PullIfNotPresent, + Env: env, + VolumeMounts: mounts, + }) + + // replaced sidecar + // the order in env is important + scalyrEnv := append([]v1.EnvVar{v1.EnvVar{Name: "SCALYR_API_KEY", Value: "abc"}, v1.EnvVar{Name: "SCALYR_SERVER_HOST", Value: ""}}, env...) + assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ + Name: "scalyr-sidecar", + Image: "scalyr-image", + Resources: generateKubernetesResources("220m", "520m", "0.9Gi", "1.3Gi"), + ImagePullPolicy: v1.PullIfNotPresent, + Env: scalyrEnv, + VolumeMounts: mounts, + }) + +} diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 4dcdfb28a..7559ce3d4 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -530,3 +530,22 @@ func (c *Cluster) needConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool { func (c *Cluster) needConnectionPooler() bool { return c.needConnectionPoolerWorker(&c.Spec) } + +// Earlier arguments take priority +func mergeContainers(containers ...[]v1.Container) ([]v1.Container, []string) { + containerNameTaken := map[string]bool{} + result := make([]v1.Container, 0) + conflicts := make([]string, 0) + + for _, containerArray := range containers { + for _, container := range containerArray { + if _, taken := containerNameTaken[container.Name]; taken { + conflicts = append(conflicts, container.Name) + } else { + containerNameTaken[container.Name] = true + result = append(result, container) + } + } + } + return result, conflicts +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 4e4685379..0b3fde5d9 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -178,6 +178,11 @@ func (c *Controller) warnOnDeprecatedOperatorParameters() { c.logger.Warningf("Operator configuration parameter 'enable_load_balancer' is deprecated and takes no effect. " + "Consider using the 'enable_master_load_balancer' or 'enable_replica_load_balancer' instead.") } + + if len(c.opConfig.SidecarImages) > 0 { + c.logger.Warningf("Operator configuration parameter 'sidecar_docker_images' is deprecated. " + + "Consider using 'sidecars' instead.") + } } func (c *Controller) initPodServiceAccount() { diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 07be90f22..c1756604b 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -44,7 +44,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.RepairPeriod = time.Duration(fromCRD.RepairPeriod) result.SetMemoryRequestToLimit = fromCRD.SetMemoryRequestToLimit result.ShmVolume = fromCRD.ShmVolume - result.Sidecars = fromCRD.Sidecars + result.SidecarImages = fromCRD.SidecarImages + result.SidecarContainers = fromCRD.SidecarContainers // user config result.SuperUsername = fromCRD.PostgresUsersConfiguration.SuperUsername diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 84a62c0fd..9c2257e78 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -9,6 +9,7 @@ import ( "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/constants" + v1 "k8s.io/api/core/v1" ) // CRD describes CustomResourceDefinition specific configuration parameters @@ -107,12 +108,14 @@ type Config struct { LogicalBackup ConnectionPooler - WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' - KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` - EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` - Sidecars map[string]string `name:"sidecar_docker_images"` - PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` + WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' + KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` + EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` + // deprecated in favour of SidecarContainers + SidecarImages map[string]string `name:"sidecar_docker_images"` + SidecarContainers []v1.Container `name:"sidecars"` + PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` // value of this string must be valid JSON or YAML; see initPodServiceAccount PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""` PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""` From 0ca30ba3d98a85972e6725da64417f16e26419ae Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Tue, 28 Apr 2020 09:31:41 +0200 Subject: [PATCH 088/104] fix params in function call (#939) Co-authored-by: Sergey Dudoladov --- pkg/cluster/k8sres_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index f9e95feef..d09a2c0aa 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1299,7 +1299,7 @@ func TestSidecars(t *testing.T) { // ise default memory limit }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) s, err := cluster.generateStatefulSet(&spec) assert.NoError(t, err) From 1d009d9595be614d67a72b9bcd520c04dc349aa1 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 28 Apr 2020 16:01:13 +0200 Subject: [PATCH 089/104] bump spilo and pooler version + update docs (#945) --- .../crds/operatorconfigurations.yaml | 2 +- .../templates/operatorconfiguration.yaml | 2 -- charts/postgres-operator/values-crd.yaml | 19 +---------- charts/postgres-operator/values.yaml | 2 +- docs/administrator.md | 8 ++--- .../reference/command_line_and_environment.md | 2 +- docs/reference/operator_parameters.md | 33 ++++++++++--------- manifests/complete-postgres-manifest.yaml | 2 +- manifests/configmap.yaml | 4 +-- manifests/operatorconfiguration.crd.yaml | 2 +- ...gresql-operator-default-configuration.yaml | 12 ++----- pkg/util/config/config.go | 2 +- 12 files changed, 33 insertions(+), 57 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 285d99b40..aeb3d2150 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -305,7 +305,7 @@ spec: type: integer ring_log_lines: type: integer - scalyr: + scalyr: # deprecated type: object properties: scalyr_api_key: diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index 5df6e6238..d28d68f9c 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -32,8 +32,6 @@ configuration: {{ toYaml .Values.configTeamsApi | indent 4 }} logging_rest_api: {{ toYaml .Values.configLoggingRestApi | indent 4 }} - scalyr: -{{ toYaml .Values.configScalyr | indent 4 }} connection_pooler: {{ toYaml .Values.configConnectionPooler | indent 4 }} {{- end }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index caa4dda4d..41b8dbefb 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -26,7 +26,7 @@ configGeneral: # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) # kubernetes_use_configmaps: false # Spilo docker image - docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 + docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 # max number of instances in Postgres cluster. -1 = no limit min_instances: -1 # min number of instances in Postgres cluster. -1 = no limit @@ -252,23 +252,6 @@ configTeamsApi: # URL of the Teams API service # teams_api_url: http://fake-teams-api.default.svc.cluster.local -# Scalyr is a log management tool that Zalando uses as a sidecar -configScalyr: - # API key for the Scalyr sidecar - # scalyr_api_key: "" - - # Docker image for the Scalyr sidecar - # scalyr_image: "" - - # CPU limit value for the Scalyr sidecar - scalyr_cpu_limit: "1" - # CPU rquest value for the Scalyr sidecar - scalyr_cpu_request: 100m - # Memory limit value for the Scalyr sidecar - scalyr_memory_limit: 500Mi - # Memory request value for the Scalyr sidecar - scalyr_memory_request: 50Mi - configConnectionPooler: # db schema to install lookup function into connection_pooler_schema: "pooler" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index e7db249f0..7f140f1de 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -26,7 +26,7 @@ configGeneral: # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) # kubernetes_use_configmaps: "false" # Spilo docker image - docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 + docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 # max number of instances in Postgres cluster. -1 = no limit min_instances: "-1" # min number of instances in Postgres cluster. -1 = no limit diff --git a/docs/administrator.md b/docs/administrator.md index 158b733ad..ed96b7d35 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -509,9 +509,8 @@ A secret can be pre-provisioned in different ways: ## Sidecars for Postgres clusters -A list of sidecars is added to each cluster created by the -operator. The default is empty list. - +A list of sidecars is added to each cluster created by the operator. The default +is empty. ```yaml kind: OperatorConfiguration @@ -527,7 +526,8 @@ configuration: - ... ``` -In addition to any environment variables you specify, the following environment variables are always passed to sidecars: +In addition to any environment variables you specify, the following environment +variables are always passed to sidecars: - `POD_NAME` - field reference to `metadata.name` - `POD_NAMESPACE` - field reference to `metadata.namespace` diff --git a/docs/reference/command_line_and_environment.md b/docs/reference/command_line_and_environment.md index ec5da5ceb..ece29b094 100644 --- a/docs/reference/command_line_and_environment.md +++ b/docs/reference/command_line_and_environment.md @@ -45,7 +45,7 @@ The following environment variables are accepted by the operator: all namespaces. Empty value defaults to the operator namespace. Overrides the `watched_namespace` operator parameter. -* **SCALYR_API_KEY** +* **SCALYR_API_KEY** (*deprecated*) the value of the Scalyr API key to supply to the pods. Overrides the `scalyr_api_key` operator parameter. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 259f04527..d98a270e6 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -93,17 +93,18 @@ Those are top-level keys, containing both leaf keys and groups. repository](https://github.com/zalando/spilo). * **sidecar_docker_images** - *deprecated*: use **sidecars** instead. A map of sidecar names to Docker images to - run with Spilo. In case of the name conflict with the definition in the cluster - manifest the cluster-specific one is preferred. + *deprecated*: use **sidecars** instead. A map of sidecar names to Docker + images to run with Spilo. In case of the name conflict with the definition in + the cluster manifest the cluster-specific one is preferred. * **sidecars** - a list of sidecars to run with Spilo, for any cluster (i.e. globally defined sidecars). - Each item in the list is of type + a list of sidecars to run with Spilo, for any cluster (i.e. globally defined + sidecars). Each item in the list is of type [Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core). - Globally defined sidecars can be overwritten by specifying a sidecar in the custom resource with - the same name. Note: This field is not part of the schema validation. If the container specification - is invalid, then the operator fails to create the statefulset. + Globally defined sidecars can be overwritten by specifying a sidecar in the + Postgres manifest with the same name. + Note: This field is not part of the schema validation. If the container + specification is invalid, then the operator fails to create the statefulset. * **enable_shm_volume** Instruct operator to start any new database pod without limitations on shm @@ -141,8 +142,8 @@ Those are top-level keys, containing both leaf keys and groups. at the cost of overprovisioning memory and potential scheduling problems for containers with high memory limits due to the lack of memory on Kubernetes cluster nodes. This affects all containers created by the operator (Postgres, - Scalyr sidecar, and other sidecars except **sidecars** defined in the operator - configuration); to set resources for the operator's own container, change the + Scalyr sidecar, and other sidecars except **sidecars** defined in the operator + configuration); to set resources for the operator's own container, change the [operator deployment manually](../../manifests/postgres-operator.yaml#L20). The default is `false`. @@ -215,12 +216,13 @@ configuration they are grouped under the `kubernetes` key. Default is true. * **enable_init_containers** - global option to allow for creating init containers in the cluster manifest to + global option to allow for creating init containers in the cluster manifest to run actions before Spilo is started. Default is true. * **enable_sidecars** - global option to allow for creating sidecar containers in the cluster manifest - to run alongside Spilo on the same pod. Globally defined sidecars are always enabled. Default is true. + global option to allow for creating sidecar containers in the cluster manifest + to run alongside Spilo on the same pod. Globally defined sidecars are always + enabled. Default is true. * **secret_name_template** a template for the name of the database user secrets generated by the @@ -585,11 +587,12 @@ configuration they are grouped under the `logging_rest_api` key. * **cluster_history_entries** number of entries in the cluster history ring buffer. The default is `1000`. -## Scalyr options +## Scalyr options (*deprecated*) Those parameters define the resource requests/limits and properties of the scalyr sidecar. In the CRD-based configuration they are grouped under the -`scalyr` key. +`scalyr` key. Note, that this section is deprecated. Instead, define Scalyr as +a global sidecar under the `sidecars` key in the configuration. * **scalyr_api_key** API key for the Scalyr sidecar. The default is empty. diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index e701fdfaa..f031a5d5b 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -7,7 +7,7 @@ metadata: # annotations: # "acid.zalan.do/controller": "second-operator" spec: - dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 + dockerImage: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 teamId: "acid" numberOfInstances: 2 users: # Application/Robot users diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 954881ed3..cbc55b446 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,7 +15,7 @@ data: # connection_pooler_default_cpu_request: "500m" # connection_pooler_default_memory_limit: 100Mi # connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-6" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7" # connection_pooler_max_db_connections: 60 # connection_pooler_mode: "transaction" # connection_pooler_number_of_instances: 2 @@ -29,7 +29,7 @@ data: # default_cpu_request: 100m # default_memory_limit: 500Mi # default_memory_request: 100Mi - docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 + docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 # enable_admin_role_for_users: "true" # enable_crd_validation: "true" # enable_database_access: "true" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index b2496c9c9..53097b96c 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -281,7 +281,7 @@ spec: type: integer ring_log_lines: type: integer - scalyr: + scalyr: # deprecated type: object properties: scalyr_api_key: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index e80bfa846..b5765a0f4 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -6,7 +6,7 @@ configuration: # enable_crd_validation: true etcd_host: "" # kubernetes_use_configmaps: false - docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2 + docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 # enable_shm_volume: true max_instances: -1 min_instances: -1 @@ -117,20 +117,12 @@ configuration: api_port: 8080 cluster_history_entries: 1000 ring_log_lines: 100 - scalyr: - # scalyr_api_key: "" - scalyr_cpu_limit: "1" - scalyr_cpu_request: 100m - # scalyr_image: "" - scalyr_memory_limit: 500Mi - scalyr_memory_request: 50Mi - # scalyr_server_url: "" connection_pooler: connection_pooler_default_cpu_limit: "1" connection_pooler_default_cpu_request: "500m" connection_pooler_default_memory_limit: 100Mi connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-6" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7" # connection_pooler_max_db_connections: 60 connection_pooler_mode: "transaction" connection_pooler_number_of_instances: 2 diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 9c2257e78..2bdeb8e1f 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -111,7 +111,7 @@ type Config struct { WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p2"` + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115"` // deprecated in favour of SidecarContainers SidecarImages map[string]string `name:"sidecar_docker_images"` SidecarContainers []v1.Container `name:"sidecars"` From 0016ebf7df2596fceb8f5b1a0fb8156501fbce97 Mon Sep 17 00:00:00 2001 From: Marcus Portmann Date: Wed, 29 Apr 2020 07:28:35 +0200 Subject: [PATCH 090/104] Allow nodePort value for the postgres-operator-ui service (#928) * Added support for specifying a nodePort value for the postgres-operator-ui service when the type is NodePort Co-authored-by: Marcus Portmann --- charts/postgres-operator-ui/templates/service.yaml | 3 +++ charts/postgres-operator-ui/values.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/charts/postgres-operator-ui/templates/service.yaml b/charts/postgres-operator-ui/templates/service.yaml index 09adff26f..bc40fbbb1 100644 --- a/charts/postgres-operator-ui/templates/service.yaml +++ b/charts/postgres-operator-ui/templates/service.yaml @@ -11,6 +11,9 @@ spec: ports: - port: {{ .Values.service.port }} targetPort: 8081 + {{- if and (eq .Values.service.type "NodePort") .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} protocol: TCP selector: app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml index 148a687c3..dd25864b2 100644 --- a/charts/postgres-operator-ui/values.yaml +++ b/charts/postgres-operator-ui/values.yaml @@ -42,6 +42,9 @@ envs: service: type: "ClusterIP" port: "8080" + # If the type of the service is NodePort a port can be specified using the nodePort field + # If the nodePort field is not specified, or if it has no value, then a random port is used + # notePort: 32521 # configure UI ingress. If needed: "enabled: true" ingress: From cc635a02e328b14daf3817ae2e413cbfb408230f Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 29 Apr 2020 10:07:14 +0200 Subject: [PATCH 091/104] Lazy upgrade of the Spilo image (#859) * initial implementation * describe forcing the rolling upgrade * make parameter name more descriptive * add missing pieces * address review * address review * fix bug in e2e tests * fix cluster name label in e2e test * raise test timeout * load spilo test image * use available spilo image * delete replica pod for lazy update test * fix e2e * fix e2e with a vengeance * lets wait for another 30m * print pod name in error msg * print pod name in error msg 2 * raise timeout, comment other tests * subsequent updates of config * add comma * fix e2e test * run unit tests before e2e * remove conflicting dependency * Revert "remove conflicting dependency" This reverts commit 65fc09054bf073755b15a3b5fa389e07d4f6f025. * improve cdp build * dont run unit before e2e tests * Revert "improve cdp build" This reverts commit e2a8fa12aa72b47e83901498a849f9eeed36eecc. Co-authored-by: Sergey Dudoladov Co-authored-by: Felix Kunde --- Makefile | 2 +- .../crds/operatorconfigurations.yaml | 2 + charts/postgres-operator/values-crd.yaml | 2 + charts/postgres-operator/values.yaml | 2 + docs/administrator.md | 11 +++ docs/reference/operator_parameters.md | 4 + e2e/Makefile | 2 +- e2e/tests/test_e2e.py | 80 ++++++++++++++++--- go.mod | 16 ++-- go.sum | 67 ++++++++++++---- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 2 + ...gresql-operator-default-configuration.yaml | 5 +- pkg/apis/acid.zalan.do/v1/crds.go | 3 + .../v1/operator_configuration_type.go | 1 + pkg/cluster/cluster.go | 15 +++- pkg/cluster/sync.go | 45 +++++++++-- pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 1 + 19 files changed, 220 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index dc1c790fe..69dd240db 100644 --- a/Makefile +++ b/Makefile @@ -97,4 +97,4 @@ test: GO111MODULE=on go test ./... e2e: docker # build operator image to be tested - cd e2e; make tools test clean + cd e2e; make tools e2etest clean diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index aeb3d2150..71260bdb6 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -62,6 +62,8 @@ spec: type: string enable_crd_validation: type: boolean + enable_lazy_spilo_upgrade: + type: boolean enable_shm_volume: type: boolean etcd_host: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 41b8dbefb..e6428c478 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -19,6 +19,8 @@ configTarget: "OperatorConfigurationCRD" configGeneral: # choose if deployment creates/updates CRDs with OpenAPIV3Validation enable_crd_validation: true + # update only the statefulsets without immediately doing the rolling update + enable_lazy_spilo_upgrade: false # start any new database pod without limitations on shm memory enable_shm_volume: true # etcd connection string for Patroni. Empty uses K8s-native DCS. diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 7f140f1de..e5cdcee47 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -19,6 +19,8 @@ configTarget: "ConfigMap" configGeneral: # choose if deployment creates/updates CRDs with OpenAPIV3Validation enable_crd_validation: "true" + # update only the statefulsets without immediately doing the rolling update + enable_lazy_spilo_upgrade: "false" # start any new database pod without limitations on shm memory enable_shm_volume: "true" # etcd connection string for Patroni. Empty uses K8s-native DCS. diff --git a/docs/administrator.md b/docs/administrator.md index ed96b7d35..45a328d38 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -458,6 +458,17 @@ from numerous escape characters in the latter log entry, view it in CLI with `PodTemplate` used by the operator is yet to be updated with the default values used internally in K8s. +The operator also support lazy updates of the Spilo image. That means the pod +template of a PG cluster's stateful set is updated immediately with the new +image, but no rolling update follows. This feature saves you a switchover - and +hence downtime - when you know pods are re-started later anyway, for instance +due to the node rotation. To force a rolling update, disable this mode by +setting the `enable_lazy_spilo_upgrade` to `false` in the operator configuration +and restart the operator pod. With the standard eager rolling updates the +operator checks during Sync all pods run images specified in their respective +statefulsets. The operator triggers a rolling upgrade for PG clusters that +violate this condition. + ## Logical backups The operator can manage K8s cron jobs to run logical backups of Postgres diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index d98a270e6..af6e93d25 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -75,6 +75,10 @@ Those are top-level keys, containing both leaf keys and groups. [OpenAPI v3 schema validation](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#validation) The default is `true`. +* **enable_lazy_spilo_upgrade** + Instruct operator to update only the statefulsets with the new image without immediately doing the rolling update. The assumption is pods will be re-started later with the new image, for example due to the node rotation. + The default is `false`. + * **etcd_host** Etcd connection string for Patroni defined as `host:port`. Not required when Patroni native Kubernetes support is used. The default is empty (use diff --git a/e2e/Makefile b/e2e/Makefile index 77059f3eb..70a2ff4e9 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -44,5 +44,5 @@ tools: docker # install pinned version of 'kind' GO111MODULE=on go get sigs.k8s.io/kind@v0.5.1 -test: +e2etest: ./run.sh diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index f46c0577e..aa6f1205d 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -142,15 +142,6 @@ class EndToEndTestCase(unittest.TestCase): }) k8s.wait_for_pods_to_stop(pod_selector) - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': True, - } - }) - k8s.wait_for_pod_start(pod_selector) except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) raise @@ -204,6 +195,66 @@ class EndToEndTestCase(unittest.TestCase): self.assertEqual(repl_svc_type, 'ClusterIP', "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_lazy_spilo_upgrade(self): + ''' + Test lazy upgrade for the Spilo image: operator changes a stateful set but lets pods run with the old image + until they are recreated for reasons other than operator's activity. That works because the operator configures + stateful sets to use "onDelete" pod update policy. + + The test covers: + 1) enabling lazy upgrade in existing operator deployment + 2) forcing the normal rolling upgrade by changing the operator configmap and restarting its pod + ''' + + k8s = self.k8s + + # update docker image in config and enable the lazy upgrade + conf_image = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" + patch_lazy_spilo_upgrade = { + "data": { + "docker_image": conf_image, + "enable_lazy_spilo_upgrade": "true" + } + } + k8s.update_config(patch_lazy_spilo_upgrade) + + pod0 = 'acid-minimal-cluster-0' + pod1 = 'acid-minimal-cluster-1' + + # restart the pod to get a container with the new image + k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') + time.sleep(60) + + # lazy update works if the restarted pod and older pods run different Spilo versions + new_image = k8s.get_effective_pod_image(pod0) + old_image = k8s.get_effective_pod_image(pod1) + self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image)) + + # sanity check + assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) + self.assertEqual(new_image, conf_image, assert_msg) + + # clean up + unpatch_lazy_spilo_upgrade = { + "data": { + "enable_lazy_spilo_upgrade": "false", + } + } + k8s.update_config(unpatch_lazy_spilo_upgrade) + + # at this point operator will complete the normal rolling upgrade + # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works + + # XXX there is no easy way to wait until the end of Sync() + time.sleep(60) + + image0 = k8s.get_effective_pod_image(pod0) + image1 = k8s.get_effective_pod_image(pod1) + + assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) + self.assertEqual(image0, image1, assert_msg) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_logical_backup_cron_job(self): ''' @@ -594,7 +645,7 @@ class K8s: def wait_for_operator_pod_start(self): self. wait_for_pod_start("name=postgres-operator") - # HACK operator must register CRD / add existing PG clusters after pod start up + # HACK operator must register CRD and/or Sync existing PG clusters after start up # for local execution ~ 10 seconds suffices time.sleep(60) @@ -724,6 +775,15 @@ class K8s: stdout=subprocess.PIPE, stderr=subprocess.PIPE) + def get_effective_pod_image(self, pod_name, namespace='default'): + ''' + Get the Spilo image pod currently uses. In case of lazy rolling updates + it may differ from the one specified in the stateful set. + ''' + pod = self.api.core_v1.list_namespaced_pod( + namespace, label_selector="statefulset.kubernetes.io/pod-name=" + pod_name) + return pod.items[0].spec.containers[0].image + if __name__ == '__main__': unittest.main() diff --git a/go.mod b/go.mod index 8b9f55509..dc6389a1c 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,20 @@ go 1.14 require ( github.com/aws/aws-sdk-go v1.29.33 + github.com/emicklei/go-restful v2.9.6+incompatible // indirect + github.com/evanphx/json-patch v4.5.0+incompatible // indirect + github.com/googleapis/gnostic v0.3.0 // indirect github.com/lib/pq v1.3.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a github.com/sirupsen/logrus v1.5.0 github.com/stretchr/testify v1.4.0 - golang.org/x/tools v0.0.0-20200326210457-5d86d385bf88 // indirect + golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b // indirect gopkg.in/yaml.v2 v2.2.8 - k8s.io/api v0.18.0 - k8s.io/apiextensions-apiserver v0.18.0 - k8s.io/apimachinery v0.18.0 - k8s.io/client-go v0.18.0 - k8s.io/code-generator v0.18.0 + k8s.io/api v0.18.2 + k8s.io/apiextensions-apiserver v0.18.2 + k8s.io/apimachinery v0.18.2 + k8s.io/client-go v11.0.0+incompatible + k8s.io/code-generator v0.18.2 + sigs.k8s.io/kind v0.5.1 // indirect ) diff --git a/go.sum b/go.sum index e8607922b..22be07f7a 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -62,10 +63,14 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkg github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w= +github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -145,6 +150,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= @@ -155,10 +161,13 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= +github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -174,10 +183,12 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -204,6 +215,7 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= @@ -216,6 +228,7 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -228,9 +241,11 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -238,7 +253,9 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= @@ -257,6 +274,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= @@ -264,7 +282,9 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -277,6 +297,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -289,7 +310,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -361,6 +382,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190621203818-d432491b9138/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -388,8 +410,8 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200326210457-5d86d385bf88 h1:F7fM2kxXfuWw820fa+MMCCLH6hmYe+jtLnZpwoiLK4Q= -golang.org/x/tools v0.0.0-20200326210457-5d86d385bf88/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b h1:zSzQJAznWxAh9fZxiPy2FZo+ZZEYoYFYYDYdOrU7AaM= +golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -431,31 +453,46 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.18.0 h1:lwYk8Vt7rsVTwjRU6pzEsa9YNhThbmbocQlKvNBB4EQ= -k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= -k8s.io/apiextensions-apiserver v0.18.0 h1:HN4/P8vpGZFvB5SOMuPPH2Wt9Y/ryX+KRvIyAkchu1Q= -k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= -k8s.io/apimachinery v0.18.0 h1:fuPfYpk3cs1Okp/515pAf0dNhL66+8zk8RLbSX+EgAE= -k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= -k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= -k8s.io/client-go v0.18.0 h1:yqKw4cTUQraZK3fcVCMeSa+lqKwcjZ5wtcOIPnxQno4= -k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8= -k8s.io/code-generator v0.18.0 h1:0xIRWzym+qMgVpGmLESDeMfz/orwgxwxFFAo1xfGNtQ= -k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= -k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= +k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/api v0.18.2 h1:wG5g5ZmSVgm5B+eHMIbI9EGATS2L8Z72rda19RIEgY8= +k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= +k8s.io/apiextensions-apiserver v0.18.2 h1:I4v3/jAuQC+89L3Z7dDgAiN4EOjN6sbm6iBqQwHTah8= +k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= +k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apimachinery v0.18.2 h1:44CmtbmkzVDAhCpRVSiP2R5PPrC2RtlIv/MoB8xpdRA= +k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= +k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= +k8s.io/client-go v0.18.2 h1:aLB0iaD4nmwh7arT2wIn+lMnAq7OswjaejkQ8p9bBYE= +k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU= +k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= +k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/code-generator v0.18.2 h1:C1Nn2JiMf244CvBDKVPX0W2mZFJkVBg54T8OV7/Imso= +k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= +k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c h1:/KUFqjjqAcY4Us6luF5RDNZ16KJtb49HfR3ZHB9qYXM= k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= +sigs.k8s.io/kind v0.5.1 h1:BYnHEJ9DC+0Yjlyyehqd3xnKtEmFdLKU8QxqOqvQzdw= +sigs.k8s.io/kind v0.5.1/go.mod h1:L+Kcoo83/D1+ryU5P2VFbvYm0oqbkJn9zTZq0KNxW68= +sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296 h1:iQaIG5Dq+3qSiaFrJ/l/0MjjxKmdwyVNpKRYJwUe/+0= +sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296/go.mod h1:ztX4zYc/QIww3gSripwF7TBOarBTm5BvyAMem0kCzOE= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index cbc55b446..8719e76a1 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -34,6 +34,7 @@ data: # enable_crd_validation: "true" # enable_database_access: "true" # enable_init_containers: "true" + # enable_lazy_spilo_upgrade: "false" enable_master_load_balancer: "false" # enable_pod_antiaffinity: "false" # enable_pod_disruption_budget: "true" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 53097b96c..2d3e614b9 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -38,6 +38,8 @@ spec: type: string enable_crd_validation: type: boolean + enable_lazy_spilo_upgrade: + type: boolean enable_shm_volume: type: boolean etcd_host: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index b5765a0f4..78312b684 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -3,11 +3,12 @@ kind: OperatorConfiguration metadata: name: postgresql-operator-default-configuration configuration: + docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 # enable_crd_validation: true + # enable_lazy_spilo_upgrade: false + # enable_shm_volume: true etcd_host: "" # kubernetes_use_configmaps: false - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 - # enable_shm_volume: true max_instances: -1 min_instances: -1 resync_period: 30m diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index bcb35c56c..36b0904d7 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -761,6 +761,9 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "enable_crd_validation": { Type: "boolean", }, + "enable_lazy_spilo_upgrade": { + Type: "boolean", + }, "enable_shm_volume": { Type: "boolean", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index c377a294b..4a0abf3ca 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -183,6 +183,7 @@ type OperatorLogicalBackupConfiguration struct { // OperatorConfigurationData defines the operation config type OperatorConfigurationData struct { EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` + EnableLazySpiloUpgrade bool `json:"enable_lazy_spilo_upgrade,omitempty"` EtcdHost string `json:"etcd_host,omitempty"` KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` DockerImage string `json:"docker_image,omitempty"` diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 244916074..2e04cb137 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -470,6 +470,14 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa } } + // lazy Spilo update: modify the image in the statefulset itself but let its pods run with the old image + // until they are re-created for other reasons, for example node rotation + if c.OpConfig.EnableLazySpiloUpgrade && !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Containers[0].Image, statefulSet.Spec.Template.Spec.Containers[0].Image) { + needsReplace = true + needsRollUpdate = false + reasons = append(reasons, "lazy Spilo update: new statefulset's pod image doesn't match the current one") + } + if needsRollUpdate || needsReplace { match = false } @@ -501,8 +509,6 @@ func (c *Cluster) compareContainers(description string, setA, setB []v1.Containe checks := []containerCheck{ newCheck("new statefulset %s's %s (index %d) name doesn't match the current one", func(a, b v1.Container) bool { return a.Name != b.Name }), - newCheck("new statefulset %s's %s (index %d) image doesn't match the current one", - func(a, b v1.Container) bool { return a.Image != b.Image }), newCheck("new statefulset %s's %s (index %d) ports don't match the current one", func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Ports, b.Ports) }), newCheck("new statefulset %s's %s (index %d) resources don't match the current ones", @@ -513,6 +519,11 @@ func (c *Cluster) compareContainers(description string, setA, setB []v1.Containe func(a, b v1.Container) bool { return !reflect.DeepEqual(a.EnvFrom, b.EnvFrom) }), } + if !c.OpConfig.EnableLazySpiloUpgrade { + checks = append(checks, newCheck("new statefulset %s's %s (index %d) image doesn't match the current one", + func(a, b v1.Container) bool { return a.Image != b.Image })) + } + for index, containerA := range setA { containerB := setB[index] for _, check := range checks { diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index a9a2b5177..0eb02631c 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -4,17 +4,17 @@ import ( "context" "fmt" - batchv1beta1 "k8s.io/api/batch/v1beta1" - v1 "k8s.io/api/core/v1" - policybeta1 "k8s.io/api/policy/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/volumes" + appsv1 "k8s.io/api/apps/v1" + batchv1beta1 "k8s.io/api/batch/v1beta1" + v1 "k8s.io/api/core/v1" + policybeta1 "k8s.io/api/policy/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // Sync syncs the cluster, making sure the actual Kubernetes objects correspond to what is defined in the manifest. @@ -252,6 +252,28 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error { return nil } +func (c *Cluster) mustUpdatePodsAfterLazyUpdate(desiredSset *appsv1.StatefulSet) (bool, error) { + + pods, err := c.listPods() + if err != nil { + return false, fmt.Errorf("could not list pods of the statefulset: %v", err) + } + + for _, pod := range pods { + + effectivePodImage := pod.Spec.Containers[0].Image + ssImage := desiredSset.Spec.Template.Spec.Containers[0].Image + + if ssImage != effectivePodImage { + c.logger.Infof("not all pods were re-started when the lazy upgrade was enabled; forcing the rolling upgrade now") + return true, nil + } + + } + + return false, nil +} + func (c *Cluster) syncStatefulSet() error { var ( podsRollingUpdateRequired bool @@ -330,6 +352,19 @@ func (c *Cluster) syncStatefulSet() error { } } } + + if !podsRollingUpdateRequired && !c.OpConfig.EnableLazySpiloUpgrade { + // even if desired and actual statefulsets match + // there still may be not up-to-date pods on condition + // (a) the lazy update was just disabled + // and + // (b) some of the pods were not restarted when the lazy update was still in place + podsRollingUpdateRequired, err = c.mustUpdatePodsAfterLazyUpdate(desiredSS) + if err != nil { + return fmt.Errorf("could not list pods of the statefulset: %v", err) + } + } + } // Apply special PostgreSQL parameters that can only be set via the Patroni API. diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index c1756604b..f66eafb1e 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -34,6 +34,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // general config result.EnableCRDValidation = fromCRD.EnableCRDValidation + result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps result.DockerImage = fromCRD.DockerImage diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 2bdeb8e1f..37ba947d6 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -157,6 +157,7 @@ type Config struct { ProtectedRoles []string `name:"protected_role_names" default:"admin"` PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` + EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"` } // MustMarshal marshals the config or panics From d76203b3f9c690f64aa5c3b3f395c78935de96e8 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 29 Apr 2020 10:56:06 +0200 Subject: [PATCH 092/104] Bootstrapped databases with best practice role setup (#843) * PreparedDatabases with default role setup * merge changes from master * include preparedDatabases spec check when syncing databases * create a default preparedDB if not specified * add more default privileges for schemas * use empty brackets block for undefined objects * cover more default privilege scenarios and always define admin role * add DefaultUsers flag * support extensions and defaultUsers for preparedDatabases * remove exact version in deployment manifest * enable CRD validation for new field * update generated code * reflect code review * fix typo in SQL command * add documentation for preparedDatabases feature + minor changes * some datname should stay * add unit tests * reflect some feedback * init users for preparedDatabases also on update * only change DB default privileges on creation * add one more section in user docs * one more sentence --- .../postgres-operator/crds/postgresqls.yaml | 20 ++ docs/user.md | 169 ++++++++++++- manifests/complete-postgres-manifest.yaml | 11 + manifests/minimal-postgres-manifest.yaml | 2 + manifests/postgresql.crd.yaml | 20 ++ pkg/apis/acid.zalan.do/v1/crds.go | 37 +++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 50 ++-- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 58 +++++ pkg/cluster/cluster.go | 118 ++++++++- pkg/cluster/cluster_test.go | 89 ++++++- pkg/cluster/database.go | 225 ++++++++++++++++-- pkg/cluster/k8sres.go | 7 + pkg/cluster/sync.go | 138 ++++++++++- pkg/controller/types.go | 3 +- pkg/spec/types.go | 3 + pkg/util/constants/roles.go | 4 + pkg/util/users/users.go | 38 ++- 17 files changed, 927 insertions(+), 65 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 78850ee3b..fdbcf8304 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -273,6 +273,26 @@ spec: type: object additionalProperties: type: string + preparedDatabases: + type: object + additionalProperties: + type: object + properties: + defaultUsers: + type: boolean + extensions: + type: object + additionalProperties: + type: string + schemas: + type: object + additionalProperties: + type: object + properties: + defaultUsers: + type: boolean + defaultRoles: + type: boolean replicaLoadBalancer: # deprecated type: boolean resources: diff --git a/docs/user.md b/docs/user.md index d7e6add0a..2d9f2be6a 100644 --- a/docs/user.md +++ b/docs/user.md @@ -94,7 +94,10 @@ created on every cluster managed by the operator. * `teams API roles`: automatically create users for every member of the team owning the database cluster. -In the next sections, we will cover those use cases in more details. +In the next sections, we will cover those use cases in more details. Note, that +the Postgres Operator can also create databases with pre-defined owner, reader +and writer roles which saves you the manual setup. Read more in the next +chapter. ### Manifest roles @@ -216,6 +219,166 @@ to choose superusers, group roles, [PAM configuration](https://github.com/CyberD etc. An OAuth2 token can be passed to the Teams API via a secret. The name for this secret is configurable with the `oauth_token_secret_name` parameter. +## Prepared databases with roles and default privileges + +The `users` section in the manifests only allows for creating database roles +with global privileges. Fine-grained data access control or role membership can +not be defined and must be set up by the user in the database. But, the Postgres +Operator offers a separate section to specify `preparedDatabases` that will be +created with pre-defined owner, reader and writer roles for each individual +database and, optionally, for each database schema, too. `preparedDatabases` +also enable users to specify PostgreSQL extensions that shall be created in a +given database schema. + +### Default database and schema + +A prepared database is already created by adding an empty `preparedDatabases` +section to the manifest. The database will then be called like the Postgres +cluster manifest (`-` are replaced with `_`) and will also contain a schema +called `data`. + +```yaml +spec: + preparedDatabases: {} +``` + +### Default NOLOGIN roles + +Given an example with a specified database and schema: + +```yaml +spec: + preparedDatabases: + foo: + schemas: + bar: {} +``` + +Postgres Operator will create the following NOLOGIN roles: + +| Role name | Member of | Admin | +| -------------- | -------------- | ------------- | +| foo_owner | | admin | +| foo_reader | | foo_owner | +| foo_writer | foo_reader | foo_owner | +| foo_bar_owner | | foo_owner | +| foo_bar_reader | | foo_bar_owner | +| foo_bar_writer | foo_bar_reader | foo_bar_owner | + +The `_owner` role is the database owner and should be used when creating +new database objects. All members of the `admin` role, e.g. teams API roles, can +become the owner with the `SET ROLE` command. [Default privileges](https://www.postgresql.org/docs/12/sql-alterdefaultprivileges.html) +are configured for the owner role so that the `_reader` role +automatically gets read-access (SELECT) to new tables and sequences and the +`_writer` receives write-access (INSERT, UPDATE, DELETE on tables, +USAGE and UPDATE on sequences). Both get USAGE on types and EXECUTE on +functions. + +The same principle applies for database schemas which are owned by the +`__owner` role. `__reader` is read-only, +`__writer` has write access and inherit reading from the reader +role. Note, that the `_*` roles have access incl. default privileges on +all schemas, too. If you don't need the dedicated schema roles - i.e. you only +use one schema - you can disable the creation like this: + +```yaml +spec: + preparedDatabases: + foo: + schemas: + bar: + defaultRoles: false +``` + +Then, the schemas are owned by the database owner, too. + +### Default LOGIN roles + +The roles described in the previous paragraph can be granted to LOGIN roles from +the `users` section in the manifest. Optionally, the Postgres Operator can also +create default LOGIN roles for the database an each schema individually. These +roles will get the `_user` suffix and they inherit all rights from their NOLOGIN +counterparts. + +| Role name | Member of | Admin | +| ------------------- | -------------- | ------------- | +| foo_owner_user | foo_owner | admin | +| foo_reader_user | foo_reader | foo_owner | +| foo_writer_user | foo_writer | foo_owner | +| foo_bar_owner_user | foo_bar_owner | foo_owner | +| foo_bar_reader_user | foo_bar_reader | foo_bar_owner | +| foo_bar_writer_user | foo_bar_writer | foo_bar_owner | + +These default users are enabled in the manifest with the `defaultUsers` flag: + +```yaml +spec: + preparedDatabases: + foo: + defaultUsers: true + schemas: + bar: + defaultUsers: true +``` + +### Database extensions + +Prepared databases also allow for creating Postgres extensions. They will be +created by the database owner in the specified schema. + +```yaml +spec: + preparedDatabases: + foo: + extensions: + pg_partman: public + postgis: data +``` + +Some extensions require SUPERUSER rights on creation unless they are not +whitelisted by the [pgextwlist](https://github.com/dimitri/pgextwlist) +extension, that is shipped with the Spilo image. To see which extensions are +on the list check the `extwlist.extension` parameter in the postgresql.conf +file. + +```bash +SHOW extwlist.extensions; +``` + +Make sure that `pgextlist` is also listed under `shared_preload_libraries` in +the PostgreSQL configuration. Then the database owner should be able to create +the extension specified in the manifest. + +### From `databases` to `preparedDatabases` + +If you wish to create the role setup described above for databases listed under +the `databases` key, you have to make sure that the owner role follows the +`_owner` naming convention of `preparedDatabases`. As roles are synced +first, this can be done with one edit: + +```yaml +# before +spec: + databases: + foo: db_owner + +# after +spec: + databases: + foo: foo_owner + preparedDatabases: + foo: + schemas: + my_existing_schema: {} +``` + +Adding existing database schemas to the manifest to create roles for them as +well is up the user and not done by the operator. Remember that if you don't +specify any schema a new database schema called `data` will be created. When +everything got synced (roles, schemas, extensions), you are free to remove the +database from the `databases` section. Note, that the operator does not delete +database objects or revoke privileges when removed from the manifest. + ## Resource definition The compute resources to be used for the Postgres containers in the pods can be @@ -586,8 +749,8 @@ don't know the value, use `103` which is the GID from the default spilo image OpenShift allocates the users and groups dynamically (based on scc), and their range is different in every namespace. Due to this dynamic behaviour, it's not trivial to know at deploy time the uid/gid of the user in the cluster. -Therefore, instead of using a global `spilo_fsgroup` setting, use the `spiloFSGroup` field -per Postgres cluster. +Therefore, instead of using a global `spilo_fsgroup` setting, use the +`spiloFSGroup` field per Postgres cluster. Upload the cert as a kubernetes secret: ```sh diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index f031a5d5b..d436695e8 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -21,6 +21,17 @@ spec: - 127.0.0.1/32 databases: foo: zalando + preparedDatabases: + bar: + defaultUsers: true + extensions: + pg_partman: public + pgcrypto: public + schemas: + data: {} + history: + defaultRoles: true + defaultUsers: false postgresql: version: "12" parameters: # Expert section diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml index af0add8e6..4dd6b7ee4 100644 --- a/manifests/minimal-postgres-manifest.yaml +++ b/manifests/minimal-postgres-manifest.yaml @@ -15,5 +15,7 @@ spec: foo_user: [] # role for application foo databases: foo: zalando # dbname: owner + preparedDatabases: + bar: {} postgresql: version: "12" diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 1ee6a1ae5..e62204c40 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -237,6 +237,26 @@ spec: type: object additionalProperties: type: string + preparedDatabases: + type: object + additionalProperties: + type: object + properties: + defaultUsers: + type: boolean + extensions: + type: object + additionalProperties: + type: string + schemas: + type: object + additionalProperties: + type: object + properties: + defaultUsers: + type: boolean + defaultRoles: + type: boolean replicaLoadBalancer: # deprecated type: boolean resources: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 36b0904d7..35037ec3c 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -421,6 +421,43 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "preparedDatabases": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "defaultUsers": { + Type: "boolean", + }, + "extensions": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "schemas": { + Type: "object", + AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "defaultUsers": { + Type: "boolean", + }, + "defaultRoles": { + Type: "boolean", + }, + }, + }, + }, + }, + }, + }, + }, + }, "replicaLoadBalancer": { Type: "boolean", Description: "Deprecated", diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index e36009208..5df82e947 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -50,24 +50,25 @@ type PostgresSpec struct { // load balancers' source ranges are the same for master and replica services AllowedSourceRanges []string `json:"allowedSourceRanges"` - NumberOfInstances int32 `json:"numberOfInstances"` - Users map[string]UserFlags `json:"users"` - MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` - Clone CloneDescription `json:"clone"` - ClusterName string `json:"-"` - Databases map[string]string `json:"databases,omitempty"` - Tolerations []v1.Toleration `json:"tolerations,omitempty"` - Sidecars []Sidecar `json:"sidecars,omitempty"` - InitContainers []v1.Container `json:"initContainers,omitempty"` - PodPriorityClassName string `json:"podPriorityClassName,omitempty"` - ShmVolume *bool `json:"enableShmVolume,omitempty"` - EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"` - LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` - StandbyCluster *StandbyDescription `json:"standby"` - PodAnnotations map[string]string `json:"podAnnotations"` - ServiceAnnotations map[string]string `json:"serviceAnnotations"` - TLS *TLSDescription `json:"tls"` - AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` + NumberOfInstances int32 `json:"numberOfInstances"` + Users map[string]UserFlags `json:"users"` + MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` + Clone CloneDescription `json:"clone"` + ClusterName string `json:"-"` + Databases map[string]string `json:"databases,omitempty"` + PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"` + Tolerations []v1.Toleration `json:"tolerations,omitempty"` + Sidecars []Sidecar `json:"sidecars,omitempty"` + InitContainers []v1.Container `json:"initContainers,omitempty"` + PodPriorityClassName string `json:"podPriorityClassName,omitempty"` + ShmVolume *bool `json:"enableShmVolume,omitempty"` + EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"` + LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` + StandbyCluster *StandbyDescription `json:"standby"` + PodAnnotations map[string]string `json:"podAnnotations"` + ServiceAnnotations map[string]string `json:"serviceAnnotations"` + TLS *TLSDescription `json:"tls"` + AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` // deprecated json tags InitContainersOld []v1.Container `json:"init_containers,omitempty"` @@ -84,6 +85,19 @@ type PostgresqlList struct { Items []Postgresql `json:"items"` } +// PreparedDatabase describes elements to be bootstrapped +type PreparedDatabase struct { + PreparedSchemas map[string]PreparedSchema `json:"schemas,omitempty"` + DefaultUsers bool `json:"defaultUsers,omitempty" defaults:"false"` + Extensions map[string]string `json:"extensions,omitempty"` +} + +// PreparedSchema describes elements to be bootstrapped per schema +type PreparedSchema struct { + DefaultRoles *bool `json:"defaultRoles,omitempty" defaults:"true"` + DefaultUsers bool `json:"defaultUsers,omitempty" defaults:"false"` +} + // MaintenanceWindow describes the time window when the operator is allowed to do maintenance on a cluster. type MaintenanceWindow struct { Everyday bool diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index e2e1d5bd1..5b4d6cdcd 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -570,6 +570,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*out)[key] = val } } + if in.PreparedDatabases != nil { + in, out := &in.PreparedDatabases, &out.PreparedDatabases + *out = make(map[string]PreparedDatabase, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]corev1.Toleration, len(*in)) @@ -763,6 +770,57 @@ func (in *PostgresqlParam) DeepCopy() *PostgresqlParam { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PreparedDatabase) DeepCopyInto(out *PreparedDatabase) { + *out = *in + if in.PreparedSchemas != nil { + in, out := &in.PreparedSchemas, &out.PreparedSchemas + *out = make(map[string]PreparedSchema, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.Extensions != nil { + in, out := &in.Extensions, &out.Extensions + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreparedDatabase. +func (in *PreparedDatabase) DeepCopy() *PreparedDatabase { + if in == nil { + return nil + } + out := new(PreparedDatabase) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PreparedSchema) DeepCopyInto(out *PreparedSchema) { + *out = *in + if in.DefaultRoles != nil { + in, out := &in.DefaultRoles, &out.DefaultRoles + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreparedSchema. +func (in *PreparedSchema) DeepCopy() *PreparedSchema { + if in == nil { + return nil + } + out := new(PreparedSchema) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceDescription) DeepCopyInto(out *ResourceDescription) { *out = *in diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 2e04cb137..387107540 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -9,6 +9,7 @@ import ( "fmt" "reflect" "regexp" + "strings" "sync" "time" @@ -227,6 +228,10 @@ func (c *Cluster) initUsers() error { return fmt.Errorf("could not init infrastructure roles: %v", err) } + if err := c.initPreparedDatabaseRoles(); err != nil { + return fmt.Errorf("could not init default users: %v", err) + } + if err := c.initRobotUsers(); err != nil { return fmt.Errorf("could not init robot users: %v", err) } @@ -343,6 +348,9 @@ func (c *Cluster) Create() error { if err = c.syncDatabases(); err != nil { return fmt.Errorf("could not sync databases: %v", err) } + if err = c.syncPreparedDatabases(); err != nil { + return fmt.Errorf("could not sync prepared databases: %v", err) + } c.logger.Infof("databases have been successfully created") } @@ -649,7 +657,8 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { // connection pooler needs one system user created, which is done in // initUsers. Check if it needs to be called. - sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) + sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) && + reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) needConnectionPooler := c.needConnectionPoolerWorker(&newSpec.Spec) if !sameUsers || needConnectionPooler { c.logger.Debugf("syncing secrets") @@ -766,19 +775,28 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { c.logger.Errorf("could not sync roles: %v", err) updateFailed = true } - if !reflect.DeepEqual(oldSpec.Spec.Databases, newSpec.Spec.Databases) { + if !reflect.DeepEqual(oldSpec.Spec.Databases, newSpec.Spec.Databases) || + !reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) { c.logger.Infof("syncing databases") if err := c.syncDatabases(); err != nil { c.logger.Errorf("could not sync databases: %v", err) updateFailed = true } } + if !reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) { + c.logger.Infof("syncing prepared databases") + if err := c.syncPreparedDatabases(); err != nil { + c.logger.Errorf("could not sync prepared databases: %v", err) + updateFailed = true + } + } } // sync connection pooler if _, err := c.syncConnectionPooler(oldSpec, newSpec, c.installLookupFunction); err != nil { - return fmt.Errorf("could not sync connection pooler: %v", err) + c.logger.Errorf("could not sync connection pooler: %v", err) + updateFailed = true } return nil @@ -949,6 +967,100 @@ func (c *Cluster) initSystemUsers() { } } +func (c *Cluster) initPreparedDatabaseRoles() error { + + if c.Spec.PreparedDatabases != nil && len(c.Spec.PreparedDatabases) == 0 { // TODO: add option to disable creating such a default DB + c.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{strings.Replace(c.Name, "-", "_", -1): {}} + } + + // create maps with default roles/users as keys and their membership as values + defaultRoles := map[string]string{ + constants.OwnerRoleNameSuffix: "", + constants.ReaderRoleNameSuffix: "", + constants.WriterRoleNameSuffix: constants.ReaderRoleNameSuffix, + } + defaultUsers := map[string]string{ + constants.OwnerRoleNameSuffix + constants.UserRoleNameSuffix: constants.OwnerRoleNameSuffix, + constants.ReaderRoleNameSuffix + constants.UserRoleNameSuffix: constants.ReaderRoleNameSuffix, + constants.WriterRoleNameSuffix + constants.UserRoleNameSuffix: constants.WriterRoleNameSuffix, + } + + for preparedDbName, preparedDB := range c.Spec.PreparedDatabases { + // default roles per database + if err := c.initDefaultRoles(defaultRoles, "admin", preparedDbName); err != nil { + return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err) + } + if preparedDB.DefaultUsers { + if err := c.initDefaultRoles(defaultUsers, "admin", preparedDbName); err != nil { + return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err) + } + } + + // default roles per database schema + preparedSchemas := preparedDB.PreparedSchemas + if len(preparedDB.PreparedSchemas) == 0 { + preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}} + } + for preparedSchemaName, preparedSchema := range preparedSchemas { + if preparedSchema.DefaultRoles == nil || *preparedSchema.DefaultRoles { + if err := c.initDefaultRoles(defaultRoles, + preparedDbName+constants.OwnerRoleNameSuffix, + preparedDbName+"_"+preparedSchemaName); err != nil { + return fmt.Errorf("could not initialize default roles for database schema %s: %v", preparedSchemaName, err) + } + if preparedSchema.DefaultUsers { + if err := c.initDefaultRoles(defaultUsers, + preparedDbName+constants.OwnerRoleNameSuffix, + preparedDbName+"_"+preparedSchemaName); err != nil { + return fmt.Errorf("could not initialize default users for database schema %s: %v", preparedSchemaName, err) + } + } + } + } + } + return nil +} + +func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix string) error { + + for defaultRole, inherits := range defaultRoles { + + roleName := prefix + defaultRole + + flags := []string{constants.RoleFlagNoLogin} + if defaultRole[len(defaultRole)-5:] == constants.UserRoleNameSuffix { + flags = []string{constants.RoleFlagLogin} + } + + memberOf := make([]string, 0) + if inherits != "" { + memberOf = append(memberOf, prefix+inherits) + } + + adminRole := "" + if strings.Contains(defaultRole, constants.OwnerRoleNameSuffix) { + adminRole = admin + } else { + adminRole = prefix + constants.OwnerRoleNameSuffix + } + + newRole := spec.PgUser{ + Origin: spec.RoleOriginBootstrap, + Name: roleName, + Password: util.RandomPassword(constants.PasswordLength), + Flags: flags, + MemberOf: memberOf, + AdminRole: adminRole, + } + if currentRole, present := c.pgUsers[roleName]; present { + c.pgUsers[roleName] = c.resolveNameConflict(¤tRole, &newRole) + } else { + c.pgUsers[roleName] = newRole + } + } + return nil +} + func (c *Cluster) initRobotUsers() error { for username, userFlags := range c.Spec.Users { if !isValidUsername(username) { diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 84ec04e3e..539038bff 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -13,6 +13,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/teams" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" ) @@ -35,7 +36,7 @@ var cl = New( }, }, k8sutil.NewMockKubernetesClient(), - acidv1.Postgresql{}, + acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test"}}, logger, eventRecorder, ) @@ -760,3 +761,89 @@ func TestInitSystemUsers(t *testing.T) { t.Errorf("%s, System users are not allowed to be a connection pool user", testName) } } + +func TestPreparedDatabases(t *testing.T) { + testName := "TestDefaultPreparedDatabase" + + cl.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{} + cl.initPreparedDatabaseRoles() + + for _, role := range []string{"acid_test_owner", "acid_test_reader", "acid_test_writer", + "acid_test_data_owner", "acid_test_data_reader", "acid_test_data_writer"} { + if _, exist := cl.pgUsers[role]; !exist { + t.Errorf("%s, default role %q for prepared database not present", testName, role) + } + } + + testName = "TestPreparedDatabaseWithSchema" + + cl.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{ + "foo": { + DefaultUsers: true, + PreparedSchemas: map[string]acidv1.PreparedSchema{ + "bar": { + DefaultUsers: true, + }, + }, + }, + } + cl.initPreparedDatabaseRoles() + + for _, role := range []string{ + "foo_owner", "foo_reader", "foo_writer", + "foo_owner_user", "foo_reader_user", "foo_writer_user", + "foo_bar_owner", "foo_bar_reader", "foo_bar_writer", + "foo_bar_owner_user", "foo_bar_reader_user", "foo_bar_writer_user"} { + if _, exist := cl.pgUsers[role]; !exist { + t.Errorf("%s, default role %q for prepared database not present", testName, role) + } + } + + roleTests := []struct { + subTest string + role string + memberOf string + admin string + }{ + { + subTest: "Test admin role of owner", + role: "foo_owner", + memberOf: "", + admin: "admin", + }, + { + subTest: "Test writer is a member of reader", + role: "foo_writer", + memberOf: "foo_reader", + admin: "foo_owner", + }, + { + subTest: "Test reader LOGIN role", + role: "foo_reader_user", + memberOf: "foo_reader", + admin: "foo_owner", + }, + { + subTest: "Test schema owner", + role: "foo_bar_owner", + memberOf: "", + admin: "foo_owner", + }, + { + subTest: "Test schema writer LOGIN role", + role: "foo_bar_writer_user", + memberOf: "foo_bar_writer", + admin: "foo_bar_owner", + }, + } + + for _, tt := range roleTests { + user := cl.pgUsers[tt.role] + if (tt.memberOf == "" && len(user.MemberOf) > 0) || (tt.memberOf != "" && user.MemberOf[0] != tt.memberOf) { + t.Errorf("%s, incorrect membership for default role %q. Expected %q, got %q", tt.subTest, tt.role, tt.memberOf, user.MemberOf[0]) + } + if user.AdminRole != tt.admin { + t.Errorf("%s, incorrect admin role for default role %q. Expected %q, got %q", tt.subTest, tt.role, tt.admin, user.AdminRole) + } + } +} diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 28f97b5cc..75e2d2097 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -27,9 +27,35 @@ const ( WHERE a.rolname = ANY($1) ORDER BY 1;` - getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` - createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` - alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` + getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` + getSchemasSQL = `SELECT n.nspname AS dbschema FROM pg_catalog.pg_namespace n + WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' ORDER BY 1` + getExtensionsSQL = `SELECT e.extname, n.nspname FROM pg_catalog.pg_extension e + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace ORDER BY 1;` + + createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` + createDatabaseSchemaSQL = `SET ROLE TO "%s"; CREATE SCHEMA IF NOT EXISTS "%s" AUTHORIZATION "%s"` + alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` + createExtensionSQL = `CREATE EXTENSION IF NOT EXISTS "%s" SCHEMA "%s"` + alterExtensionSQL = `ALTER EXTENSION "%s" SET SCHEMA "%s"` + + globalDefaultPrivilegesSQL = `SET ROLE TO "%s"; + ALTER DEFAULT PRIVILEGES GRANT USAGE ON SCHEMAS TO "%s","%s"; + ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO "%s"; + ALTER DEFAULT PRIVILEGES GRANT SELECT ON SEQUENCES TO "%s"; + ALTER DEFAULT PRIVILEGES GRANT INSERT, UPDATE, DELETE ON TABLES TO "%s"; + ALTER DEFAULT PRIVILEGES GRANT USAGE, UPDATE ON SEQUENCES TO "%s"; + ALTER DEFAULT PRIVILEGES GRANT EXECUTE ON FUNCTIONS TO "%s","%s"; + ALTER DEFAULT PRIVILEGES GRANT USAGE ON TYPES TO "%s","%s";` + schemaDefaultPrivilegesSQL = `SET ROLE TO "%s"; + GRANT USAGE ON SCHEMA "%s" TO "%s","%s"; + ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT SELECT ON TABLES TO "%s"; + ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT SELECT ON SEQUENCES TO "%s"; + ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT INSERT, UPDATE, DELETE ON TABLES TO "%s"; + ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT USAGE, UPDATE ON SEQUENCES TO "%s"; + ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT EXECUTE ON FUNCTIONS TO "%s","%s"; + ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT USAGE ON TYPES TO "%s","%s";` + connectionPoolerLookup = ` CREATE SCHEMA IF NOT EXISTS {{.pooler_schema}}; @@ -221,43 +247,141 @@ func (c *Cluster) getDatabases() (dbs map[string]string, err error) { } // executeCreateDatabase creates new database with the given owner. -// The caller is responsible for openinging and closing the database connection. -func (c *Cluster) executeCreateDatabase(datname, owner string) error { - return c.execCreateOrAlterDatabase(datname, owner, createDatabaseSQL, +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeCreateDatabase(databaseName, owner string) error { + return c.execCreateOrAlterDatabase(databaseName, owner, createDatabaseSQL, "creating database", "create database") } -// executeCreateDatabase changes the owner of the given database. -// The caller is responsible for openinging and closing the database connection. -func (c *Cluster) executeAlterDatabaseOwner(datname string, owner string) error { - return c.execCreateOrAlterDatabase(datname, owner, alterDatabaseOwnerSQL, +// executeAlterDatabaseOwner changes the owner of the given database. +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeAlterDatabaseOwner(databaseName string, owner string) error { + return c.execCreateOrAlterDatabase(databaseName, owner, alterDatabaseOwnerSQL, "changing owner for database", "alter database owner") } -func (c *Cluster) execCreateOrAlterDatabase(datname, owner, statement, doing, operation string) error { - if !c.databaseNameOwnerValid(datname, owner) { +func (c *Cluster) execCreateOrAlterDatabase(databaseName, owner, statement, doing, operation string) error { + if !c.databaseNameOwnerValid(databaseName, owner) { return nil } - c.logger.Infof("%s %q owner %q", doing, datname, owner) - if _, err := c.pgDb.Exec(fmt.Sprintf(statement, datname, owner)); err != nil { + c.logger.Infof("%s %q owner %q", doing, databaseName, owner) + if _, err := c.pgDb.Exec(fmt.Sprintf(statement, databaseName, owner)); err != nil { return fmt.Errorf("could not execute %s: %v", operation, err) } return nil } -func (c *Cluster) databaseNameOwnerValid(datname, owner string) bool { +func (c *Cluster) databaseNameOwnerValid(databaseName, owner string) bool { if _, ok := c.pgUsers[owner]; !ok { - c.logger.Infof("skipping creation of the %q database, user %q does not exist", datname, owner) + c.logger.Infof("skipping creation of the %q database, user %q does not exist", databaseName, owner) return false } - if !databaseNameRegexp.MatchString(datname) { - c.logger.Infof("database %q has invalid name", datname) + if !databaseNameRegexp.MatchString(databaseName) { + c.logger.Infof("database %q has invalid name", databaseName) return false } return true } +// getSchemas returns the list of current database schemas +// The caller is responsible for opening and closing the database connection +func (c *Cluster) getSchemas() (schemas []string, err error) { + var ( + rows *sql.Rows + dbschemas []string + ) + + if rows, err = c.pgDb.Query(getSchemasSQL); err != nil { + return nil, fmt.Errorf("could not query database schemas: %v", err) + } + + defer func() { + if err2 := rows.Close(); err2 != nil { + if err != nil { + err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err) + } else { + err = fmt.Errorf("error when closing query cursor: %v", err2) + } + } + }() + + for rows.Next() { + var dbschema string + + if err = rows.Scan(&dbschema); err != nil { + return nil, fmt.Errorf("error when processing row: %v", err) + } + dbschemas = append(dbschemas, dbschema) + } + + return dbschemas, err +} + +// executeCreateDatabaseSchema creates new database schema with the given owner. +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeCreateDatabaseSchema(databaseName, schemaName, dbOwner string, schemaOwner string) error { + return c.execCreateDatabaseSchema(databaseName, schemaName, dbOwner, schemaOwner, createDatabaseSchemaSQL, + "creating database schema", "create database schema") +} + +func (c *Cluster) execCreateDatabaseSchema(databaseName, schemaName, dbOwner, schemaOwner, statement, doing, operation string) error { + if !c.databaseSchemaNameValid(schemaName) { + return nil + } + c.logger.Infof("%s %q owner %q", doing, schemaName, schemaOwner) + if _, err := c.pgDb.Exec(fmt.Sprintf(statement, dbOwner, schemaName, schemaOwner)); err != nil { + return fmt.Errorf("could not execute %s: %v", operation, err) + } + + // set default privileges for schema + c.execAlterSchemaDefaultPrivileges(schemaName, schemaOwner, databaseName) + if schemaOwner != dbOwner { + c.execAlterSchemaDefaultPrivileges(schemaName, dbOwner, databaseName+"_"+schemaName) + c.execAlterSchemaDefaultPrivileges(schemaName, schemaOwner, databaseName+"_"+schemaName) + } + + return nil +} + +func (c *Cluster) databaseSchemaNameValid(schemaName string) bool { + if !databaseNameRegexp.MatchString(schemaName) { + c.logger.Infof("database schema %q has invalid name", schemaName) + return false + } + return true +} + +func (c *Cluster) execAlterSchemaDefaultPrivileges(schemaName, owner, rolePrefix string) error { + if _, err := c.pgDb.Exec(fmt.Sprintf(schemaDefaultPrivilegesSQL, owner, + schemaName, rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix, // schema + schemaName, rolePrefix+constants.ReaderRoleNameSuffix, // tables + schemaName, rolePrefix+constants.ReaderRoleNameSuffix, // sequences + schemaName, rolePrefix+constants.WriterRoleNameSuffix, // tables + schemaName, rolePrefix+constants.WriterRoleNameSuffix, // sequences + schemaName, rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix, // types + schemaName, rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix)); err != nil { // functions + return fmt.Errorf("could not alter default privileges for database schema %s: %v", schemaName, err) + } + + return nil +} + +func (c *Cluster) execAlterGlobalDefaultPrivileges(owner, rolePrefix string) error { + if _, err := c.pgDb.Exec(fmt.Sprintf(globalDefaultPrivilegesSQL, owner, + rolePrefix+constants.WriterRoleNameSuffix, rolePrefix+constants.ReaderRoleNameSuffix, // schemas + rolePrefix+constants.ReaderRoleNameSuffix, // tables + rolePrefix+constants.ReaderRoleNameSuffix, // sequences + rolePrefix+constants.WriterRoleNameSuffix, // tables + rolePrefix+constants.WriterRoleNameSuffix, // sequences + rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix, // types + rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix)); err != nil { // functions + return fmt.Errorf("could not alter default privileges for database %s: %v", rolePrefix, err) + } + + return nil +} + func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin bool) (result []string) { if rolsuper { result = append(result, constants.RoleFlagSuperuser) @@ -278,8 +402,67 @@ func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin return result } -// Creates a connection pooler credentials lookup function in every database to -// perform remote authentication. +// getExtension returns the list of current database extensions +// The caller is responsible for opening and closing the database connection +func (c *Cluster) getExtensions() (dbExtensions map[string]string, err error) { + var ( + rows *sql.Rows + ) + + if rows, err = c.pgDb.Query(getExtensionsSQL); err != nil { + return nil, fmt.Errorf("could not query database extensions: %v", err) + } + + defer func() { + if err2 := rows.Close(); err2 != nil { + if err != nil { + err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err) + } else { + err = fmt.Errorf("error when closing query cursor: %v", err2) + } + } + }() + + dbExtensions = make(map[string]string) + + for rows.Next() { + var extension, schema string + + if err = rows.Scan(&extension, &schema); err != nil { + return nil, fmt.Errorf("error when processing row: %v", err) + } + dbExtensions[extension] = schema + } + + return dbExtensions, err +} + +// executeCreateExtension creates new extension in the given schema. +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeCreateExtension(extName, schemaName string) error { + return c.execCreateOrAlterExtension(extName, schemaName, createExtensionSQL, + "creating extension", "create extension") +} + +// executeAlterExtension changes the schema of the given extension. +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeAlterExtension(extName, schemaName string) error { + return c.execCreateOrAlterExtension(extName, schemaName, alterExtensionSQL, + "changing schema for extension", "alter extension schema") +} + +func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doing, operation string) error { + + c.logger.Infof("%s %q schema %q", doing, extName, schemaName) + if _, err := c.pgDb.Exec(fmt.Sprintf(statement, extName, schemaName)); err != nil { + return fmt.Errorf("could not execute %s: %v", operation, err) + } + + return nil +} + +// Creates a connection pool credentials lookup function in every database to +// perform remote authentification. func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { var stmtBytes bytes.Buffer c.logger.Info("Installing lookup function") @@ -305,7 +488,7 @@ func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { templater := template.Must(template.New("sql").Parse(connectionPoolerLookup)) - for dbname, _ := range currentDatabases { + for dbname := range currentDatabases { if dbname == "template0" || dbname == "template1" { continue } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 43190491b..9b92f2fb5 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1470,6 +1470,13 @@ func (c *Cluster) generateSingleUserSecret(namespace string, pgUser spec.PgUser) return nil } + //skip NOLOGIN users + for _, flag := range pgUser.Flags { + if flag == constants.RoleFlagNoLogin { + return nil + } + } + username := pgUser.Name secret := v1.Secret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 0eb02631c..0c4c662d4 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -3,6 +3,7 @@ package cluster import ( "context" "fmt" + "strings" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" @@ -108,6 +109,11 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { err = fmt.Errorf("could not sync databases: %v", err) return err } + c.logger.Debugf("syncing prepared databases with schemas") + if err = c.syncPreparedDatabases(); err != nil { + err = fmt.Errorf("could not sync prepared database: %v", err) + return err + } } // sync connection pooler @@ -563,6 +569,7 @@ func (c *Cluster) syncDatabases() error { createDatabases := make(map[string]string) alterOwnerDatabases := make(map[string]string) + preparedDatabases := make([]string, 0) if err := c.initDbConn(); err != nil { return fmt.Errorf("could not init database connection") @@ -578,12 +585,24 @@ func (c *Cluster) syncDatabases() error { return fmt.Errorf("could not get current databases: %v", err) } - for datname, newOwner := range c.Spec.Databases { - currentOwner, exists := currentDatabases[datname] + // if no prepared databases are specified create a database named like the cluster + if c.Spec.PreparedDatabases != nil && len(c.Spec.PreparedDatabases) == 0 { // TODO: add option to disable creating such a default DB + c.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{strings.Replace(c.Name, "-", "_", -1): {}} + } + for preparedDatabaseName := range c.Spec.PreparedDatabases { + _, exists := currentDatabases[preparedDatabaseName] if !exists { - createDatabases[datname] = newOwner + createDatabases[preparedDatabaseName] = preparedDatabaseName + constants.OwnerRoleNameSuffix + preparedDatabases = append(preparedDatabases, preparedDatabaseName) + } + } + + for databaseName, newOwner := range c.Spec.Databases { + currentOwner, exists := currentDatabases[databaseName] + if !exists { + createDatabases[databaseName] = newOwner } else if currentOwner != newOwner { - alterOwnerDatabases[datname] = newOwner + alterOwnerDatabases[databaseName] = newOwner } } @@ -591,13 +610,116 @@ func (c *Cluster) syncDatabases() error { return nil } - for datname, owner := range createDatabases { - if err = c.executeCreateDatabase(datname, owner); err != nil { + for databaseName, owner := range createDatabases { + if err = c.executeCreateDatabase(databaseName, owner); err != nil { return err } } - for datname, owner := range alterOwnerDatabases { - if err = c.executeAlterDatabaseOwner(datname, owner); err != nil { + for databaseName, owner := range alterOwnerDatabases { + if err = c.executeAlterDatabaseOwner(databaseName, owner); err != nil { + return err + } + } + + // set default privileges for prepared database + for _, preparedDatabase := range preparedDatabases { + if err = c.execAlterGlobalDefaultPrivileges(preparedDatabase+constants.OwnerRoleNameSuffix, preparedDatabase); err != nil { + return err + } + } + + return nil +} + +func (c *Cluster) syncPreparedDatabases() error { + c.setProcessName("syncing prepared databases") + for preparedDbName, preparedDB := range c.Spec.PreparedDatabases { + if err := c.initDbConnWithName(preparedDbName); err != nil { + return fmt.Errorf("could not init connection to database %s: %v", preparedDbName, err) + } + defer func() { + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + }() + + // now, prepare defined schemas + preparedSchemas := preparedDB.PreparedSchemas + if len(preparedDB.PreparedSchemas) == 0 { + preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}} + } + if err := c.syncPreparedSchemas(preparedDbName, preparedSchemas); err != nil { + return err + } + + // install extensions + if err := c.syncExtensions(preparedDB.Extensions); err != nil { + return err + } + } + + return nil +} + +func (c *Cluster) syncPreparedSchemas(databaseName string, preparedSchemas map[string]acidv1.PreparedSchema) error { + c.setProcessName("syncing prepared schemas") + + currentSchemas, err := c.getSchemas() + if err != nil { + return fmt.Errorf("could not get current schemas: %v", err) + } + + var schemas []string + + for schema := range preparedSchemas { + schemas = append(schemas, schema) + } + + if createPreparedSchemas, equal := util.SubstractStringSlices(schemas, currentSchemas); !equal { + for _, schemaName := range createPreparedSchemas { + owner := constants.OwnerRoleNameSuffix + dbOwner := databaseName + owner + if preparedSchemas[schemaName].DefaultRoles == nil || *preparedSchemas[schemaName].DefaultRoles { + owner = databaseName + "_" + schemaName + owner + } else { + owner = dbOwner + } + if err = c.executeCreateDatabaseSchema(databaseName, schemaName, dbOwner, owner); err != nil { + return err + } + } + } + + return nil +} + +func (c *Cluster) syncExtensions(extensions map[string]string) error { + c.setProcessName("syncing database extensions") + + createExtensions := make(map[string]string) + alterExtensions := make(map[string]string) + + currentExtensions, err := c.getExtensions() + if err != nil { + return fmt.Errorf("could not get current database extensions: %v", err) + } + + for extName, newSchema := range extensions { + currentSchema, exists := currentExtensions[extName] + if !exists { + createExtensions[extName] = newSchema + } else if currentSchema != newSchema { + alterExtensions[extName] = newSchema + } + } + + for extName, schema := range createExtensions { + if err = c.executeCreateExtension(extName, schema); err != nil { + return err + } + } + for extName, schema := range alterExtensions { + if err = c.executeAlterExtension(extName, schema); err != nil { return err } } diff --git a/pkg/controller/types.go b/pkg/controller/types.go index 0d86abec8..b598014c9 100644 --- a/pkg/controller/types.go +++ b/pkg/controller/types.go @@ -1,9 +1,10 @@ package controller import ( - "k8s.io/apimachinery/pkg/types" "time" + "k8s.io/apimachinery/pkg/types" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" ) diff --git a/pkg/spec/types.go b/pkg/spec/types.go index e1c49a1fd..08008267b 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -31,6 +31,7 @@ const ( RoleOriginInfrastructure RoleOriginTeamsAPI RoleOriginSystem + RoleOriginBootstrap RoleConnectionPooler ) @@ -180,6 +181,8 @@ func (r RoleOrigin) String() string { return "teams API role" case RoleOriginSystem: return "system role" + case RoleOriginBootstrap: + return "bootstrapped role" case RoleConnectionPooler: return "connection pooler role" default: diff --git a/pkg/util/constants/roles.go b/pkg/util/constants/roles.go index c2c287472..87c9c51ce 100644 --- a/pkg/util/constants/roles.go +++ b/pkg/util/constants/roles.go @@ -14,4 +14,8 @@ const ( RoleFlagCreateDB = "CREATEDB" RoleFlagReplication = "REPLICATION" RoleFlagByPassRLS = "BYPASSRLS" + OwnerRoleNameSuffix = "_owner" + ReaderRoleNameSuffix = "_reader" + WriterRoleNameSuffix = "_writer" + UserRoleNameSuffix = "_user" ) diff --git a/pkg/util/users/users.go b/pkg/util/users/users.go index 112f89b43..345caa001 100644 --- a/pkg/util/users/users.go +++ b/pkg/util/users/users.go @@ -73,26 +73,44 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM } // ExecuteSyncRequests makes actual database changes from the requests passed in its arguments. -func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(reqs []spec.PgSyncUserRequest, db *sql.DB) error { - for _, r := range reqs { - switch r.Kind { +func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(requests []spec.PgSyncUserRequest, db *sql.DB) error { + var reqretries []spec.PgSyncUserRequest + var errors []string + for _, request := range requests { + switch request.Kind { case spec.PGSyncUserAdd: - if err := strategy.createPgUser(r.User, db); err != nil { - return fmt.Errorf("could not create user %q: %v", r.User.Name, err) + if err := strategy.createPgUser(request.User, db); err != nil { + reqretries = append(reqretries, request) + errors = append(errors, fmt.Sprintf("could not create user %q: %v", request.User.Name, err)) } case spec.PGsyncUserAlter: - if err := strategy.alterPgUser(r.User, db); err != nil { - return fmt.Errorf("could not alter user %q: %v", r.User.Name, err) + if err := strategy.alterPgUser(request.User, db); err != nil { + reqretries = append(reqretries, request) + errors = append(errors, fmt.Sprintf("could not alter user %q: %v", request.User.Name, err)) } case spec.PGSyncAlterSet: - if err := strategy.alterPgUserSet(r.User, db); err != nil { - return fmt.Errorf("could not set custom user %q parameters: %v", r.User.Name, err) + if err := strategy.alterPgUserSet(request.User, db); err != nil { + reqretries = append(reqretries, request) + errors = append(errors, fmt.Sprintf("could not set custom user %q parameters: %v", request.User.Name, err)) } default: - return fmt.Errorf("unrecognized operation: %v", r.Kind) + return fmt.Errorf("unrecognized operation: %v", request.Kind) } } + + // creating roles might fail if group role members are created before the parent role + // retry adding roles as long as the number of failed attempts is shrinking + if len(reqretries) > 0 { + if len(reqretries) < len(requests) { + if err := strategy.ExecuteSyncRequests(reqretries, db); err != nil { + return err + } + } else { + return fmt.Errorf("could not execute sync requests for users: %v", errors) + } + } + return nil } func (strategy DefaultUserSyncStrategy) alterPgUserSet(user spec.PgUser, db *sql.DB) (err error) { From 865d5b41a75a98bbc6c015b67a5f228f0dd2e652 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 29 Apr 2020 17:26:46 +0200 Subject: [PATCH 093/104] set event broadcasting to Infof and update rbac (#952) --- .../templates/clusterrole.yaml | 5 +++++ docs/user.md | 19 +++++++++++++++---- manifests/operator-service-account-rbac.yaml | 5 +++++ pkg/controller/controller.go | 2 +- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 0defcab41..bd34e803e 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -49,6 +49,11 @@ rules: - events verbs: - create + - get + - list + - patch + - update + - watch # to manage endpoints which are also used by Patroni - apiGroups: - "" diff --git a/docs/user.md b/docs/user.md index 2d9f2be6a..3683fdf61 100644 --- a/docs/user.md +++ b/docs/user.md @@ -53,8 +53,19 @@ them. ## Watch pods being created +Check if the database pods are coming up. Use the label `application=spilo` to +filter and list the label `spilo-role` to see when the master is promoted and +replicas get their labels. + ```bash -kubectl get pods -w --show-labels +kubectl get pods -l application=spilo -L spilo-role -w +``` + +The operator also emits K8s events to the Postgresql CRD which can be inspected +in the operator logs or with: + +```bash +kubectl describe postgresql acid-minimal-cluster ``` ## Connect to PostgreSQL @@ -736,14 +747,14 @@ spin up more instances). ## Custom TLS certificates -By default, the spilo image generates its own TLS certificate during startup. +By default, the Spilo image generates its own TLS certificate during startup. However, this certificate cannot be verified and thus doesn't protect from active MITM attacks. In this section we show how to specify a custom TLS certificate which is mounted in the database pods via a K8s Secret. Before applying these changes, in k8s the operator must also be configured with the `spilo_fsgroup` set to the GID matching the postgres user group. If you -don't know the value, use `103` which is the GID from the default spilo image +don't know the value, use `103` which is the GID from the default Spilo image (`spilo_fsgroup=103` in the cluster request spec). OpenShift allocates the users and groups dynamically (based on scc), and their @@ -805,5 +816,5 @@ spec: Alternatively, it is also possible to use [cert-manager](https://cert-manager.io/docs/) to generate these secrets. -Certificate rotation is handled in the spilo image which checks every 5 +Certificate rotation is handled in the Spilo image which checks every 5 minutes if the certificates have changed and reloads postgres accordingly. diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 667941a24..266df30c5 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -50,6 +50,11 @@ rules: - events verbs: - create + - get + - list + - patch + - update + - watch # to manage endpoints which are also used by Patroni - apiGroups: - "" diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 0b3fde5d9..26b6b1b87 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -76,7 +76,7 @@ func NewController(controllerConfig *spec.ControllerConfig, controllerId string) } eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(logger.Debugf) + eventBroadcaster.StartLogging(logger.Infof) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: myComponentName}) c := &Controller{ From 5af4379118a41664f404a95c40ff17a872f15e04 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 30 Apr 2020 09:58:07 +0200 Subject: [PATCH 094/104] [UI] add toggle for connection pooler (#953) * [UI] add toggle for connection pooler * remove team service logger * fix new.tag.pug and change port in Makefile --- ui/Dockerfile | 5 ++-- ui/Makefile | 2 +- ui/app/src/edit.tag.pug | 1 + ui/app/src/new.tag.pug | 25 ++++++++++++++++- ui/app/src/postgresql.tag.pug | 9 ++++++ ui/manifests/deployment.yaml | 2 ++ ui/manifests/ui-service-account-rbac.yaml | 1 + ui/operator_ui/main.py | 34 +++++++++++++++++++++-- ui/operator_ui/spiloutils.py | 28 ++++++++++++++++--- ui/requirements.txt | 2 +- ui/start_server.sh | 2 ++ 11 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 ui/start_server.sh diff --git a/ui/Dockerfile b/ui/Dockerfile index 3e1ae8756..5ea912dbc 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:3.6 MAINTAINER team-acid@zalando.de -EXPOSE 8080 +EXPOSE 8081 RUN \ apk add --no-cache \ @@ -29,6 +29,7 @@ RUN \ /var/cache/apk/* COPY requirements.txt / +COPY start_server.sh / RUN pip3 install -r /requirements.txt COPY operator_ui /operator_ui @@ -37,4 +38,4 @@ ARG VERSION=dev RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" /operator_ui/__init__.py WORKDIR / -ENTRYPOINT ["/usr/bin/python3", "-m", "operator_ui"] +CMD ["/usr/bin/python3", "-m", "operator_ui"] diff --git a/ui/Makefile b/ui/Makefile index e7d5df674..29c8d9409 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -36,4 +36,4 @@ push: docker push "$(IMAGE):$(TAG)$(CDP_TAG)" mock: - docker run -it -p 8080:8080 "$(IMAGE):$(TAG)" --mock + docker run -it -p 8081:8081 "$(IMAGE):$(TAG)" --mock diff --git a/ui/app/src/edit.tag.pug b/ui/app/src/edit.tag.pug index 9029594bd..c1d94e589 100644 --- a/ui/app/src/edit.tag.pug +++ b/ui/app/src/edit.tag.pug @@ -137,6 +137,7 @@ edit o.spec.numberOfInstances = i.spec.numberOfInstances o.spec.enableMasterLoadBalancer = i.spec.enableMasterLoadBalancer || false o.spec.enableReplicaLoadBalancer = i.spec.enableReplicaLoadBalancer || false + o.spec.enableConnectionPooler = i.spec.enableConnectionPooler || false o.spec.volume = { size: i.spec.volume.size } if ('users' in i.spec && typeof i.spec.users === 'object') { diff --git a/ui/app/src/new.tag.pug b/ui/app/src/new.tag.pug index fe9d78226..6293a6c7a 100644 --- a/ui/app/src/new.tag.pug +++ b/ui/app/src/new.tag.pug @@ -239,6 +239,18 @@ new | | Enable replica ELB + tr + td Enable Connection Pool + td + label + input( + type='checkbox' + value='{ enableConnectionPooler }' + onchange='{ toggleEnableConnectionPooler }' + ) + | + | Enable Connection Pool (using PGBouncer) + tr td Volume size td @@ -493,6 +505,9 @@ new {{#if enableReplicaLoadBalancer}} enableReplicaLoadBalancer: true {{/if}} + {{#if enableConnectionPooler}} + enableConnectionPooler: true + {{/if}} volume: size: "{{ volumeSize }}Gi" {{#if users}} @@ -516,13 +531,14 @@ new - {{ odd }}/32 {{/if}} + {{#if resourcesVisible}} resources: requests: cpu: {{ cpu.state.request.state }}m memory: {{ memory.state.request.state }}Mi limits: cpu: {{ cpu.state.limit.state }}m - memory: {{ memory.state.limit.state }}Mi{{#if restoring}} + memory: {{ memory.state.limit.state }}Mi{{/if}}{{#if restoring}} clone: cluster: "{{ backup.state.name.state }}" @@ -542,6 +558,7 @@ new instanceCount: this.instanceCount, enableMasterLoadBalancer: this.enableMasterLoadBalancer, enableReplicaLoadBalancer: this.enableReplicaLoadBalancer, + enableConnectionPooler: this.enableConnectionPooler, volumeSize: this.volumeSize, users: this.users.valids, databases: this.databases.valids, @@ -552,6 +569,7 @@ new memory: this.memory, backup: this.backup, namespace: this.namespace, + resourcesVisible: this.config.resources_visible, restoring: this.backup.state.type.state !== 'empty', pitr: this.backup.state.type.state === 'pitr', } @@ -598,6 +616,10 @@ new this.enableReplicaLoadBalancer = !this.enableReplicaLoadBalancer } + this.toggleEnableConnectionPooler = e => { + this.enableConnectionPooler = !this.enableConnectionPooler + } + this.volumeChange = e => { this.volumeSize = +e.target.value } @@ -892,6 +914,7 @@ new this.odd = '' this.enableMasterLoadBalancer = false this.enableReplicaLoadBalancer = false + this.enableConnectionPooler = false this.postgresqlVersion = this.postgresqlVersion = ( this.config.postgresql_versions[0] diff --git a/ui/app/src/postgresql.tag.pug b/ui/app/src/postgresql.tag.pug index be7173dbe..9edae99d3 100644 --- a/ui/app/src/postgresql.tag.pug +++ b/ui/app/src/postgresql.tag.pug @@ -92,6 +92,8 @@ postgresql .alert.alert-success(if='{ progress.masterLabel }') PostgreSQL master available, label is attached .alert.alert-success(if='{ progress.masterLabel && progress.dnsName }') PostgreSQL ready: { progress.dnsName } + .alert.alert-success(if='{ progress.pooler }') Connection pooler deployment created + .col-lg-3 help-general(config='{ opts.config }') @@ -122,9 +124,11 @@ postgresql jQuery.get( '/postgresqls/' + this.cluster_path, ).done(data => { + this.progress.pooler = false this.progress.postgresql = true this.progress.postgresqlManifest = data this.progress.createdTimestamp = data.metadata.creationTimestamp + this.progress.poolerEnabled = data.spec.enableConnectionPooler this.uid = this.progress.postgresqlManifest.metadata.uid this.update() @@ -160,6 +164,11 @@ postgresql this.progress.dnsName = data.metadata.name + '.' + data.metadata.namespace } + jQuery.get('/pooler/' + this.cluster_path).done(data => { + this.progress.pooler = {"url": ""} + this.update() + }) + this.update() }) }) diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index 6138ca1a8..ccaecd312 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -44,6 +44,8 @@ spec: value: "http://postgres-operator:8080" - name: "OPERATOR_CLUSTER_NAME_LABEL" value: "cluster-name" + - name: "RESOURCES_VISIBLE" + value: "False" - name: "TARGET_NAMESPACE" value: "default" - name: "TEAMS" diff --git a/ui/manifests/ui-service-account-rbac.yaml b/ui/manifests/ui-service-account-rbac.yaml index 2e09797a0..d4937b5a2 100644 --- a/ui/manifests/ui-service-account-rbac.yaml +++ b/ui/manifests/ui-service-account-rbac.yaml @@ -39,6 +39,7 @@ rules: - apiGroups: - apps resources: + - deployments - statefulsets verbs: - get diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index 5a3054f0e..a294ae081 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -25,7 +25,7 @@ from flask import ( from flask_oauthlib.client import OAuth from functools import wraps from gevent import sleep, spawn -from gevent.wsgi import WSGIServer +from gevent.pywsgi import WSGIServer from jq import jq from json import dumps, loads from logging import DEBUG, ERROR, INFO, basicConfig, exception, getLogger @@ -44,6 +44,7 @@ from .spiloutils import ( create_postgresql, read_basebackups, read_namespaces, + read_pooler, read_pods, read_postgresql, read_postgresqls, @@ -80,6 +81,7 @@ OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-nam OPERATOR_UI_CONFIG = getenv('OPERATOR_UI_CONFIG', '{}') OPERATOR_UI_MAINTENANCE_CHECK = getenv('OPERATOR_UI_MAINTENANCE_CHECK', '{}') READ_ONLY_MODE = getenv('READ_ONLY_MODE', False) in [True, 'true'] +RESOURCES_VISIBLE = getenv('RESOURCES_VISIBLE', True) SPILO_S3_BACKUP_PREFIX = getenv('SPILO_S3_BACKUP_PREFIX', 'spilo/') SUPERUSER_TEAM = getenv('SUPERUSER_TEAM', 'acid') TARGET_NAMESPACE = getenv('TARGET_NAMESPACE') @@ -312,6 +314,7 @@ DEFAULT_UI_CONFIG = { def get_config(): config = loads(OPERATOR_UI_CONFIG) or DEFAULT_UI_CONFIG config['read_only_mode'] = READ_ONLY_MODE + config['resources_visible'] = RESOURCES_VISIBLE config['superuser_team'] = SUPERUSER_TEAM config['target_namespace'] = TARGET_NAMESPACE @@ -397,6 +400,22 @@ def get_service(namespace: str, cluster: str): ) +@app.route('/pooler//') +@authorize +def get_list_poolers(namespace: str, cluster: str): + + if TARGET_NAMESPACE not in ['', '*', namespace]: + return wrong_namespace() + + return respond( + read_pooler( + get_cluster(), + namespace, + "{}-pooler".format(cluster), + ), + ) + + @app.route('/statefulsets//') @authorize def get_list_clusters(namespace: str, cluster: str): @@ -587,6 +606,17 @@ def update_postgresql(namespace: str, cluster: str): spec['volume'] = {'size': size} + if 'enableConnectionPooler' in postgresql['spec']: + cp = postgresql['spec']['enableConnectionPooler'] + if not cp: + if 'enableConnectionPooler' in o['spec']: + del o['spec']['enableConnectionPooler'] + else: + spec['enableConnectionPooler'] = True + else: + if 'enableConnectionPooler' in o['spec']: + del o['spec']['enableConnectionPooler'] + if 'enableReplicaLoadBalancer' in postgresql['spec']: rlb = postgresql['spec']['enableReplicaLoadBalancer'] if not rlb: @@ -1006,7 +1036,7 @@ def init_cluster(): def main(port, secret_key, debug, clusters: list): global TARGET_NAMESPACE - basicConfig(level=DEBUG if debug else INFO) + basicConfig(stream=sys.stdout, level=(DEBUG if debug else INFO), format='%(asctime)s %(levelname)s: %(message)s',) init_cluster() diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py index 33d07d88a..8d1996fb5 100644 --- a/ui/operator_ui/spiloutils.py +++ b/ui/operator_ui/spiloutils.py @@ -1,7 +1,7 @@ from boto3 import client from datetime import datetime, timezone from furl import furl -from json import dumps +from json import dumps, loads from logging import getLogger from os import environ, getenv from requests import Session @@ -18,6 +18,15 @@ session = Session() OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-name') +COMMON_CLUSTER_LABEL = getenv('COMMON_CLUSTER_LABEL', '{"application":"spilo"}') +COMMON_POOLER_LABEL = getenv('COMMONG_POOLER_LABEL', '{"application":"db-connection-pooler"}') + +logger.info("Common Cluster Label: {}".format(COMMON_CLUSTER_LABEL)) +logger.info("Common Pooler Label: {}".format(COMMON_POOLER_LABEL)) + +COMMON_CLUSTER_LABEL = loads(COMMON_CLUSTER_LABEL) +COMMON_POOLER_LABEL = loads(COMMON_POOLER_LABEL) + def request(cluster, path, **kwargs): if 'timeout' not in kwargs: @@ -85,6 +94,7 @@ def resource_api_version(resource_type): return { 'postgresqls': 'apis/acid.zalan.do/v1', 'statefulsets': 'apis/apps/v1', + 'deployments': 'apis/apps/v1', }.get(resource_type, 'api/v1') @@ -149,7 +159,7 @@ def read_pod(cluster, namespace, resource_name): resource_type='pods', namespace=namespace, resource_name=resource_name, - label_selector={'application': 'spilo'}, + label_selector=COMMON_CLUSTER_LABEL, ) @@ -159,7 +169,17 @@ def read_service(cluster, namespace, resource_name): resource_type='services', namespace=namespace, resource_name=resource_name, - label_selector={'application': 'spilo'}, + label_selector=COMMON_CLUSTER_LABEL, + ) + + +def read_pooler(cluster, namespace, resource_name): + return kubernetes_get( + cluster=cluster, + resource_type='deployments', + namespace=namespace, + resource_name=resource_name, + label_selector=COMMON_POOLER_LABEL, ) @@ -169,7 +189,7 @@ def read_statefulset(cluster, namespace, resource_name): resource_type='statefulsets', namespace=namespace, resource_name=resource_name, - label_selector={'application': 'spilo'}, + label_selector=COMMON_CLUSTER_LABEL, ) diff --git a/ui/requirements.txt b/ui/requirements.txt index 5d987416c..7dc49eb3d 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -1,5 +1,5 @@ Flask-OAuthlib==0.9.5 -Flask==1.1.1 +Flask==1.1.2 backoff==1.8.1 boto3==1.10.4 boto==2.49.0 diff --git a/ui/start_server.sh b/ui/start_server.sh new file mode 100644 index 000000000..e2c3980cc --- /dev/null +++ b/ui/start_server.sh @@ -0,0 +1,2 @@ +#!/bin/bash +/usr/bin/python3 -m operator_ui From be208b61f1f795bd79490832dc798cf262ba0cf8 Mon Sep 17 00:00:00 2001 From: Petr Barborka Date: Thu, 30 Apr 2020 17:10:16 +0200 Subject: [PATCH 095/104] Fix S3 backup list (#880) * Fix S3 backup list Co-authored-by: Petr Barborka --- ui/operator_ui/spiloutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py index 8d1996fb5..ea347a84d 100644 --- a/ui/operator_ui/spiloutils.py +++ b/ui/operator_ui/spiloutils.py @@ -322,7 +322,7 @@ def read_basebackups( f=configure_backup_cxt, aws_instance_profile=use_aws_instance_profile, s3_prefix=f's3://{bucket}/{prefix}{pg_cluster}{suffix}/wal/', - )._backup_list(detail=True) + )._backup_list(detail=True)._backup_list(prefix=f"{prefix}{pg_cluster}{suffix}/wal/") ] From d52296c3235956ea4cb34f78f6e4cf0aa374d09d Mon Sep 17 00:00:00 2001 From: Rafia Sabih Date: Mon, 4 May 2020 14:46:56 +0200 Subject: [PATCH 096/104] Propagate annotations to the StatefulSet (#932) * Initial commit * Corrections - set the type of the new configuration parameter to be array of strings - propagate the annotations to statefulset at sync * Enable regular expression matching * Improvements -handle rollingUpdate flag -modularize code -rename config parameter name * fix merge error * Pass annotations to connection pooler deployment * update code-gen * Add documentation and update manifests * add e2e test and introduce option in configmap * fix service annotations test * Add unit test * fix e2e tests * better key lookup of annotations tests * add debug message for annotation tests * Fix typos * minor fix for looping * Handle update path and renaming - handle the update path to update sts and connection pooler deployment. This way no need to wait for sync - rename the parameter to downscaler_annotations - handle other review comments * another try to fix python loops * Avoid unneccessary update events * Update manifests * some final polishing * fix cluster_test after polishing Co-authored-by: Rafia Sabih Co-authored-by: Felix Kunde --- .../crds/operatorconfigurations.yaml | 4 ++ charts/postgres-operator/values-crd.yaml | 9 ++- charts/postgres-operator/values.yaml | 3 + docs/reference/cluster_manifest.md | 2 +- docs/reference/operator_parameters.md | 6 ++ e2e/tests/test_e2e.py | 63 ++++++++++++++++--- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 4 ++ ...gresql-operator-default-configuration.yaml | 3 + pkg/apis/acid.zalan.do/v1/crds.go | 34 ++++++---- .../v1/operator_configuration_type.go | 1 + .../acid.zalan.do/v1/zz_generated.deepcopy.go | 5 ++ pkg/cluster/cluster.go | 3 +- pkg/cluster/cluster_test.go | 33 +++++++++- pkg/cluster/k8sres.go | 6 +- pkg/cluster/resources.go | 21 +++++++ pkg/cluster/sync.go | 32 ++++++++++ pkg/controller/operator_config.go | 1 + pkg/controller/postgresql.go | 4 +- pkg/util/config/config.go | 1 + 20 files changed, 205 insertions(+), 31 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 71260bdb6..ffcef7b4a 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -117,6 +117,10 @@ spec: type: object additionalProperties: type: string + downscaler_annotations: + type: array + items: + type: string enable_init_containers: type: boolean enable_pod_antiaffinity: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index e6428c478..98a399c8b 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -67,6 +67,11 @@ configKubernetes: # keya: valuea # keyb: valueb + # list of annotations propagated from cluster manifest to statefulset and deployment + # downscaler_annotations: + # - deployment-time + # - downscaler/* + # enables initContainers to run actions before Spilo is started enable_init_containers: true # toggles pod anti affinity on the Postgres pods @@ -262,11 +267,11 @@ configConnectionPooler: # docker image connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" # max db connections the pooler should hold - connection_pooler_max_db_connections: "60" + connection_pooler_max_db_connections: 60 # default pooling mode connection_pooler_mode: "transaction" # number of pooler instances - connection_pooler_number_of_instances: "2" + connection_pooler_number_of_instances: 2 # default resources connection_pooler_default_cpu_request: 500m connection_pooler_default_memory_request: 100Mi diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index e5cdcee47..5578a5ed1 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -63,6 +63,9 @@ configKubernetes: # annotations attached to each database pod # custom_pod_annotations: "keya:valuea,keyb:valueb" + # list of annotations propagated from cluster manifest to statefulset and deployment + # downscaler_annotations: "deployment-time,downscaler/*" + # enables initContainers to run actions before Spilo is started enable_init_containers: "true" # toggles pod anti affinity on the Postgres pods diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index c87728812..576031543 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -165,7 +165,7 @@ These parameters are grouped directly under the `spec` key in the manifest. If `targetContainers` is empty, additional volumes will be mounted only in the `postgres` container. If you set the `all` special item, it will be mounted in all containers (postgres + sidecars). Else you can set the list of target containers in which the additional volumes will be mounted (eg : postgres, telegraf) - + ## Postgres parameters Those parameters are grouped under the `postgresql` top-level key, which is diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index af6e93d25..a81cabfc4 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -200,6 +200,12 @@ configuration they are grouped under the `kubernetes` key. of a database created by the operator. If the annotation key is also provided by the database definition, the database definition value is used. +* **downscaler_annotations** + An array of annotations that should be passed from Postgres CRD on to the + statefulset and, if exists, to the connection pooler deployment as well. + Regular expressions like `downscaler/*` etc. are also accepted. Can be used + with [kube-downscaler](https://github.com/hjacobs/kube-downscaler). + * **watched_namespace** The operator watches for Postgres objects in the given namespace. If not specified, the value is taken from the operator namespace. A special `*` diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index aa6f1205d..18b9852c4 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -428,7 +428,7 @@ class EndToEndTestCase(unittest.TestCase): k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) # wait a little before proceeding with the pod distribution test - time.sleep(k8s.RETRY_TIMEOUT_SEC) + time.sleep(30) # toggle pod anti affinity to move replica away from master node self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) @@ -465,21 +465,24 @@ class EndToEndTestCase(unittest.TestCase): pg_patch_custom_annotations = { "spec": { "serviceAnnotations": { - "annotation.key": "value" + "annotation.key": "value", + "foo": "bar", } } } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) + # wait a little before proceeding + time.sleep(30) annotations = { "annotation.key": "value", "foo": "bar", } self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-service-annotations,spilo-role=master", annotations)) + "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-service-annotations,spilo-role=replica", annotations)) + "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) # clean up unpatch_custom_service_annotations = { @@ -489,6 +492,40 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(unpatch_custom_service_annotations) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_statefulset_annotation_propagation(self): + ''' + Inject annotation to Postgresql CRD and check it's propagation to stateful set + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + patch_sset_propagate_annotations = { + "data": { + "downscaler_annotations": "deployment-time,downscaler/*", + } + } + k8s.update_config(patch_sset_propagate_annotations) + + pg_crd_annotations = { + "metadata": { + "annotations": { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + }, + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) + + # wait a little before proceeding + time.sleep(60) + annotations = { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + } + self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): ''' @@ -528,7 +565,7 @@ class EndToEndTestCase(unittest.TestCase): k8s.update_config(patch_toleration_config) # wait a little before proceeding with the pod distribution test - time.sleep(k8s.RETRY_TIMEOUT_SEC) + time.sleep(30) # toggle pod anti affinity to move replica away from master node self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) @@ -694,10 +731,18 @@ class K8s: def check_service_annotations(self, svc_labels, annotations, namespace='default'): svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items for svc in svcs: - if len(svc.metadata.annotations) != len(annotations): - return False - for key in svc.metadata.annotations: - if svc.metadata.annotations[key] != annotations[key]: + for key, value in annotations.items(): + if key not in svc.metadata.annotations or svc.metadata.annotations[key] != value: + print("Expected key {} not found in annotations {}".format(key, svc.metadata.annotation)) + return False + return True + + def check_statefulset_annotations(self, sset_labels, annotations, namespace='default'): + ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=sset_labels, limit=1).items + for sset in ssets: + for key, value in annotations.items(): + if key not in sset.metadata.annotations or sset.metadata.annotations[key] != value: + print("Expected key {} not found in annotations {}".format(key, sset.metadata.annotation)) return False return True diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 8719e76a1..537055b18 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -30,6 +30,7 @@ data: # default_memory_limit: 500Mi # default_memory_request: 100Mi docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + # downscaler_annotations: "deployment-time,downscaler/*" # enable_admin_role_for_users: "true" # enable_crd_validation: "true" # enable_database_access: "true" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 2d3e614b9..23b5ff0fc 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -93,6 +93,10 @@ spec: type: object additionalProperties: type: string + downscaler_annotations: + type: array + items: + type: string enable_init_containers: type: boolean enable_pod_antiaffinity: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 78312b684..9ae1b3b26 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -31,6 +31,9 @@ configuration: # custom_pod_annotations: # keya: valuea # keyb: valueb + # downscaler_annotations: + # - deployment-time + # - downscaler/* enable_init_containers: true enable_pod_antiaffinity: false enable_pod_disruption_budget: true diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 35037ec3c..ad1b79a45 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -21,48 +21,48 @@ const ( // PostgresCRDResourceColumns definition of AdditionalPrinterColumns for postgresql CRD var PostgresCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{ - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Team", Type: "string", Description: "Team responsible for Postgres cluster", JSONPath: ".spec.teamId", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Version", Type: "string", Description: "PostgreSQL version", JSONPath: ".spec.postgresql.version", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Pods", Type: "integer", Description: "Number of Pods per Postgres cluster", JSONPath: ".spec.numberOfInstances", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Volume", Type: "string", Description: "Size of the bound volume", JSONPath: ".spec.volume.size", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "CPU-Request", Type: "string", Description: "Requested CPU for Postgres containers", JSONPath: ".spec.resources.requests.cpu", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Memory-Request", Type: "string", Description: "Requested memory for Postgres containers", JSONPath: ".spec.resources.requests.memory", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Status", Type: "string", Description: "Current sync status of postgresql resource", @@ -72,31 +72,31 @@ var PostgresCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{ // OperatorConfigCRDResourceColumns definition of AdditionalPrinterColumns for OperatorConfiguration CRD var OperatorConfigCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefinition{ - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Image", Type: "string", Description: "Spilo image to be used for Pods", JSONPath: ".configuration.docker_image", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Cluster-Label", Type: "string", Description: "Label for K8s resources created by operator", JSONPath: ".configuration.kubernetes.cluster_name_label", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Service-Account", Type: "string", Description: "Name of service account to be used", JSONPath: ".configuration.kubernetes.pod_service_account_name", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Min-Instances", Type: "integer", Description: "Minimum number of instances per Postgres cluster", JSONPath: ".configuration.min_instances", }, - apiextv1beta1.CustomResourceColumnDefinition{ + { Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp", @@ -888,6 +888,14 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "downscaler_annotations": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, "enable_init_containers": { Type: "boolean", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 4a0abf3ca..d3a9f6ec2 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -62,6 +62,7 @@ type KubernetesMetaConfiguration struct { PodRoleLabel string `json:"pod_role_label,omitempty"` ClusterLabels map[string]string `json:"cluster_labels,omitempty"` InheritedLabels []string `json:"inherited_labels,omitempty"` + DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` ClusterNameLabel string `json:"cluster_name_label,omitempty"` NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 5b4d6cdcd..5879c9b73 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -180,6 +180,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura *out = make([]string, len(*in)) copy(*out, *in) } + if in.DownscalerAnnotations != nil { + in, out := &in.DownscalerAnnotations, &out.DownscalerAnnotations + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.NodeReadinessLabel != nil { in, out := &in.NodeReadinessLabel, &out.NodeReadinessLabel *out = make(map[string]string, len(*in)) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 387107540..31b8fa155 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -711,8 +711,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { updateFailed = true return } - - if !reflect.DeepEqual(oldSs, newSs) { + if !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) { c.logger.Debugf("syncing statefulsets") // TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet if err := c.syncStatefulSet(); err != nil { diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 539038bff..4562d525e 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -28,19 +28,48 @@ var eventRecorder = record.NewFakeRecorder(1) var cl = New( Config{ OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, Auth: config.Auth{ SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, + Resources: config.Resources{ + DownscalerAnnotations: []string{"downscaler/*"}, + }, }, }, k8sutil.NewMockKubernetesClient(), - acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test"}}, + acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test", Annotations: map[string]string{"downscaler/downtime_replicas": "0"}}}, logger, eventRecorder, ) +func TestStatefulSetAnnotations(t *testing.T) { + testName := "CheckStatefulsetAnnotations" + spec := acidv1.PostgresSpec{ + TeamID: "myapp", NumberOfInstances: 1, + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + } + ss, err := cl.generateStatefulSet(&spec) + if err != nil { + t.Errorf("in %s no statefulset created %v", testName, err) + } + if ss != nil { + annotation := ss.ObjectMeta.GetAnnotations() + if _, ok := annotation["downscaler/downtime_replicas"]; !ok { + t.Errorf("in %s respective annotation not found on sts", testName) + } + } + +} + func TestInitRobotUsers(t *testing.T) { testName := "TestInitRobotUsers" tests := []struct { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 9b92f2fb5..534ae7b8e 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -6,6 +6,7 @@ import ( "fmt" "path" "sort" + "strconv" "github.com/sirupsen/logrus" @@ -1182,12 +1183,15 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef return nil, fmt.Errorf("could not set the pod management policy to the unknown value: %v", c.OpConfig.PodManagementPolicy) } + annotations = make(map[string]string) + annotations[rollingUpdateStatefulsetAnnotationKey] = strconv.FormatBool(false) + statefulSet := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: c.statefulSetName(), Namespace: c.Namespace, Labels: c.labelsSet(true), - Annotations: map[string]string{rollingUpdateStatefulsetAnnotationKey: "false"}, + Annotations: c.AnnotationsToPropagate(annotations), }, Spec: appsv1.StatefulSetSpec{ Replicas: &numberOfInstances, diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index b38458af8..3528c46f4 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -853,3 +853,24 @@ func (c *Cluster) updateConnectionPoolerDeployment(oldDeploymentSpec, newDeploym return deployment, nil } + +//updateConnectionPoolerAnnotations updates the annotations of connection pooler deployment +func (c *Cluster) updateConnectionPoolerAnnotations(annotations map[string]string) (*appsv1.Deployment, error) { + c.logger.Debugf("updating connection pooler annotations") + patchData, err := metaAnnotationsPatch(annotations) + if err != nil { + return nil, fmt.Errorf("could not form patch for the deployment metadata: %v", err) + } + result, err := c.KubeClient.Deployments(c.ConnectionPooler.Deployment.Namespace).Patch( + context.TODO(), + c.ConnectionPooler.Deployment.Name, + types.MergePatchType, + []byte(patchData), + metav1.PatchOptions{}, + "") + if err != nil { + return nil, fmt.Errorf("could not patch connection pooler annotations %q: %v", patchData, err) + } + return result, nil + +} diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 0c4c662d4..697fc2d05 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -3,6 +3,7 @@ package cluster import ( "context" "fmt" + "regexp" "strings" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -358,6 +359,8 @@ func (c *Cluster) syncStatefulSet() error { } } } + annotations := c.AnnotationsToPropagate(c.Statefulset.Annotations) + c.updateStatefulSetAnnotations(annotations) if !podsRollingUpdateRequired && !c.OpConfig.EnableLazySpiloUpgrade { // even if desired and actual statefulsets match @@ -397,6 +400,30 @@ func (c *Cluster) syncStatefulSet() error { return nil } +// AnnotationsToPropagate get the annotations to update if required +// based on the annotations in postgres CRD +func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[string]string { + toPropagateAnnotations := c.OpConfig.DownscalerAnnotations + pgCRDAnnotations := c.Postgresql.ObjectMeta.GetAnnotations() + + if toPropagateAnnotations != nil && pgCRDAnnotations != nil { + for _, anno := range toPropagateAnnotations { + for k, v := range pgCRDAnnotations { + matched, err := regexp.MatchString(anno, k) + if err != nil { + c.logger.Errorf("annotations matching issue: %v", err) + return nil + } + if matched { + annotations[k] = v + } + } + } + } + + return annotations +} + // checkAndSetGlobalPostgreSQLConfiguration checks whether cluster-wide API parameters // (like max_connections) has changed and if necessary sets it via the Patroni API func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration() error { @@ -939,6 +966,11 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql } } + newAnnotations := c.AnnotationsToPropagate(c.ConnectionPooler.Deployment.Annotations) + if newAnnotations != nil { + c.updateConnectionPoolerAnnotations(newAnnotations) + } + service, err := c.KubeClient. Services(c.Namespace). Get(context.TODO(), c.connectionPoolerName(), metav1.GetOptions{}) diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index f66eafb1e..4eed44924 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -73,6 +73,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodRoleLabel = fromCRD.Kubernetes.PodRoleLabel result.ClusterLabels = fromCRD.Kubernetes.ClusterLabels result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels + result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations result.ClusterNameLabel = fromCRD.Kubernetes.ClusterNameLabel result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 2a9e1b650..c243f330f 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -487,7 +487,9 @@ func (c *Controller) postgresqlUpdate(prev, cur interface{}) { if pgOld != nil && pgNew != nil { // Avoid the inifinite recursion for status updates if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) { - return + if reflect.DeepEqual(pgNew.Annotations, pgOld.Annotations) { + return + } } c.queueClusterEvent(pgOld, pgNew, EventUpdate) } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 37ba947d6..d5c4b6671 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -34,6 +34,7 @@ type Resources struct { SpiloPrivileged bool `name:"spilo_privileged" default:"false"` ClusterLabels map[string]string `name:"cluster_labels" default:"application:spilo"` InheritedLabels []string `name:"inherited_labels" default:""` + DownscalerAnnotations []string `name:"downscaler_annotations"` ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` PodRoleLabel string `name:"pod_role_label" default:"spilo-role"` PodToleration map[string]string `name:"toleration" default:""` From 76d43525f7463fb7598ea83736052e9d4ee91325 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 4 May 2020 16:23:21 +0200 Subject: [PATCH 097/104] define more default values for opConfig CRD (#955) --- charts/postgres-operator/values.yaml | 2 +- manifests/configmap.yaml | 10 +++---- pkg/controller/operator_config.go | 42 ++++++++++++++-------------- pkg/util/config/config.go | 2 +- pkg/util/util.go | 14 ++++++++-- 5 files changed, 39 insertions(+), 31 deletions(-) diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 5578a5ed1..cb8e29081 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -109,7 +109,7 @@ configKubernetes: # Postgres pods are terminated forcefully after this timeout pod_terminate_grace_period: 5m # template for database user secrets generated by the operator - secret_name_template: '{username}.{cluster}.credentials' + secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" # group ID with write-access to volumes (required to run Spilo as non-root process) # spilo_fsgroup: "103" diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 537055b18..0a740e198 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -50,16 +50,16 @@ data: # inherited_labels: application,environment # kube_iam_role: "" # log_s3_bucket: "" - # logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" # logical_backup_s3_access_key_id: "" - # logical_backup_s3_bucket: "my-bucket-url" + logical_backup_s3_bucket: "my-bucket-url" # logical_backup_s3_region: "" # logical_backup_s3_endpoint: "" # logical_backup_s3_secret_access_key: "" - # logical_backup_s3_sse: "AES256" - # logical_backup_schedule: "30 00 * * *" + logical_backup_s3_sse: "AES256" + logical_backup_schedule: "30 00 * * *" master_dns_name_format: "{cluster}.{team}.{hostedzone}" - # master_pod_move_timeout: 10m + # master_pod_move_timeout: 20m # max_instances: "-1" # min_instances: "-1" # min_cpu_limit: 250m diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 4eed44924..389240c09 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -33,28 +33,28 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result := &config.Config{} // general config - result.EnableCRDValidation = fromCRD.EnableCRDValidation + result.EnableCRDValidation = util.CoalesceBool(fromCRD.EnableCRDValidation, util.True()) result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps - result.DockerImage = fromCRD.DockerImage + result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115") result.Workers = fromCRD.Workers result.MinInstances = fromCRD.MinInstances result.MaxInstances = fromCRD.MaxInstances result.ResyncPeriod = time.Duration(fromCRD.ResyncPeriod) result.RepairPeriod = time.Duration(fromCRD.RepairPeriod) result.SetMemoryRequestToLimit = fromCRD.SetMemoryRequestToLimit - result.ShmVolume = fromCRD.ShmVolume + result.ShmVolume = util.CoalesceBool(fromCRD.ShmVolume, util.True()) result.SidecarImages = fromCRD.SidecarImages result.SidecarContainers = fromCRD.SidecarContainers // user config - result.SuperUsername = fromCRD.PostgresUsersConfiguration.SuperUsername - result.ReplicationUsername = fromCRD.PostgresUsersConfiguration.ReplicationUsername + result.SuperUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.SuperUsername, "postgres") + result.ReplicationUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.ReplicationUsername, "standby") // kubernetes config result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations - result.PodServiceAccountName = fromCRD.Kubernetes.PodServiceAccountName + result.PodServiceAccountName = util.Coalesce(fromCRD.Kubernetes.PodServiceAccountName, "postgres-pod") result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition result.PodEnvironmentConfigMap = fromCRD.Kubernetes.PodEnvironmentConfigMap @@ -64,31 +64,31 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ClusterDomain = util.Coalesce(fromCRD.Kubernetes.ClusterDomain, "cluster.local") result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat - result.EnablePodDisruptionBudget = fromCRD.Kubernetes.EnablePodDisruptionBudget - result.EnableInitContainers = fromCRD.Kubernetes.EnableInitContainers - result.EnableSidecars = fromCRD.Kubernetes.EnableSidecars + result.EnablePodDisruptionBudget = util.CoalesceBool(fromCRD.Kubernetes.EnablePodDisruptionBudget, util.True()) + result.EnableInitContainers = util.CoalesceBool(fromCRD.Kubernetes.EnableInitContainers, util.True()) + result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True()) result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate result.OAuthTokenSecretName = fromCRD.Kubernetes.OAuthTokenSecretName result.InfrastructureRolesSecretName = fromCRD.Kubernetes.InfrastructureRolesSecretName - result.PodRoleLabel = fromCRD.Kubernetes.PodRoleLabel + result.PodRoleLabel = util.Coalesce(fromCRD.Kubernetes.PodRoleLabel, "spilo-role") result.ClusterLabels = fromCRD.Kubernetes.ClusterLabels result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations - result.ClusterNameLabel = fromCRD.Kubernetes.ClusterNameLabel + result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name") result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName - result.PodManagementPolicy = fromCRD.Kubernetes.PodManagementPolicy + result.PodManagementPolicy = util.Coalesce(fromCRD.Kubernetes.PodManagementPolicy, "ordered_ready") result.MasterPodMoveTimeout = time.Duration(fromCRD.Kubernetes.MasterPodMoveTimeout) result.EnablePodAntiAffinity = fromCRD.Kubernetes.EnablePodAntiAffinity - result.PodAntiAffinityTopologyKey = fromCRD.Kubernetes.PodAntiAffinityTopologyKey + result.PodAntiAffinityTopologyKey = util.Coalesce(fromCRD.Kubernetes.PodAntiAffinityTopologyKey, "kubernetes.io/hostname") // Postgres Pod resources - result.DefaultCPURequest = fromCRD.PostgresPodResources.DefaultCPURequest - result.DefaultMemoryRequest = fromCRD.PostgresPodResources.DefaultMemoryRequest - result.DefaultCPULimit = fromCRD.PostgresPodResources.DefaultCPULimit - result.DefaultMemoryLimit = fromCRD.PostgresPodResources.DefaultMemoryLimit - result.MinCPULimit = fromCRD.PostgresPodResources.MinCPULimit - result.MinMemoryLimit = fromCRD.PostgresPodResources.MinMemoryLimit + result.DefaultCPURequest = util.Coalesce(fromCRD.PostgresPodResources.DefaultCPURequest, "100m") + result.DefaultMemoryRequest = util.Coalesce(fromCRD.PostgresPodResources.DefaultMemoryRequest, "100Mi") + result.DefaultCPULimit = util.Coalesce(fromCRD.PostgresPodResources.DefaultCPULimit, "1") + result.DefaultMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.DefaultMemoryLimit, "500Mi") + result.MinCPULimit = util.Coalesce(fromCRD.PostgresPodResources.MinCPULimit, "250m") + result.MinMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.MinMemoryLimit, "250Mi") // timeout config result.ResourceCheckInterval = time.Duration(fromCRD.Timeouts.ResourceCheckInterval) @@ -115,8 +115,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath // logical backup config - result.LogicalBackupSchedule = fromCRD.LogicalBackup.Schedule - result.LogicalBackupDockerImage = fromCRD.LogicalBackup.DockerImage + result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *") + result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup") result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index d5c4b6671..d8c92ba3e 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -83,7 +83,7 @@ type LogicalBackup struct { LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` - LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:"AES256"` + LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""` } // Operator options for connection pooler diff --git a/pkg/util/util.go b/pkg/util/util.go index 46df5d345..5701429aa 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -147,7 +147,7 @@ func Coalesce(val, defaultVal string) string { return val } -// Yeah, golang +// CoalesceInt32 works like coalesce but for *int32 func CoalesceInt32(val, defaultVal *int32) *int32 { if val == nil { return defaultVal @@ -155,6 +155,14 @@ func CoalesceInt32(val, defaultVal *int32) *int32 { return val } +// CoalesceBool works like coalesce but for *bool +func CoalesceBool(val, defaultVal *bool) *bool { + if val == nil { + return defaultVal + } + return val +} + // Test if any of the values is nil func testNil(values ...*int32) bool { for _, v := range values { @@ -166,8 +174,8 @@ func testNil(values ...*int32) bool { return false } -// Return maximum of two integers provided via pointers. If one value is not -// defined, return the other one. If both are not defined, result is also +// MaxInt32 : Return maximum of two integers provided via pointers. If one value +// is not defined, return the other one. If both are not defined, result is also // undefined, caller needs to check for that. func MaxInt32(a, b *int32) *int32 { if testNil(a, b) { From bb3d2fa678afd579184a6ee81d13fba9e03459dd Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 5 May 2020 12:52:54 +0200 Subject: [PATCH 098/104] Bump v1.5.0 (#954) * bump to v1.5.0 * update helm charts and docs * update helm charts and packages * update images for spilo, logical-backup and pooler --- charts/postgres-operator-ui/Chart.yaml | 6 +- charts/postgres-operator-ui/index.yaml | 27 ++++++- .../postgres-operator-ui-1.5.0.tgz | Bin 0 -> 3786 bytes .../templates/deployment.yaml | 4 + charts/postgres-operator-ui/values.yaml | 10 ++- charts/postgres-operator/Chart.yaml | 4 +- charts/postgres-operator/index.yaml | 76 ++++++------------ .../postgres-operator-1.2.0.tgz | Bin 6799 -> 0 bytes .../postgres-operator-1.3.0.tgz | Bin 19063 -> 0 bytes .../postgres-operator-1.4.0.tgz | Bin 42200 -> 14223 bytes .../postgres-operator-1.5.0.tgz | Bin 0 -> 15843 bytes charts/postgres-operator/values-crd.yaml | 10 +-- charts/postgres-operator/values.yaml | 8 +- docs/index.md | 31 +++++-- manifests/complete-postgres-manifest.yaml | 2 +- manifests/configmap.yaml | 2 +- manifests/postgres-operator.yaml | 2 +- ...gresql-operator-default-configuration.yaml | 4 +- pkg/controller/operator_config.go | 2 +- pkg/util/config/config.go | 2 +- ui/manifests/deployment.yaml | 2 +- 21 files changed, 105 insertions(+), 87 deletions(-) create mode 100644 charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz delete mode 100644 charts/postgres-operator/postgres-operator-1.2.0.tgz delete mode 100644 charts/postgres-operator/postgres-operator-1.3.0.tgz create mode 100644 charts/postgres-operator/postgres-operator-1.5.0.tgz diff --git a/charts/postgres-operator-ui/Chart.yaml b/charts/postgres-operator-ui/Chart.yaml index a6e46ab3e..13550d67e 100644 --- a/charts/postgres-operator-ui/Chart.yaml +++ b/charts/postgres-operator-ui/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: postgres-operator-ui -version: 1.4.0 -appVersion: 1.4.0 +version: 1.5.0 +appVersion: 1.5.0 home: https://github.com/zalando/postgres-operator description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience keywords: @@ -14,8 +14,6 @@ keywords: maintainers: - name: Zalando email: opensource@zalando.de -- name: siku4 - email: sk@sik-net.de sources: - https://github.com/zalando/postgres-operator engine: gotpl diff --git a/charts/postgres-operator-ui/index.yaml b/charts/postgres-operator-ui/index.yaml index 0cd03d6e5..114e6a4d7 100644 --- a/charts/postgres-operator-ui/index.yaml +++ b/charts/postgres-operator-ui/index.yaml @@ -1,9 +1,32 @@ apiVersion: v1 entries: postgres-operator-ui: + - apiVersion: v1 + appVersion: 1.5.0 + created: "2020-05-04T16:36:04.770110276+02:00" + description: Postgres Operator UI provides a graphical interface for a convenient + database-as-a-service user experience + digest: ff373185f9d125f918935b226eaed0a245cc4dd561f884424d92f094b279afe9 + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - ui + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + name: postgres-operator-ui + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-ui-1.5.0.tgz + version: 1.5.0 - apiVersion: v1 appVersion: 1.4.0 - created: "2020-02-24T15:32:47.610967635+01:00" + created: "2020-05-04T16:36:04.769604808+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience digest: 00e0eff7056d56467cd5c975657fbb76c8d01accd25a4b7aca81bc42aeac961d @@ -26,4 +49,4 @@ entries: urls: - postgres-operator-ui-1.4.0.tgz version: 1.4.0 -generated: "2020-02-24T15:32:47.610348278+01:00" +generated: "2020-05-04T16:36:04.768922456+02:00" diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..6d64ee3b55582f3420da6c1d78ad883affe7a09e GIT binary patch literal 3786 zcmV;*4mI%~iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PK8ibK5wQa6a=_^wQ5(&16H0)SGQq<+=4Z-jr(-M`b(N+S{6P zL1at9m;x98l{{mk$5dtI;H^}3yBp5N>E-ZStXrS}abm5PXG z-Z%HvoZKIzkc56fAt`48%zTR^N%_-vd#-0iC__OLWnK+(Zh?0>x4`>TNCcnJ2qh30 z3zAG|NMfK&p%{@6VZ;Rx2sxV}qsWv65D`U&M505aLmY`>Mni;Dq5!;2P*7wcS`$9S z047Q$GHA8NR87*M8}e!E6Nw3ncx#Dwt!uoy;UbcOwuIZ)O+kW$NmFy?9GFrw%T zH6&4jGhHDQ8gpw(C{u(o6b2Da35^4dgh`%?5dWSJz>UxXU_=k|zq40X_G{Iyva-eo z<%1?Qt1rmouK&vYKO=F9^3epqhW+31`}@uP-|P9i{r?nY4_@Jjq_Ki*(am;U#T2f! z2bU8nf$EL^@1wVGoDmmOq7+6jqA_ZKR~W`bAk2uMWEi6a#bJn$5GgUzLV?2_PX#EP zCNWVcUCR=f#55$wJY^~X-?H|!v>4HGDiBf%bV|l(nZJPrAaG2j5_duQCO%(sK_55`~ck`BMN&q-a^W*gFbCGx0TO30&0R zHY3(}E+=T@67UhI2{v&ku7#~@Ku<@&xB`g^2#a!5dXY7<7eF#RRKggUrW!_&O3KCt z?#v_ou}Vfj}c<5hnT% zQ!hZ}8$r`xq3MRGZV;L2;r*#WQsO-toVswzBno9(6-uwVN>yN(puoRaP`*4$=z9?d zU~AnGw3eAUIPeZUyVBw~PNhO|MyB|N3{kfYvsaGp)iwmp*Wm+|bQog*_CH80O`RfQ zj7qnb2?=#olt~JEiMsh@3YOM1R(FYRl8@nZpfM&~r~vE(@4&YU-qZ8eumueYx=;yc zJi-Cj3t(D8NSJ1OneUM*AF={;L@|!4xa&0N|p#|z*B=m@BS3DI?#7}P6_3RSMW zF|LRbDD7!xT2U}MT|R|2Y7IwaI~iJkQfFyZAka?Fna zQtIL@nIXwzsS1Ne_^|Ed=?*3Z8h3V;I=Ly>WaZtbX8W;GS`dl?!h!#*Vi z_V2Csb|o(>?Y~kxx6a;Oo?N)7VE zbC}U-U_1cr$7yTq1F{(Ej7UXkOdz!i8JGoB#D`ZYyAr|>;_)xKJ=6%3^63BM38)n@Xu0aWs^7Nq%^*P@n0~~UX}3+ zej45x!n``W(6HJ@)%NDku5~&Bkx9r5Nytr{N#>45=4mOop>eD&U2Saw(eAb$d_LOk z=v+8v3-ZuWN!p^aByRz;7GNv-$8v$`|CzZ2FSlRf$~a51WLRoiWii&;+XoLY5tOMB z*ngGIU!~o^mi7A(rd#*iV8+6`A-N;|kR`3h3(hKwTN3#v%ECB}HZz6$bGZn0v}R4< zOOvRus+I7UtW)MRXx_Jn2KDVleL#B$P zYQhEmWFiyp^?@^bK#T2oXcB8mj%35o9Mi{fdOfYbKA1(_hp;}xqt z2!nrX7s4Q#U_5o@q-6xpHtll>RU%j($&@fM#>g4o8GNNy$>Xo<8&W0}VIkI=o#nz< zq;Ym`L=yU1@H8=`a7}+zA=cm!j?0|moP4qB}wE8Oo}KRrLjcyAZ)8fReHT&!gPOc zl+2HXp$nOAO+ z;RN$}Yei*|-vsk{uvD$c80Pb(ZU&Y7rlmHEG?zer=c)w{KDI?;TM5-((suEax)w>oYKF(uBZo*QtnX)}&$1 z6Gwx=)%)`|%}JSDlts!<%tI2JWQ`()W(BHugOl^4%XjBjM}yNZ;H)e?vYB-2X-4&N zufl%(=KaOx$@$gU(c6=&H%GsmJhY=`cGB7|&rdGiy+1!bxw!gpdU5*8o0Ba&YUVbr z?cnn0{PoGDnaRcA=y+TFjr^$f=ueK`UaZ0S*=f87_|MWC$VJ&w#jhLI=>6%{@w>Bs zoW9<^_xn}7M?92QF=f|wU>BLoOJdR0TUt#}18Y%@tYu6{)v{l#M9Sn+4Y{lK zV{)@rHpun)!+R=ubQ|DQqSy*>5%;^wk7uMEtYjNc8sH&jiQxU7=poIzI8o=s;}OS@uOcNd8BWyxH+HQcP?9KXueL4ygx#<(7`$crv5 z|4JM4zH09|X5#P}KBin%EWyh7vKQ5iY29S;duc+ow)p|zmquP2RFTuI`_NCzLo+T# zIe{$AyZl+{#njYoTPJtL%$yrm?~sJo$fBn2mhbhmH9KSlXy#LD&_Z-nAZxT)O|$PU zSN|wwgZ?js^zW(v{jS&i{&&B>-{0x~rznl@e`_Z?yO{4I#(XOwG$)T%RU!&CW2lsL zS%_gNyq0S@!Xo$yKQ7@gpHtbz*HiXVs-Iha6IFIpyjeM>W$-1vwg~iH9uJm6+FujL zeIypkK0{kEDMO_B+_L`UnqoWopXaN7r5e~E|Gj>|6NTdI%ZtpWAw3U|M%McrTZWJ-Sy6h zI5z%0{FDeB(OZnN(7<1QcHte1?|?JYQV$6d6cAH}uI0YExVlhWptT3bd^+U}J{(^_ zM1{26F;y+|Hygg?4nK*O`CGi0j9dCo@l(!NYk^4ntZ8DteUjF5SKcJnb9YFtt>>%P zd%wGM|4XO8+y75e8j2m65;O}nt>TO#zq-~YS8t7bqx9SShLQm6w%7I?uj_QYOaCC~ zdO_#V^*yiGIrQ6yfAiXb=VgaOzsGe`2Jco^P8mo#MyUcg!~@a`+u<r z>x>Tk{$aP%8+mOq9AUptdYutzk9uCz@AdY>-hL-K?2%zaGK1sccUb9Wi+ijJe=)zy zD*IW3HFsKN54C%(%21uT+ba8&-ERdZ&4|`>R5spoW&Fh2T~~`YYi`y7sHIOl@ZKxk zBo*-z5vrXyhS^*-{$zk=O&;8WS-V>?t9N*K*y(nAKPInw9(p($?R&j`)a!Tq`(e}z z5BGch-u`Gf-0z16k>`^zjM_cY9qyBm9Qebq+a^fD!@eK=Ie4{fCa?c4>9xa9HNhpl zPP;z{V|Q0Gw&vIJE~fR|+U|Dyhy8+Gzqgv%m0j7DAF%vy00030|11aSEC5yj0N5d2 A1^@s6 literal 0 HcmV?d00001 diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml index da0280e61..6247ec933 100644 --- a/charts/postgres-operator-ui/templates/deployment.yaml +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -41,6 +41,10 @@ spec: value: "http://localhost:8081" - name: "OPERATOR_API_URL" value: {{ .Values.envs.operatorApiUrl }} + - name: "OPERATOR_CLUSTER_NAME_LABEL" + value: {{ .Values.envs.operatorClusterNameLabel }} + - name: "RESOURCES_VISIBLE" + value: {{ .Values.envs.resourcesVisible }} - name: "TARGET_NAMESPACE" value: {{ .Values.envs.targetNamespace }} - name: "TEAMS" diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml index dd25864b2..90e9daa66 100644 --- a/charts/postgres-operator-ui/values.yaml +++ b/charts/postgres-operator-ui/values.yaml @@ -8,7 +8,7 @@ replicaCount: 1 image: registry: registry.opensource.zalan.do repository: acid/postgres-operator-ui - tag: v1.4.0 + tag: v1.5.0 pullPolicy: "IfNotPresent" rbac: @@ -25,8 +25,8 @@ serviceAccount: # configure UI pod resources resources: limits: - cpu: 300m - memory: 3000Mi + cpu: 200m + memory: 200Mi requests: cpu: 100m memory: 100Mi @@ -36,12 +36,14 @@ envs: # IMPORTANT: While operator chart and UI chart are idendependent, this is the interface between # UI and operator API. Insert the service name of the operator API here! operatorApiUrl: "http://postgres-operator:8080" + operatorClusterNameLabel: "cluster-name" + resourcesVisible: "False" targetNamespace: "default" # configure UI service service: type: "ClusterIP" - port: "8080" + port: "8081" # If the type of the service is NodePort a port can be specified using the nodePort field # If the nodePort field is not specified, or if it has no value, then a random port is used # notePort: 32521 diff --git a/charts/postgres-operator/Chart.yaml b/charts/postgres-operator/Chart.yaml index 89468dfa4..cd9f75586 100644 --- a/charts/postgres-operator/Chart.yaml +++ b/charts/postgres-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: postgres-operator -version: 1.4.0 -appVersion: 1.4.0 +version: 1.5.0 +appVersion: 1.5.0 home: https://github.com/zalando/postgres-operator description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes keywords: diff --git a/charts/postgres-operator/index.yaml b/charts/postgres-operator/index.yaml index 53181d74a..63c7b450d 100644 --- a/charts/postgres-operator/index.yaml +++ b/charts/postgres-operator/index.yaml @@ -2,11 +2,33 @@ apiVersion: v1 entries: postgres-operator: - apiVersion: v1 - appVersion: 1.4.0 - created: "2020-02-20T17:39:25.443276193+01:00" + appVersion: 1.5.0 + created: "2020-05-04T16:36:19.646719041+02:00" description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes - digest: b93ccde5581deb8ed0857136b8ce74ca3f1b7240438fa4415f705764a1300bed + digest: 43510e4ed7005b2b80708df24cfbb0099b263b4a2954cff4e8f305543760be6d + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + name: postgres-operator + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-1.5.0.tgz + version: 1.5.0 + - apiVersion: v1 + appVersion: 1.4.0 + created: "2020-05-04T16:36:19.645338751+02:00" + description: Postgres Operator creates and manages PostgreSQL clusters running + in Kubernetes + digest: f8b90fecfc3cb825b94ed17edd9d5cefc36ae61801d4568597b4a79bcd73b2e9 home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -23,50 +45,4 @@ entries: urls: - postgres-operator-1.4.0.tgz version: 1.4.0 - - apiVersion: v1 - appVersion: 1.3.0 - created: "2020-02-20T17:39:25.441532163+01:00" - description: Postgres Operator creates and manages PostgreSQL clusters running - in Kubernetes - digest: 7e788fd37daec76a01f6d6f9fe5be5b54f5035e4eba0041e80a760d656537325 - home: https://github.com/zalando/postgres-operator - keywords: - - postgres - - operator - - cloud-native - - patroni - - spilo - maintainers: - - email: opensource@zalando.de - name: Zalando - name: postgres-operator - sources: - - https://github.com/zalando/postgres-operator - urls: - - postgres-operator-1.3.0.tgz - version: 1.3.0 - - apiVersion: v1 - appVersion: 1.2.0 - created: "2020-02-20T17:39:25.440278302+01:00" - description: Postgres Operator creates and manages PostgreSQL clusters running - in Kubernetes - digest: d10710c7cf19f4e266e7704f5d1e98dcfc61bee3919522326c35c22ca7d2f2bf - home: https://github.com/zalando/postgres-operator - keywords: - - postgres - - operator - - cloud-native - - patroni - - spilo - maintainers: - - email: opensource@zalando.de - name: Zalando - - email: kgyoo8232@gmail.com - name: kimxogus - name: postgres-operator - sources: - - https://github.com/zalando/postgres-operator - urls: - - postgres-operator-1.2.0.tgz - version: 1.2.0 -generated: "2020-02-20T17:39:25.439168098+01:00" +generated: "2020-05-04T16:36:19.643857452+02:00" diff --git a/charts/postgres-operator/postgres-operator-1.2.0.tgz b/charts/postgres-operator/postgres-operator-1.2.0.tgz deleted file mode 100644 index bd725688c2d5b4a1f6942dede4e49830ed3e59c1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6799 zcmV;A8gS(wiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PHa(mwK;ad(BIS?PXPJpDcM@-uTc;PEz}}wx+-t zkVG^COaKgJTG9UYt!`j04k=NVth^U!U^o64{W$aXQ(^VPQ%qrbKnlWvA z3zjC|-P1D|3z>?L zroS89xvk~S{gXT_6F+lpgpwne4+EBE<)6deZf_98+(cSrmR_BBZh;S;TOiV$S#E&I z7*ZzLl$)Z|*?+x-D9MfG+JMfb6mkke1^_?iW3DB)+yq=sh2$fcDw`$2Or?ATGix(5 z>i4I@&hl|DQfdD)OPGvRzs0V>IbU2T9h*_mh2m?_h4OnBqD19!S28Q+91Ah4l@vi2 zOePW)q)bT5gyfoF@RW&Ug!QFSxsLeXeA~U42LO^GhJSajs^aIL}pVTxrf%~)F1=kL>eygs}n&E&nZCz=~BZ6^qJ-~$O4mLv->8IQzd0ZanZnk_(0fG|ACljMv?np@KgcEJCH zG&dH;93@Xq0Gv3AK}fJOP7}~%EaF&6^GPLJ&Px&06RIDp$n+y6Bc56M;RQmU8aMqN zkC>3tF1^AY8HYqy$u8?=Dh_s9xFTV9pz8%ex;XWu{gE$RiT5n!L6E8VP)cPP62gq& z@(N$QWn-SWzk*1~iI{$5ddlqxI!E;9JC+dvr(AN)5^#lbO<*8nnx!0tudn^u6>~Tt z*IQt60g_)p%q$x-!y!{KTnIZ;xrIce!lMLAf|;d|@>JG@Pp>$KG|U z&UuG`a~s8ojpPxbVys5lKT#S^ob>I%t2DC<$PG8};|t@(GaMhC^$@&`;&7%6_SJzr zLs<>6iq5%qWDdap89E-mmT&vSsU+~b!{KPyJLqQrMu1cH31ps*xh7_l#xfakgB(<9 zJFmrh&>g}LAeAFc0FVk9mQ_a3=?)#XsgQR7%08_GnsF^u3}bFDIG0FY!=s$Yrg_GM z2F-uVxv|F0QYLih>WttZy$OU_$fyYb-;@CM1`a~RBxIV;xwH^*jq?N|hNGw^o;^U# zxsFtt3Y&6i9lLQGy04*cBeW`{9*>|iVTs|Lz>i0Upp1xl#I&2iV4Fj4986avtaE`O zhM~_D{k6)ikO*eROTz$6#x5j*&a{fSF=g+OE>W&Emo`CWzfk%dtpn)n(|oID7C7r@ zjNDxgY;LMK_md&Bf@oj=j@`l^F-;21vqVI0+7@F|a)2Am;(J)s9^n+0@p#b$Zt{$4 zEJKvT-i>)F6~_w@QrLJLfGb#oR@zP;^U6kZ=*N7*^2EaN`!k3Y8cj&rA77M=3>P!5 zYxEVR(HB5bCUiANCG#gLVhO=#aV(r3fP@4eFk{42@)$dTBUiMe#=r82c(j0;cuC2! zl1?LM+#zK8+JIK|tpHXhQJqU_jUbs^OC#`>7lF5GI!(AC(aEG0z$O!<@&bh>!YUaf zQ?aRJXQtv1%NNC~c~FOXa{Ll8Hr(Pr?}R*@Gl5vZ0F;Jd6J9Kg&Pkny<2-h{q7uu* z5o$t6Ipqaj)j7+2qM5Nek8G|vXobzY>4(OB#*86Dn?N-Shg3LpFvBK4{g!l_g}YSP z&Cx^{ix~M4w=-r*!i+fxIpbR3kW92n8w`*#DJI-lkBKi=azP;gtVlWPF`Qv0k8p}l z@&r;b^%@c-79)r_GOnFmnrO!3JVBa*RAlt;_;N(!RLyzlVUJ*UkOqZ>TkZ)fy&fbW zi%8BQNH8;%Co$5FB|JJ&F&vlt2rr2r zA3Kghn=l@`DK$|#Wyq+d9eL>o`l_47W5>FvQnZLspk*pi(*>OK1+~?hb_jcEYK29u z@SNj3YZ`xOXCNl+Nm80wWD^0GbD@<)r3Q1R1+s`!AS%nL^GX7rbG<+eyj+gMTD8#0 z)|6${7#21!Hi(^ZUNkk5ks#(OB&6ozWm9{HsJ#UnIn|QTEuW@%+fEvN~mxVIC>XjVWi!_fpvQlEc zr?#hBK%w+7d!L(&}7#`;{4xN_MqUwy-)LdnOQx3MYo3 zkDxOg?0NRMm~lHJDsc~_bWaK~g75{JWnwN8KIJi_PHs7<)%&K3DK+&4v(bzjKb`#0 z6^hY&Qd2SKvht_UDxz9(SiT!U=SioK>Q36QB&q5cw4Dkc1#MPmqgQ4Zw%=Bmu14|h zM8&7P447bRM<+iy)42jrcB1m+&|7d8Oc-T(X?gBBQbbu^paB&BxCJO*Zv?}Ew~@UW zd@H29aqRILz7};zUxHfG+9vGY2^;}G)6m>aXo8@5q0*Y=#>i_cDF4xyA2pQf#f(e1 zV8RxTU0Z%QtrqiyTORv~O@EkRmI`ouV3mpX`iWkzPg&p8@t%pp1tN|;e<$BJN zm5vXyCC5pu`M&CWEe5tpJ#V`Fk$BgKUd{)v;iGnNeB`I<0 zLjw}}BxQqfy%tWKF-07B93sdO#N!_S#Bl61VZM=c* z41y>(R;8h@5!N8OE&+7T`Qp=vruyfn=X}B3Z&p~f-Z3cJw2O={^^){78Guf$S4M7O z*i^mt&V#pa5l&dZY4{Fb(5&}buWi?tX*FC^Q*|q|99thzY8V2wt~L>?TJ_zGrD@up z-&YF%60PE^p0R8yigT{Q_x@?LPh_t!KjO6u170CWnQtRh#}_Ve2OIt z%2#f=H9VR4DRqp+MvVcUb3ImuI|z`dsi_>Wnhiyt!$eUdD>JD_LDS$~CINk4zTQpTygi44V4u3fV zrQ!9_iKph_h54XgN13DJVHM4Bfo^!jE)3R~Mj6zM>c@z2s&Z+IE)}j*6%Z=iQ7Sfq z4r;cpd)IZ2oAWxDomJ2f6dS8KFA(tHCnDiyVJuHep;VYpkGwIyrsXhWb~b`eAKUN8 zLi=`+6q-+wY|SBfb@sAw2#{kgOz4V@fal!^N0}wvVFl#P;X7yKAcUNhi$_|pw8v8R zSxI(bXjRY~LxuKexP(5=v;*kphJvN!t5qK7*&dAZ=-h>ra^s|xnTaezZmP{fkt_=S z*})&7*$Z9GO@aM#_!bPGqCaXJQ~h%Iw)@I?Hb~zlz&blpXi+N(zpAl!irdayIg>@i zl9wzx&piW<={NXb4lgGn0(3Gq0PGUh3ti!`D2#r=00skig8#YsN73cN6g36wcmPFf zi0LD5G@w^zAe)A9o@Ojb9=F?TE?;zRUcd15yRLqxO@GIC9vjU~o=}{0xt}$nr;D!p z-OY7EBfxU2Qq*CEiHe<_#;A(8}UY|g## zn$DcnI@L^CA99RTN=7G2nUmLLu+K4}R|geHRG1XSuIYqI3stbIxHt-O&S))bjNK_B zzonM@*a(aH7O5vCkfk+?LgnQKnZ)x(31&qEq6XB2EE7$O$Gi1jE&+p|PT$sN7t8Go z#iw5}cvLPZpthp`o|9u4OT)&RV72@yM&Vp*IavoRyBRw7LU>w=Gd_k(Zjt&bNw z?z@D-A7&OD$(RPQBR+dNTg_J}n(KGX?*IZ-@^q55>znIJ;F+uyRh_0Q!JUpr#2TpIXr=#M+zJt-tRNR#+=KY;wrJ&1X+-1^=Zn>OL ztn*i!%NF`U_1lc#fBqK{c&}n!tTTNP0w~M6(_F;7kG=02 zp16$~{Q&~Heb=I?=>>b;KQi0ioC-5$+-5NzKSiHrgxVT~N@Nr0cFWbBA1*JUcghpa4DY=! z-on)tOu2PVQ#XMQ%d%cmBq0Q>T*l5iEhKv6?Uv#B+Jse*$xnj>v|bh=kik#(zkWEYXaQ;7R}LD!@{~t4|p@ zm`m1aMCU~UU#{!p)%6jg=35pEp{p>cMW62N2mXF`jn3WMc+oPW%J3$UFOfWdKcWlkw)hzE-yFyc zFhbsAy8)34qrRezn3QxaK9;mC%B(H05%CjKLHY}IWc0)4<6FbCq=|#l&?gnHjkmW( zFw;WX33UEyx_>pDC1|eQ+i={v-%FfWteY|K#BDsDO-J*6ZDdOdKN2!Z^7saJ=shmU z--Jn{0oS2l<4}|Ue)l7CCvmA~c~^=YlZ|>tre^<1a_ef&hLgx~2n&<*;3e}8Z3{Lk)ScYFTtVV=v&x_;1O7WI4(>FO$| z9}c8XEVnbI#b*jg^v++90Y4v(xn;xPT*!C?M~-PuRl$2KOHR54#c>PG*m*PDSuXOp&DShs8*B}}AC@R!gmoBE;# zZo=J>hnBfyvhZhhjQEUZ{oyZX?3sQUC9UMIYHX_Yc(b(i;x5cslY^I+HpI4%hm(94-Y^k{FF~dZrAB+ zcS30Ga0CEtN9>NoH@VyTiEfAk-rSN9?0{6Xg`u|BmA0vPxj`0hbs}UUB#(m~vZvg6 zxl-3JozY&3nk8aRdtZBZXQ$e+rFAP;--0^0R{j^>)EzH_xk>&%AMP)&|2!M)ZRP(% zJU;Nwe=BuB{i>1rO(V++p%VK=k=4@vI+)@K zL~#Q9xa5cxqQ4e#?kxwazgaO4uyo=Rpi`heQ|i1kf=I_*0+J+w$vACU_SXU|lTj8C zn8N0-XihH^VeF4fAKtp)2SAqOnkA#9l>z`G-H@Z9X{?XF=?$34lKjvAY7Aco^ybk!r_5Z={^7{W^c(B#~5AifbZ)U1!-f=nl z4Y~Rai^?x?X$y(z5Bi%CUD;-eBNl{7%4{@yTchiJQT9tZSzd_bP^*nR!CTfA0Jn=E zfPd&V5R<1q$kcFeAJJAY_)G7j$EEj@8BWK$J4->S&SU7o?_g$Z_u!ey)2fMb-`{;S zx>B)hf`D_(YGf*C)V%U_wcoc?ru6$>)M3_|Xim=iWTP)(q}82vY`(U?{&TyltCi)?k(1hW%B>pJUCVXkxSZc@z83vI`tbfg-nh=-8b#t|7 z@Z`hU$JeK?&cY8TuTBp?emD&s3O}75eth%cefZ(!f4(~Uc#nA0=Tl=Py3MfFfzgs- zhl91;zkE4#Rj_?+bi1uq?G5G#l$Ro|f>_>Dx(ZU`AmED1_3tYI)DJGMs9*cMMzGc8 z#`0;@s+koBRcljXZN)oA&Hg;kb@u;yt#`XNZjApO443Tx-Q8!~^M4QW+@uXwdgn){ z$F1d{P9qr5pAS31buobLQqdPXBFzgGZn?k%?h=>v)s|Lz_Fbj(#%Dm$H1-<^^}RT-)+6*o6s|4yRGJ9zc7g zt3;rF2hepGtX%u@P-1ss+=h!_1=-4LZf>!-RiQl-L>Lk&LLxVR(sE69PEJPQ-5n+rujsC z;x6A5{LAAWe2~ckl%$%75@dA$s^q<(cYGF}S*3Zf14k-Nm4u&<&L9@r1ih)Sefqyc zKj@7=>puOzcrlyy@qfjiW-j{`Ab(vVT?`sL>6wcxc+wlQ^WaI(rdjah|AHO(nQ5VN z18?|ARc+ z{D1$9`Ty2NW$Nhwt;)IypYuODF<^PG;kTL=usgi>q<|jepEe~R-G+NF2?5=`lK!gO z5p?`7#~l6-%cfs1Y2f}gW#B(<%D`%yM431czG~(`|Bmj>yXW+QZqD3m^1xDbe;cU- zy|%7#L-l5xQ*fJ8aGO(bn^W*sIR!g6M28V{Rjuu@bHk!G;dN;6o` ze2#o)!@t0WX04hGAB?nsfAu5Oy(P&p5}mmC6n37b4u(E<;aCk(m|% zU%eiA8g@z$dR27iIvDMc)4N*8X+uD(BTnzGi#Xi?szmF6LQUJGjvFL(+<{+aT!QAw zf)diGgeT?t;>x`waCCb7)QN33Ix|+p+B${Ycf9ohX>uV9f7)~iox9>S4u0wQnc8XH x>gWWzpeW{8#4(qx`&+gdDYqFZw;3t7&-U3q+vgsi{|^8F|NrJQ?uP)P004?pXx9J$ diff --git a/charts/postgres-operator/postgres-operator-1.3.0.tgz b/charts/postgres-operator/postgres-operator-1.3.0.tgz deleted file mode 100644 index 460fed53286e21075290ed1d7cdcc1778100f95c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19063 zcmV)XK&`(YiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH;nd=y3ZD4=p62ndL@A)$njyX107AV)JK0Rp6ugc|VJyPdmR zvbTHe?j<=A2wgfz?;usAO9zoIO`3=(y*DWWf(SzXzn$H^+iM{S$oGA}KdV2z?9RM- zGxO%ndvD&nXQf%r#6VU_TOosSG!qNKO{ok z$nhJ!SBFJt!i#~B*M&fSkB#LprdY@;x8>fs|B(l?5}hE!5;Ua+8EOS)wL1P(t2C+* z1rAvQLs&WVDoLakkSJ0MFfb6~APX=G2NsONOpujIrF3Wy45W?aAj1O2Mo|Q10tBT1 zpq)(*847ZcRhVfD)B-cdSy^pxu!-QzHoeL~TY~#wBu3$Mum^!E(xE+*W^h)kPy*?z zLJ1t-mB2vKHe5+z9FYO}LKw%;6roT8)=H4H!h#VLhY=KH5DaL+2vW<}r&!v?7~p4O z+bSF?0H81)!*0SWh47arQbJ7xuTY)Ia2@S8KL4KUKLaCekbRvD;3exnEId5iZT*L< zLqguJ|2OeqMk7HHoLvjD`zj1HWh6{dnQS7{+F;cx0KhU?9Rp2>LZMA*fE34Iq*5&w zje&Zbi7$uqKXfE*G7*$X3;a072K^KOKnl~7P={k2rpH*Q!wd$_cbI>LgPpce2gvm9erIoGBNEhQCB2V({T4@L`GX~3;Yb*NUYicnhB5zb}}46e(- zNCL+M_T`QiYqsbzXwt?nY3Dc2fa}aO%V~ih|F;Fp(h(HPVHCf)KPV?MKEB&n^X#+;e zzLS_9lB^bBRx3#ugnkNK&RK@%SwcZA5c!RB4EWjC8E6MX^cW;7jq5OqBb=1tXT(U> zy$DXQj16rQI=u}yLC(u?6F4+rE^SzGy?~yPW-J&dlldPcCi>kla6r`2#bFs ze&iryAt;Q4IunB#pz9p7(7-@W=m`fcRubc&7SzwSv5?_k_EkB$qcSiMhkQp!wmx5# zZ%Ailhzx>+CWy<2HxuIwW{4xWtQ9lxFazt0^wOTj@Rs~yFhB+^9X+C2i0k^$6x0G- zuYy^a#Y#ec3v$ka1rrE|#&Dfps_!|c3`7LwyPOCIr&!k+tnchB1Yy9)mY5;khL|BKywK^<8(m>}l~`RVRQH6(YDD(PQ?!o2 zMgHSiN^diy^XtoER|-FsHiqmL8CMeD)D3NfT?fNdsLM@1jO$iN;@baA}QyU3+ z_69;iKp_8*IE<3am_*~r&><%ATZR#{k(|z8wdqL0Lhwjx{Mojp=*6;x1=2PS zbpmldNJ7N-GBJYFal%)=S&LWzu@tXg5%&?k5EKWQ42;wQ4eR>FQAZOZd=UmwXD~xU zx@SF?uP*Hg=yzn1-0N6|2z7~3X#u~b;d-7WvNkJZWM3Szwt>OX7KKEAE#P^%LRbZS zhixR8Br@4JV*<@3F_486hZ+=?NdAjOz>H=Ie#vS9S~{L`GXNZB;#C$THd8bXQy>Wq z9L*p}#!BN+6h(8WcdYcPy|`KOe=!8j5S%@l#8@@~_0P{w!RRrAN7on!)$$ct$Yc-( z7-cZfHj48oEa?lBjU*KuP4egpD5Ujx^V0pxA5ZyT+P~P~gv8X?6cv}ny};S)h9tyTs7jFDg1&u$3G#4p7AQ&Jhgq#ES6PjRz*m;&4)}Q>q2kN<0Tt-m*YCe^ z5&aK8`K^DQ8Io4Wuqw{_N~VC9tp8Ar$NFy+8rtaX`hOEoc6M+e$RI3Qz(Nif2@-O4 zD{N-rhs9uqS`ZlAx37XPrHIY4ViXQIGemnFppEFa0ojWcVmT!*h7nXgt61qQ$BSz~ zB^aD}mH||q1mhE5GBE=(41q(C!|$w=0YqrfPr{O7Ga893;HPu~gA|T_3z$Zu9T{Mb zHjFlsw1eI@j3kT%#CdHQ^+BcR3x6J#^%7~ued zGLSa>CFG!L;23^am}oXYe(1|6l;;4V4T+~tM5Q9h3#E{YfoPIx5-Kc^ z!*Gnlv<|hHZ_YQf^2Y0Up(;RX4;Mw_(92M}^dI%QVBmOVlVCYWal)ybfy4-l8+YdV<0U%JgqGBNH6mn&BIll}C0$;Y$%E0}n*?;<|B1iJ_@pw4UYHvT(q|N{3iL zIK+AtVwADc>};fI{I4h)1(g3qqA8jh@Dll79Tw*1{}-wbYZUrc{(lqCYmxswMgNup zy}Z=`UWW!ilQ0++cn1puCk*eeb`Yt#3F1`eqq0`L@9>J^1H&*3QJ=e{90*}lV< z#w)kakEhD4)Bn$zyfkc98rO+jODA$Y-79fAzQc4^%tE?Nu;_FPasWZ$km5jSex;Gq zPyx%zPRj~bL2{@nU~#$QV*#s3F3kmQLFV~az&gkky5R6-&Wi;dCDBFAqqy=_HmQ91 zKZC!PdjK!t{~@6v5$^s!;p(^N|8L}RmH!P4E))NG(qb^m1VJ4u#+ida9lf1j)!Jr2 z)rue?t7=Wsdf>;?NpK*d6hD!I>qrG4Csandx>6}8*H@(6x6kW%@>~Bnw34*l0;yNL z4*0V3zpxMw|DQ%-Z_oeU$Rn5NFsqeyYV$FU*>{oa{@+SyMH6XRD>TTQtywKlqx&MF zuQI~Kf^i0Od)c_Yu8}QDCmq*U1k_|RkKoCis>N4@(;$HOAoq0EW)YQ47x6X#xq^T? zs#FmdRZEP8N&GJ2=cbhUH2^A*16VT_8Xm#gEY2nzem}3HQLo3O1wlaA-ekihjghN^ zx_7zHt&BtNYb=K;Yr=JLzD_bK3L$y#v_R%Q`zwC-?5am+rGHJ#;=B(h zzEL_?zuYrPHbUY(AV0o@*dg>vC0%v^eftJ``6yj=0Db#vJw9`o2{}ExG?FB{V&PH4 zdH+I=9HcT{ZM{nUk4{Wz71vr9AC;s_h>DMOdjztGUP`v04GUbY)Zfj*+;G!UH}<6#c=b8$!u z&QSY_dW2U9Z%xHTS43Qvqc`4~pMe(un{fZLfR^4cp;Ed_Z~_2m%*L5@9Gwm+o$T^^ zQMTPTX7ToBmlnGxK}V4c^4;=%-)Z&$$~o={7u@hIR&~8}Ur0;12m{Mxbu^EytQM>#9M})k+|NBNBSNp%)#USsVKn@F`n-j=e zU!Pas+uyjyiRb!P@k2$JD4Kz-1WuTGBf;h+|I5Vi-Km`&h(hg_})#OnER>%O7prA^jib>I>a5MuI z{vevRSZE4#icSGI!LSOIiQt0Ke+BdvDt#X&82wjzVKxQx|B?P=GpJxEAVJ1Kcb{2B zpo-13Dgsq{EL{<(;w)B0U_XUF=!7u@ZDT=POf0KVSsA()G;j(PfkP}BuaVbM9%7voD3xqCkByJ#R5Cl!75*EmEn8k{8Ynrd;@U+HJ ztTDaIg&hEdOR6a3762JC-2VDIXHg!+Bs`F^S>(m|CVaCTfipVT zbkV@OTW6hT;;z<2ci=9==d0U@4&~7hUB39l(^s5UueJ3wO+t+FR$Ugxwcu49QC$7vWUxt?h2By#i+|3&Q@=} zOFqIx`^3lFDwe_Vh$j6ifIMSP7ZsW#Gp#4l)%p=+`6ByQdW$?ND`8s z1$ZPSXC9q_#8|e7ljs`ey(rb)bv9>Io7V!a$lY=Wgt{r-zkO9CeFVQREGVyG@(T?F z1=Kmzm75NL-SLJES~w7B*r0{@ zDlIlCwRL>E9z;Q~d2K^c6nr>YsC>_s3r$)+g6S89qj?Y7Usy=q;`hG-jH09AJte>^ z!YK+mZk|itY875N6jUO|2KV2f7P)0xDeOq+my4ySI*j*>Ero{WBd<&0X~}z%m%_l2 zyRdnyQfNwYPQIiHSfnfth3cFSI65oFnF|lhnPaQ4Rnu*Hs3R~767~vRo1{&$bTWml zoym8;NVVOa3%yk<^wj2;oTmr_lW$_4SBF=`G0!K%&#Qy-%mwrsV6v}1`ciJ8SC(LM zx(7B02o@xl?rcN1R}Hem`nriPxRl<`v&|xi#qtb!Mdk7CiS!BuN9I}kTDm0Sgy9x~ z64Xs2Em)L`ZXe`v)FIO}59gJ@c_SYnPU`$-n=xcrJG*#&F6Y5(D0r>*XNXt+$#9A6r&KGWfbnkJeI=C*_jLc|Fn1F+vneYeE9#$(gnPF1hD-5e>Li` zFpv0;>bLiQ-^}yx`TxpE=zqfL*NHHC>Gbq_iY4?U6n%F^htKHtUL8L9F;eluX&f$VQSJblf1@J&ln(7>nZ-R@ za1MsbSN_8Q6PU-)VSabn`n&l;>y|yxeXSeqbWomN5(rnZKH58 z_k-6Kg-ug{~{;m<84ws~2HqHPQVDGso9${-3){Ny^x)NvHz(Ir$^C3RkOad$NQ zKj_=cPm=iOYj$aNV}amb^<$RDgYq##-bYD*>y1gszdLy3OFHOV)5J)U&V+ahB7|g& z^6ALx&7eZrND|YN&MedN$A_imB{nL#%94|&Rz~*dj_0Vm1za;s>d(P$z; zijbBJgsUPN2D9J1ZXrqyZob&cBM}v;3(Oxl$&B82Jo)@Vj zBO{dRP^CIDRUM(#L~6quDMK1-Lqfb(OAu2;5Uh|uS6Y&CWIQU`ywkaOm<((-0xt|5 zNarVS@D72q>yocM>zD@|m?48^Ktc@AnY<(gEH;)Cl4Tf3jDa{HC^X&ZbOoeCyCjVA zp!mO4uKh&EpK(RFjN=l~#d}@fS8?Kz?|?ikO<^G<3vL6i0wTq1=~T$uK<&l5P0ezZ^D{Qo|DNjZ(B_&DFqXFvW4>5V-z7cid++>CAfU&7GOD zB<<;B`7dnxF^X)E%fEW$A)mP@pY}D0Q;YJXh|O7SPNcR(xqRMQkUUo=@6UAXP7cPEfX(A zTX^!l`x2?r_`LBFsS3;ITX;U-B7A;Rhxq=a_Wdc;%dH!X!7Px=*PrL#WTlK3VXy$C z=}p2b$}NZds7tbCdijJ7wcS7D>$-?46k&*cBPd`Zn0>u{aads8pZxITAA{9aB(5$z z{vZY#2n$963gc`HMgk)(n8kqJ4lF@{W_XRqR|%g`ahh|1WV7=6D`#dPbil(26Tz`6 zUod>aMj@{V7U-b?v#~;gLRl!$=%kJg;sR!j1r~xN36!RrGkfZ@5>#>906ZZRJl9bH zq~T)*&W4df5OWq_3>1!eAlpRH?rBQI+H-YCU{N}4 z9#he1Au6*bA_G#Qd|CLJGIN}j)dmN$w>pJBy6`b!`H4zOkPf}A9HncR zPWO+{X!O=n{t}cASA~}{SP7FNRnE__$M+<*w+$nGf6XGi8Hl6|Wko?q7`lI@F9~#P z7Btv6UxE@}LOQDRStN;pM#$t_&`G%|&_GjozBWN`tK7!I*I^JWY(T!kSlYcG@8ru& z-5eDCl4%kGBZ-+joCO@a1izm#7)}^C05O9Z(3}}Eo<>}sXG%r1X)A%=ZpB!ZA-&PI z6141^6qS;a+&;zYSBlQRc0ywQ^~G4gdaoa&lVhV&V`EzSXgS}y35ogE7V0vzSDKbu zPEKqen{V4b>r1Wsgdj$R^ZG3=p-pUZT&jyDx)~koYnfjtjS5zov9dicRjA5}WL!{rqbuBzmu( zU);V|pfFMY|Lm*js@hlRW4e^Q4v3JH08|LaXWZi$^_m$^MCL%5UVRsje%xE!z4 zs!)~S*(2_@THqHNqK;ICG*+rLsp>{rjYg{pRW;IthlHxbLNyITLbV~{iM6+@MXVz- zx(Ja)MmRysXXO`=M4xOHFgA*Eu6KfxnH_~@2oq#EEx^?wjnpBAMh2rg(ijFqBO+j< zMj>Iwa9j-|8{-C}AwsQ(P!p++3=a*}ghm)N;fByq1J(!+HHPYq2pFVH1O>IgL~~X_ z7ckQnsC5`FOay1P=~V{W5{#}b;B>IJn{7zL$)p+Fc|S}nCI0DBRtXFwZNrrm^3|0p z2rg9OUr2>Uzmd0_+%-@Mpamm@VryTxKgMIWyj2qKEpJc za7|=mi9Y|+3-w=mJo$e<16jm2^5_5R@KALl_xr!$jo#+}dm~S+kso}Fe&jVM)(~rx z9Aan_AJN_(8LroK3c*vtiEb&85s95TMZ|X|dtnrAPwhZAYXeD3vD?iEw%UgAe>dGW zZ%Ut)`s_pPzPJNB_L>u9?{V(Q(Y6W6&(^PBw7u%Mn6TpQiIeE$sYsHwwSKAWS5>?@BG>0nor8Sccfg|8|QcQ zT=mbr3x8YhZMk@E$K4k@cD%Uo_k}Gxta*R;d{U##-O>YopM9pwqHWdAY+8Qx@bd{x zgDSRQErZAVPc2t`GQ2ux*Ex#*X!n_zZusQSXJt_X%FRv#Kh5k)_22(Va=qUs&Y1V} zZ@F8C1neJu^;G}E3qNKQE#E8F;+kPa-3@0-bWa#^#eZtxjJiA8FU%>SN!Ms@{k3@K z;Rn5glZiY{yQ!l(ZrihBN2IQMK#dt4jt;q$essu#>?-qOuJ-O6+VEQS((0v?K8~u= zzFvh&3);6iIQYu+o_|anjt_pcu*0S?PtNC@-a7UDV&iKM&COi-&86$dha?AQ{FRXx zv8T!~w(FDb3+DVB)@Z=9HD^nnEW57X(iKN$_&w*K^LDogzjGAy2L$$==Ejc}JpVGd`|!Vlu11Yp@+7G(tXg5`s0$Aix~{PWI^rD1DI@B6UPAD2sC-lM5rYu2NWz8ihXx~Rs;Bm9vT{$K zqcwJx1C#vYi;YuMo!qh2(VJy!tY0!Nv)F;=%S%k}txPRrpn@AuX`|aUf9AeL_8$hn zcPRJZrqw?jO0LJ zy?y68%h&<6hkkXW((`o(mXG_u_(Qv0#oGU%sx#p5+7TPo@ZOiI4lBZ^L_`_4B|WXa zy!egV$8(k%D>we#JnBr3vLy9Qm zU4E{4@#D$&vyPRZ9*jy4ik`UZ+-Q4)N_$Ixl>BQ}_5KaJ4C-_$q2|K_t=JoTb9O%1 zz4QJv&9F+J#3>#Qc&y529qMu6t?NFlAE+<2!_+%yX3hSCOVkQ6 z>`r~~Ns|Hoo!8F)VcUaYS!?@G&x+gs?f3+L)0x`%(^LIBJz8F@Q~eIJ+VvU@rgd-U z|HtGi12**1#&4{BZ2y8v+v`zkU^-o#71UD_Z^wGg8dzRXhzUagJQ$ZbZ z=2DA$gPYFO4Cu1G*u#?R(qE)D=0bMWYw>(CH)F__)L#!Bi}_^aCx4gUpfSen z92NON6rhyB`w zXeQNH|Fm!Jh5Kt#<31*)@0}Yt^6c&(F4cZ9{M{cv{-^VSjp?mFKHsnLk9Wu_%e2?W z<2Bej7aJYy+Uvp}S3ca49JFE0IK_$@`Qnyt zAb5DKT202ZoDouUOM~zZTDb4)g*EIq-+%hej6H_Eo4-z8JFLT87}()>$(avJt~)V3 zjOS0Q5_H@HV!*7nu>%d%(QK2it4}m1e2Q3*}E$ zXxZeaTiyFSn^bjN+l+O|>sM)OX4m^@xT5~hxnt*Qs^7Lh-Z-^)6aT>83iZH0 zN=}D*RfXnP)e91+$gfX4TQs6U_@k0*j*Y6;V%9%LD~&joot&b+{_TUsM+4_|>)qx1 zYEjqY7LU7?aB5YZCAU6VJ4;ckq@|N-erIEq_2)ZHUD4|P-IzU#zbiAfXL{@UEh3L^ zdN(O!-h(Sg)2x--e**1<{cdD(OxMrnzVmV3#*zU#y2P#l8@}9HHK*Z%?|R?9|LOOY zZEbV|yB$AO@AkP;)!%7x{OFl@^T4_GuYQ_-^@n}w4FmFyO{o<=@z$PE?Kbw@S$Ry4 zw0@13SAIA+t>&}g@ws2w2X<_z)c!d!t#pl&{v)dVv|vb$3*7QA+UG8Nr{RLDeGZSG zG^X5|fa1%#_F8$=Zsq!%37fwknC|Zk@3g(D$}> z_}6=jhHvUP`23@0s}%F9jy*hU{~*JN$9DCx?>e{M79F^u=HHWc&FFBnp5LQ=ZSSo- zSEA{qtwg!Q%j_dsZNHo`a`b9q-yrpz)ti#8tgbs^Pe`lt&A!Q5vSGxY;O9;Dj=C{2 zr_)dI!v{9}qi%!Vx1x(zcv=Sz>b$8J=KEbLEvS8S zGTgtj>-7b7(v;)7TD}`_x6HtRMx7RZwX*y#V6*BS?(pB2jt^V(^!noKIXlkKow}?$ zrw%x^cI2qwN8|qttJ*(ddFNIX_G{|a$A<@Ppvxz8*-ktut2?^vuWl!n-IPG3$r z+OtN>*u0fHz8(ozv~Ktr{CxGoVYRPa&KXg+^nuRQrtqfIo;0wH+S6+EJG0qlorX5~ zVBFVBmQ-Ig{lnNfyJ9-{H;i0J_8xz5pZ)QTk0+R}EcgGq%;KRf8@As(B5_2cuaE@ z+daqs#&@55_X#(-WkUCE$}FN;rEiIPqv{SBvvKsp{ew^XmwAzXqv3?Ajm-lRhCOR~ z;Zsm0p(TM&S}}ch`IuG}#}93|;8s~~afAMGRU4nZH!3ZxW9Pwbht(*X8(Dne#Ye}6 zFTQqh$lr5yt*6buj8!_+djOTSTUF+lsjaI1JnGioVZFa$h>@yFD^e3e%yKFj#>;=@<`mJ=2L-7NiGfyg)nKQ3Nuj(n!nJ4R?9k55VIbUzPwn?(N zL&H+rzFD0);`HI}krQHfkDn5xJ<_cBn468B^eh`(-|t;6G@{t;3cr-@adB0^sD3+& zy;nSIVBV&`TOYf+^W*T6zh@PTO&@mCpAMVb*$8F~?LFHV6jyV@>2#HjqdwpE$D=JP z`Yn#n-J4mn+M)Q_Ni(@kbHe8>9=YP#*+aIL{@auQ4us&DTMT|KHs zrOHtkl^32R&(tp&w`a+?-2<()$CogYhqwMQ{qW2a+7s=&)nZ1awry3v?X(Fe;i@C* zuR6b9sZHwCkN2#&Ik;`yBF(KC~w z7^=40^~3a@>;7qQCFaDT)6HY^GM=SYYw==B%34*;C&jM}95_DiX>_gl>gz7Vez0ML z--TblZCr2hwpLh+WkX(Y_bV!f>ATLXy+-Z7ypvyi@!toI?f2Ip`u6g%Rhr!0GIhAQ z)0p{J0SIsbd*mjO#SH^^GADV?M)&UF`et=cRhv-rj#H?R|)DsvRr$ zR;s7()@aqOgmUF=_L=DyG#59o+C8AgPyUm;Dw4-^8UMui<^Hsk#UH6kcA2ikR#e=3 zV1B>cormpHn%lb8>(l(7W15FwpX`n+{+t@?zisM|{_3b_%Z9C(^?lQ(-&DVQB(dA8&~_(qzn4K zOI9ts_bIe1%f8pVU--agist1XKJ9tl7(TE@7jJ)5xp$jQE^ zp5-a9o=@+!$jPz&vS!Zh8mBiBdH3$FIlJ(A?vcmuZ#n%ux7_`l)R60^mbVB!eQu^g zV?1M2*K83x$2j=Zr$gG}6>nr5JTa@5D)_6+0~HUKxwkd=+|Cn~+HbNYq%;~(_Taal zr_`xkVp;be4r`Tl`dqmhU2eq+ZQ5{dt-q@0TzCZtA@uuywTqeeP~AHKNRo=eHx~81|RjGkNU!V^b%UZ+d$E9KU7<>sP#A zN_$rE$rSbTjHxvvHB%>9Hr{z}+tr|`XH8qxANurY>3$c>YA0>@^}WY?w@+jSp?V zpSiL6eD!^0LRy|PYS4@d|TH1#Hx$D~Hv|Vc! z3?KB<(fXBEjNY)JRmky{r%G`zdTq`PpHK8XfdtCw&UMc?N()0?w!x--#*i8`wR2%XJa3AU!2$XpO~eC&);l(^uwpQxw-Vs ze~#SknVJ3K9ya^n!Oi`S8Y(}^`RL}Yz^`kSU62^GL9=j=Nzvxd!@6kerA zT22}BdH7g(B4dv&z1q|tADr)Nn0Dyw@AGm;Cw<&+I@|2=Ty3JZ;q)UV^6*VD*Ft7j zJk@36S=F9)$tCr%rN>;paeZRi7r|?ejp}!`>!@03pB&vW_v+&jchyyir|R)O~~u4(mt1#EJ5%O>>*^_+)O_cp%&&x2MQ?qpUOIxyr!WcdxVe%W3! zar~jDmpANMac6Y>yUBYa>P<;i>!NP8%3E@B#(-$kg!O4%Hjp*zwqMZhk10okW;~tu z9<}rAmtRcYgn#>o$u{cADC6^s!KSOXUKmGjfq#GaH2KTr`nex|ar(|L=LTob&{+1A zIP&9&_x@?lu{Bb9oLrv9En46E$+i8rBetxp)5jL@!>r-IeRBWwxHE|-!^>PfZ7$w@ zX~)iM2WR}+WnY(lH&2p_jE|JzgGQe`kTUK_-@jX~JJ*uCJ*{8Wy34Pu>_vX_C~M8y zN0(GTc9}3}%9Jbha-u)qIrQsx}^;?%0VFP=R+GUayO#T_qxd67Hr`scUv*5Azg z>HfiAif=4e?z6i8OiXM=oxGWz+0uN+TB4Yx_3bgc8*MuMbT~Dv__;IVKf2vNb?uO^ z)}NfTeo5A*sLHg>5BIAxc2JFP7oV|jN)7Wbrq|UcWb3jsO^Z*hUTACB`TRib z#!fG)ocp;(@Ssg`FIv>P8@w{FX|>yyt9j8)oBwe*x1sH`=ihw!G~WN9;;RbJV!u4` z(+lmiZX*+NFSq<2q%FF1%Q~)N+QnIQYGibI-l*M#h3`Dxuw(l@fd9B~$l2f99&c8; z+T`WSCtEsw)fn4|-G3Un>cgcUe^D`l`fB~0oLyROqLF>FZbJPT$@}juqbl7y7aP~3 zVP20vSJe1^%fjXxiRZa_f3KGNBk59fEG@(P~@>6$STWL-R zu87?p+pgBR%@^MPZ1R=qb>}af^lmZ#D`AOKsVI$R;hksS>{sE(i*1hyyzC$F;q|Gi zDxWrL*|->WXa`+w_DOOWwV_&0vwGdPe!u$a-uExEBoY!M|nH$K6skfGi%j}ymI%$Vrk4zf^1otLN55HON>5TF{elw2nuRJ*J02!8@Hvjj&9UiyScKv+Vm87Mk zo9;;arE;+iBSHV%8xbE(SloQy&sMtDu}&dR+TEMKci~^-t0wH-(xhF~lEa1@JMQ%~ zl7C&8w*UAy!bS$-2>aY^BYnZ{rT+kpRr%2n-dN{!{-NM z3r@_+AirO~?FGAK>GH#sD~{~9VPLJy56V=W|Mji5*1r~|9eVNf@a|8Cj4+((vNLno z=Z{9PXRWpqy8Ej)##ddjqSf*vljCn5jh_GIgj2Qy%RfEQiL0e|tdR%^+ja_H{nVo1n-8rzG2qyE?2AF`&TPwx5gPSBEs% z(3N=b!ITY|M?0=N+2H=8zecX_f9gTtkB6pwn{(n?$;=JiPd^Cxbx5g6Wfyl?U-Pr< zg-c`R_Iv-R?UQBQpFK#@XJTiDj_g_Tw*%=EwR&U6ubNe2Mns!ezQ3C4aHx1_`F530 zCcPN-eQ?KT4`-A*c{A_YFTVtjedpZVy!Vehp8P29!f!h|<+a#%{7l{kF2wpMVS8xH zjcZGhTTb4)yZQN!%$e)1=a!4ByC|9HmuNW?mRD}*`^*2)4Z8Dp-uw0&r`jJ}x8>r` zA8y&QH*4m|!H+3pkk*uvFnmMv!zHdx-9K^tdq0k?z+&v zs(9w@wS>SM=t+W@@Cm>jkoqR+4}6Lr_Igjwxe?A zoU;oCs@5H>H7EIxJs0*|SX$@f>SfH}#Ku zo4MS(%X0lM*JAeXs`Jn3ceX9*{K;q4?A^u=>b9^#qhYz3%dSFl2)KFUyIM;h;4H|fBKx2pJ^IjM?HZ1?4~%EfeBYuZ1}X!}v0 zj@)(aCwImj-_mU#5&hXx|Ac!hMm9cM?3_5)V)OpfhhhH@AJ6k& z;R=<~z~HaT2`K;b-_WoyO{m-XZ+Jvl=-cz(H}N>4_%qN%upDF8I)15~?jiEH$qcm8 zEWy!yQOrQ#UVSq-%%lYwYLx~#16ge(nIz(-9@jf>AaEKy*5iqNO>~WAc?4Xw(+;K9T4z z6D%a05B3)kBPf#+z2bW$#8PCHG^NCpRvK6Md*I3stiK~Xt;L=!4sAPV7fK{x7N}6T zg?|Crefd}Ig&^7dUkc%hQ!2)oAg2X>l1o&y{N_`1ats^xb1(^hA28mL9+D?wkjS5E= z9s`5xWC3j@zjuGYuQ?82lpRno(;55DO5+GBZR3RdOTvHvngXoZ;=Ft-CG9p_bfW*P z6Ex%uINxDP41Os1a0(HN1qUe=h_zTbJFu~k1??KMq8Am2iB3`Rz&Qhsa{Fn4pC1C5 zVkJmg45=qfu|kTlCuYIOcg+N&Rf7Dnlm^_YREKKSstBc39ib3!w2C)35w%M(wRjSC z4{xmkN;PN(C|VdD1&XGnb5zj+rCI<Cz;f0T}~r zu@D?jxdOF8_iU&-juvx|03$}SP$ABQl?NnVlM|Q+;9O^(6(9;lT(ZmwUjUSXWBQ4| z1PT%V;4qV)(3o?67}~}Wlo-xa;-P@GVwn_b-pbGh$g+;{;YZFO1h6Lg)sRUu=_qJn z7{B_Rt6OxZ)br%;n09pSMO^nCr7JN04(~Zfj7t6oj#iD#uii0tge*$3?7L2?t(ETL zxdL~%$fL34`=*BumaVsoVcSs1Y%64B&>X&yBIZtqP62%}=PigyNC7x);pu`l@@rcn zbC8J~${nLi$h}0Rj7CCDtz&7Bv;kf3@|Ruy5mOfcjAaQE1##iNJan`p`0TSXsBXCq zq_NUEJT(0Um@Xh?7v)8W@^O+AimWA+W>ZLPF1=Io@7%i}%c8)yh2f}4h_0*4d9kqO zgygG2_Ij5PW&VQGDy9{j_c7-yz4%Hmkh;LGS+_8ie4C7o0+>O-L=TNL14U*_u)-dJ zE_8|P-Xln-gXeR!$;3Cn?;jY&5kShZ&+i=kU^&R&D&!gvA-bpJm$-o(W>QQ`9#a-_ z{Qu$!Dk=jb_(29CfM$T&6&g;kj19>%I=u}?o3(Ri5{E(JF9y${AZg0Pf0jl;z@Ft7 zi!dM3&L`dEbwWogN{R`KWDBVQe*F`ak-=Dwu^BiU0|7(xhzJZ0qi|rwSQa@SD};;V z^1>*O8mp7#ZkH#wP`VaYqQb(olLU(b6o@MZ1t8`*xE@NknRpJv&=!{kWWgxHD5NFu zcb1c_B7fwBF2C!;zkyT+!O#@q`NI4O0sRmQV{Itjwb4cb3try2u7;il0>m;zCAfVNz)v>i9qT{eczNwz$ruBmzR|0?+{eHZjV^nL`0br$Y+u zEtr4+{|)7+15rtF2o{amII|8F^f~tq9waeg$2KNq_ZQ@@n3f=&Q8idO;%NH+hC%Cme01N!nyb5s~?}JXnB2h;k@j<~pes z3V;s$Ty+6n^y4w{^B7V=+K$Jpg6JzFIgo)dgs{5#L8YxW)Onf`xvIoWc}!&0#O#C? z%!*`9fs#ZONQkZ=U35T=Sdj{aL=+C=AWzOLuLYruD6V21D|XviTW6INFq%PRhUm)f z=VJr%;F=(SCu1=OHybQPv1c2IEun0S;^rgBy5eO=e>XT7!;=SNAtr4!0fbN2&3$ZyiFeGP)IvQ}e`+M0wWz?;cKxu(rwj_S^RXHe7 zWndrA@}*A6;u`$1V~r;B=#>b zeT)SZO(_|g<|NUtfk+tyN$ZTPn9oQZq7f)2(+oK?8aQd+aO^A+tKp#&yzES55F|7~ z99RV2Cct#Qm+ycMVJTA#H!rL&aV{mXMBJmq2sDzu&5Na|7Or!=(*nOhKWSqS*bhdM z&N?hFzDlYpbOr}<$h~~~p1ZkFLwc7)-Qb=h_^&5#0_7{N)(wTJt2JN+yC?V{|B4qT353nfnRt83T)lD&uX9^ta)}A9%P;_#Pt0seVA`G_+jY+Vz zK*M?nCPLU2q!`NPi-#zsIdTQ&z+4l;x?;}J2*3P$z|lf{DM2XW=ag5G91(I9&M>Ju zRL(c@UZ>ZJjCuyNUt$ggUETx;fGU#;q;PByFBLTKY|&+IT=;<=nz0OmMk+H&V>r-@ z5g;T&bk7;!kJLew;)2Ieq6+>c+SO4XTp;mkhoeDUUW;EidSn%XxQ+*@1-M=%D)r(8 zIDZet%4#L;g7ScNO^+6lWX_60y^Fdf*6^WnWr_M#A8%Tii}oV5%-OBbt*>bNMslIB z+dG9}N#1exT~s?nIhco|(>vmCcAgXf;pc&9Q zI!PSIs7yAIX>G6yBN*j0I|(NqAOOo``5GpJ70ATt20XqN+D36wha{!RX}OXVQB+I| z{P-CuM_c%P zTaw6m5;EH?7$xT7Ld5K_ok_Bfq#W_@9oq?CN{8VVfab|w+ z5Raw;94A-&CXeiv0u#4l7M;rhxRYf$EgbwrU^tEp60!ji=Rn%*1-lM1=kjI6G(Tt? z=Cw?bzlwHJF?5A6QSC)rxIibFX2M{0j5!7Qy}}G-(}JjjpoF+?UXmO;&(xd~A&jDA zA@}?)TID57#GZ*pc`?WmodKClhTu3z@w0_da!Y%VFfzQr=y{r;K%u5MXy6qj$0&$- ztPy>p)=|f#!#{x-3q^u%D@ZE?EjM=<36oq9lNZqH{0oh6j$h#5@I=e&L9*1$r znyTd~1vSf4r3a-;Oh8)nh`4n&h7=r7w86nfEFCJ*CnaVjRH9Wz#by{(uE&oGe~@Cp zNIOzad2PWYHwkkzO$w7JMrskf8c>X2@sLR5L;?^|C`*cS7|fB?P-sA^fL0%Rh0vCm zv!GJT&cX{&U)mzY+74Tso2kkJfLL8_ic|YJKrr6SUHQTAw86#8Y;XkxK=g(KS=qsI(d(c?tQ(-0|CGCwRFelYW({(tsf{kBk&uIOP!H(ZzAJ2Wo(|kji$OvB3shY9s7n{ zSUBMRqSsT&Sg z>hm-(9{RmmtLAV>ED$(I4P@@N$%}A%X;SjAm zOHN;Q2n60VssQ`;B7P$u!h?vXPoM7kkac^?MGz+z;Lqxot7k#6wNaNkc1L*4oD4^o zP1Li%na%9k5*t9xi02Ah-|QUgqVOu!nZZ6`^`5_AHRDmlakLrVtd5Af5kNU3_lm>M zAzQI+WsTiJ--_ZeeLj0@_!B`&aXSIYw(Pi>Nch@Nhx#6h@PKq=BOI{1YB?POQJFtfgGgfDv_TCd2z4N$dyr=6l8EJGeNnj6xj3EUo@ikGm*b$Em^ai4I!PU zTCoDb<&GRj0(a0;k4T%2TL?CZw<*)wL$F)$R0MP8g4ag_BfJg{&3zt6i8=*=m7vnb z_D;s(=v-ke{6%^8SCEK*h9Pxq^&JW*C;Z^VuBRFLX>@Z~N9>)9a@Io51 z2Amjd_EhlU>F*m)o)p&sv7~)@d3rtH6(31g-bfrE2HPD1k2?c2E>LanJo(t+k=kWu z4vr&$92pk(8wia0a87pk9`be(H2A$N&&E@*`M6Rx7H4x|JxlOg8wdPXSfoPG?N~`9 zH?Qls@-aXojnODcYHj*4*YJo9DziA6h`pZ&kyHlB?A$vd1TqNJFA{x_DUuHw5{QH$ zZ?2<_kg-qaei_r_dyB;&5?bRHz8<&3buAZ|+v#tTlL>@~I14W%aPVa;(gLdjx=>Rw z4coh(bZCBdMcumxX|Ca8m^lZmf+ATWGLQ|} zK}@gU;i;pOQE)%L1Y3jy17U$MqBvWPrg|d}K`a=$djfZp@M)SK**WhKhEBf&r65sJ z5qjjugdnV-YEe;7mAX+hGJUzI=c54uzvEnP9Cak5aKP>b(fDXMZj>ldpTx9Q&%m&% zLlA#=J`ID(U6gzo|L8=B?Rg{W3e_jj!T@OY2WRk=cPwD^Csphst6UK=?RT*0m<3A0 zJ|a)ej~m_uz|)`b>40f4yM-x+!{N<04^puxsESDJIgYsKg0H3aT7)3@1y%;x?y-G@#G`s^k@9Kf*A4#(T_wZ z?=>0Er>Jr~Z;*&gIJ1S)+T?Nmy4V}aJP?3h=i@RD#)^^(04(qxn z3)&p80TF_3rPVvn7ugb6e7H2f^&Q^}Pb|zhlVA@w*TOJj1I&}$cf1wAQyfc&!6pf3 z@470N#;!nD(9AeobEnJJ!zP12^FN0O5E;KwN`nyK$Q>(^NgFz;8!2EaV7I_Noqtjq z!B2w#JuhYy>_vvcNii8gF_vZrox5pde?sCSpobq{RQ@?7(LcWj; zx(=`v6ypSs$3ZX>=Q~u2(gj)t)s#3|1lLPRID!QeLPfwm(6P5;3EP^$p_be1gJKc^ zrbXQDzLwe=LmBC&mu`Yx#A1%_Bg))lFLA==vW7Mv68y#t0aMjj^Znf{4jhq@4C<{W zxg8JM$RTVjwz*vPCxui2{B+V1;}m0&$cwgQs<=mQoE*VN{jt;!Fcm^~jy58-Jhev` z%~1|U<|HIgXl@EF!i_Yzzi)DDW_}T<7T;<^!EXLcctt|e38~h;=gId^pF$Z0O>G*g zMvX?krH%^tS!2LRsV2`R=GBvQ&i| zZB(fR7`e9D2S+yIfK{9P#$*_D?tvMW(Sm@MH7oDt6El-mG&DOrpN9iV@_ghm(hNVw zi1G}Npg>3WBU8zMB7|+yRlA1dXwQMGT7$GCD4909EH9%Z9ZW8~1FWl{LL;^L9z9?W z^P%$FjZX{m@^u2x3Op!{j8yRJUBDPgbHq%u4T9Hk-P{$);aTl|7Jx8akUdFQ_o8k>9OH`7vYOT^2O9u)>`4(#>)VC2a7lCh})!K^3S7#oe zcVLyHC4AES%qRXqLqS|1b*KO3o;K5$)9YKrZcXLCNF6q%f~&|cd~Zc03+qK9f~-70 zqWW9XxXEo&YdQx)d{d+sOtkVjjAqA2{OPGoofu$^aC*hb48!oL`&jrLK*$U5Gl+B z2ckha8TqI(crDoR$GjoQR!uE|j3NZ+SHgS0aI=G&5B|Qlg_K8I?u~r@0xmpUSf=0{ z9hz20cat#+>+d6dr}JbcYRE5$kP-eT_Ixshmh3=3sQJDc`i;rUo#(G+`H=J8#@&T! zqbAlCjQX!atznL3HiN^pS~oExYjbs>#+Ed&pKFP$#`FEACEfrnicO8gRg%F33z$W; zHW12-rypIEGrX1^wxP+*rxbdUHLKclAuaGun&`i8lZ4CM{wn4f{`PD#p^~+N?1UH$ zSdp^Q4{VYV+rb&0{e8(5m;!Ty#KVUMLXc2v`2>q7lefxo-a9Z||GRj$w)v zfK?{=F<4(BQ*vRpVejAe-l-@-%ehy0SM%yjD{b42JL$jUs*%)1m&!3YuB@;bekuWw zcTm~bHtnp~;5ZWfod*1kSF}4YJzAQRx&QYoL2gr;5qI=LPwRmrfXSxa4Za-MpSN*V> zU>j~YsSp?9i_FUzh%mILr{%(85qBzlFCdTb@4)eo5WOZOD%K7V`F&{fSXc^IfZl*p z*XnXG;>1#6-*I8lpXa;v z2D{kfIXXnYJ6cItXTkA;bvl(*@iKFfHUaemYS+{}kJxBqvNoCO6)TuJZ4Rw&IW>4~ z0seF+_W;BXuJ7#ITpI1q_d@^Ru)5`JgKo@=eAfKi;WGGm$pS?+zu(+mG$W$RMOUF* z{dEBo533Jq0;gk;C7h2sHG8z25Fkd%+$e2^V1G01_}<(|=#vUJ=>nc3&nXPRQSkA{ zJ&ar`Ej?k_A|S$6A$*7xU7>X6?zIj+7)~_5BuqPpfj^HN-kpiR4C-cUk1TR`#cu}H ziy-3&x_8?tH^T*i=sXJV33w#Idnv}BGPFU^$jpeWFDnqdY%vj)aFx8I1| z))p|6L{KEKlwfF|ILS{IuwraWvOg=_KNfY$Ej;T3pz`0STK8PK6`>{G46m~TASxWOGgooTO{qlk(+{UE#AXbKZ(GVB1yh=NM+ibuo1 zWe#jDSa@Oy=rs9N3T4sFjS1Tu+!4X{1ra{7iTQv`kK=F0CpC+F8*e4iL`$di;EhM$Dy-bgrUIttw>QSHoUBrrBL?h=| z5yXZ-ilKzP?@c}}=Ny_3oXDBRGSDQY9X|kP5*)wYUsHLZOK?t+kM zTSth2b>?v~K;)<1(O?hHl#+FWD22C4EXPPC+S^z>xKTrU$15z>!6EU}gcDn^jf%KO zfejd;LrNf<7LwAkX}s zQ)FDr46XPcs5?{t`&I)g4J0uXRHcnli}8-}Tk^=7I)A+2k9s9P(j`0S{c#Vc9{479 z)iSlI>b^5$bp3L_+q_lLir;Bq5Au0EJrB@ia`TI#*^p?x4!Jro+1X@4y{=hOfc2Ra zA)(xKzBiLp@fT|DpuGFJA{>~OML1FepB1xo6$}?HSCh?eufo71H>H%`gdgoj$D%6+ z(gdZq z(51NZ6!utcsnjD5xutc5URdxOEFEki)8H|dkDop#Nc$|Dz8(gO;f!{cETV=e!BGdi zXCx%7KW-a6&r>$-&CYidOP!}!q4!+yK(Ep_D=Y~X^G{AUK#(^nT}dSaUC{>H1p?4o zpoht_#L1W$2uE}&SPY|pbTDBOxs`cv_^1jd&;6CX6jbmk((sfotP!-0h+Q>Nj z+v6Glj#(a@X(K$9$EWmS(rdIt>nP!Zh4}tlkq?M5wy(boJW?WZK6$Vzc)vMtg|w;| zMVSLu8u%ZsXgZH<;Yb4JT{ar^G5SP^v0y#<%paq5{VP6xd4|O{UioNVk?^Z#U3lTg zwI*4BV@;Pb4jw3b*|8#Q8#FjvGj)1z z?tjGqRg3XCnD=&c9`WzOUvq7%qk%UB?oM}oV>?AfWv_ckn4R>TTI>5VWx1{)uVR$qvPqDDF9dl0i9l?5>KgushpHWHeA65680$-8na# z+Ppv`p*-P!fak9O*Qd_(ubZsCkV|Wk6^bOf+t-u=I4=5@7^{IvvkSjIfki%oel~ve zc0HQ$>_jg0`*!^=I2!Q_p#pcd4-cn!sg=4u8VNZmOC=I6deu}@nu~3EJyA8y1ma$t zul=}<2n(-~$aw}6?sSQN(JS>HesfLK&_qGRVWr-AgQ_VV zybb%SLH`A@e_`BX@1BnzBL7o5AUTF#%5jE&2Tw$C`1DR=8OYZ7%~pAu@7T4asI&4y zKzSmT&6zt_t?6PpT}`y_YmbDL*Azy82krIR1hC{Brh7z$6GtMMt0r}=B%z&w(Wu%j zH&^M^<6)-SxOMkrMLmINTT<*Q_gBY$67j8q{n#XUiGR|h1eV}~=HRB{Dz#m0^mE+z z`T95|rvP(i9lM?=fW(L3Hp2&e^jmFr`9!AF702wA>ZK ze8%Zml+>&osM;-35IIq5b|P|0o3h%g*LLcX?s9y?BC>KBNqdFRD!hi`V?tLUgK5x_ ztrWL)Ldw6;#rgKk=tJUaO$_DQ6Gq_oSd>DOjp3-K`1e3>LTLRVU_A0x$HQKZ+Q->< z_}Cji5pXQE@@bisL;e1QS}G7fdKltX+o6n%(6dY2`+Tu}9!y~{BJ2|Pn?PH6ph4-| z9o@Ye&97Q(l6!|(X5D~>0i6nptfPUym}#!u7zNyucHxga7A1N8t@0=XZ5uZVfAM)i zgTLyA)Ji1SH3L8x&dq=k!yVSo+*D)+~JmttBN4= zaCp6NniJUe@Joj0{0{aD(RGlOHcvY~1N?bhR(#;&iJ4FYbWjLnv!zg`H)H+52@Tf{ z@^!fuXzEnblZl|>+1tG zeaf{ur;|Pr#kLjWPjyGg7q+yL;R*gjEtu)H$=L?1`HJQHXbmIl%7!Nzr}k7%Yhbw6 z63{CT@yefPp{oOQ_06Ae%oE702l8S96dFB+-vBaa=;dyMAn!v3+zop)XU{sT`tNoK^XD@$2>No}>Olbm#2nql`GvFkIDQ=4D*r_5>WeF3Crh z^`ed!$W%jW_rQ721n>N{TcBE*Yj!+Pkk=;)=##9V1`NmLTJ|qi18faw0nv%_{HVU! zb;yeT`ZDABi*uPArVt~D6P^{nF)B&xGDQhhfRTumvkJB`OzD97caPlKziPI8FJ%?n zj6tqe1BZsxNWVXhj%dFxlmts9!u76~GlkZg@)j<>Z8@5ERJ0JjPcEL$7;Kh{F3ibY z1!Dji1Y1I;JSX{A}D8(l) zuEiQaYZQAdFSy1F9Gpo^qsxNozQ)uWdk#VBF~#d(U9`loD(>KvJ=%V|`dY0z2X1st z_4WF3czAnpwDbFV7wp0J1~__0_gU3cLe=d)+3V@*KSJNsGSw>B@9HAsgc9$MUAU(O zyE6kX4lYkX?*MDR%%&=S)}K9>QLVdGCB8pQh%5$;G?R|B!nJT|kc;lM=s+p$b?8CS zI!ARBH@@{^1bCiz7WdwD1~p{(ko&s{+Lj(^^fXY=(F<-6-8V@C>r%b8S{fvPHyDpV z`LZS7Mj-KVlRRGe4|{Ve#BAL>?CpUk*%;b%R98#MWKj<%%p1E1lbe){MRSH%rmAx* zp3^&0b?Li9(7I`n`W*(l!<5wYzgv-itv*y2JKnu6!R9seBfC4cwAE&I-8xH%bq4z{ zAcGwM1Nb;xFHzPI)9|xuhKlFZX^I5BaxP-w9eA9+v1?dyw-~lARfia(RU#@pe<<@| zcw2N>?tQ)Ts-7u;$L1-YX%dS)Z6K>xz}EXaArIitug|$6A8B`&k^ry!kDa}n@HWf8 z8rf^=6#au5&;X;cJEQ}=EmjeN7_dHSyV=^mQnFg>DVhuyyxSv;&cKb2%^x3+p7nE3 zpZZYnH%w6f3KO|+y*=cO<&=H&5gjM(0B3X&l!;(7vC-*ZZfRE^z6Vbhj0vJI`U1s{ zVSkTae1FY@%IJV1i~Eq46yuHKFK!e9)hMa+5saGJoYT%&xpH@ zp)XrSXyjxhnEJ9loks#has`iUv!WrIHZ+&yzIJmgV$a(-<5OAK1$@tf?b$F|pLM}u z9^qv#x37PKlm>$XT!fs8R=IDG-#x5(1GwSM*g$TVH1)ala8~fpne7($V*F}YJ_T$R z_s$1g7Ki(m=l;6w%7RW<51$$6UTsOZv<38qKYPFO{up!PT=rXH z2^YMpX-MPrlA9fZB>H5Z^gn6*F$bs$dxeWz5`TttqTMZ#c(lWq`dSn-dWN_Ls zkOF62oiLTb(%Ja$`MBgYo`vK1R@3=b1nz0u+x2jwHPc@@G*a7X{J5!^W=uCDSo}A%rl%HzX#_IpCE1^Zt6Sul z(Mad2&-E*9=NBkj=6?`?1agaq+Xg&6ZV(at3uVoa+~EdZ;^rYYMR&M=wnAUsddF>l z<3Jp_49pViO~<7STp} zZ8%JZ5fLfGG##&rkX3HRGmXoUwTtNE=gt%xIeATW9B3X^SGn~%L9ss3T`~Nt3>;KV zU|hkf_Za}wuacRj8eiH3y609ezT$DD=+z(Vn6p8=xT2^xrIRKhu9qxe*m4&K_xjgGp7AJ$DfTO0 z*R48l z52@EZEjisSDB}u~yMME{{QT{8EcSMA&djBeg{Y#n!B?F-kH!j`a-cPSk8;9TX^t3U zh%DM*Qmp|r`~%b5(qK~nq=d^8;gv2vr$9=uNvyB&Er#YnAsIGuJr$(;ZFyw-Y*x#v zb!c8MwL5gONw-wXOl9PE4rE)c{8g7NGJw+||7J?74XKwMR6VrPW~(ZcZMxWfr_(R) z9<)UK4|Edc@$eR8CSTvvV1V)}f{Vtde8PH`T)^u=c1=ODWzM;Aq!$XWT|`F@fWxh) z5`${{-gk`|OPE3xaB2{;kj*D=MDh@QHJUwT)B2d&4-MMp+;$&Oo^`*Q`auS31Nw*u zc!WwqVQ7U@bzr11enV_Lm2CwL%8eS}Hqw_jy>hyeRaLNu*vvgFE11%cuekG7o+atw zGR`urjw;w{r>;`r$w)Ey7q}j8es9jgwt&u~?|trl>F3;-<^8Gq(#P-PCEl9r%tQY6 zxh=C-Yo$kwleE*+;zM3O^Tqz=eQlBe>7A17)6v=K_WBsI-=$jTUst@0Le^@DmxEtF zS<8!MuWYV9uEEA)QbW@Y=EG0E(`{i>&b~9^Q^~r4j11ukCV6?N>FN2wVQ?7-#a0hW zc+-VUhs)iSp9BxZRMpaBISo~Sb>k0FgrZ{jm=g%{E|6}|JXo?oc@>)yoI0pwJRnsz z?pq+0o$cwuY^rbQOw4Vj;-Q&RLra_EV+u2)Qn^yYs}5=&fd9P(9;`QsalEA_&b-{8 zWSP`fD|hDM8uesry=QppUYQfW#$464m!c4N{1ZpeIhE8c<&~jslHC+hOYK>UJ$M8Wgy2c}ezU@-*g#i8CWuyY`c_ zJ1dTg-rtv45;ddl+9C08{y+@=wl)9`*I8u>o1P)`>5>5)XbtUI`>HYw#KX};i6&^0*YUF+;2 z*zxmg(w=Hm&*;$w$j+d_hk^G@#_p#5ryE&|0SW{~P*$pSghk?PAWp7DXny|~KsnCL zBG%%7Me)cZq5@s7hn&Mj36j+a9{xy^@*x4jhuvo3CEd1SP8#WT=f3~Ix>+{G z+e4sOrZWSIkUmq~MmKW3QOKlZ_gAVZ);u+Te6Ae9AB~!>u$9UBjfsMbbx{n zu>h-ptjo=bo?1K1D!00`6)~<;g{R3^`Rp?hiL(l z;ML&Jti7k4|54f{e@(s46g82MnuHgQShJv>t9REGk>G_l-vRM352A3uhcN;QU|0l$ z+5MIg6k%Iv!?KS?%?FqZMTN2wz1PcE|E&EZCU3K^c~jW*{e}59dXnU;#h>r>nCjb> zxbIG^c;nh0y2rOr!z=ae?yTJvM6Jr^c<$q)@AKx zpk)F77Clp8QKGMBlm$dA$!&pwVbv;2pkjR6nS4m$2h_Q9$0T)7Rd?GxDP^}cd)w{0 zwlaoT?{fDcM|`_HFQt|R-Y;Bno5yv*?4T0y`#{j@?sGF!_K-RJ09Lbd0^`H$>=!lq zjEtQ|f;w-)C~{dM12qCV{h*#W+qaCSuPbc6=z4ed{PCxSb||jP@^nv^X4u~z*XBQ# z+YS}JOpruFOb4kk(EPtwl$0U+{Iu?nDAv*f;c-sG>hkH+;&(G2lr%{j{tO=>H)96E z590du_*RaX00=i%u7n=V^pBuN07K04JJP~i0a|5$M(7EiRkIBIP#9>U9KyfYxRBB= zioPl<=Pr?d&VldcA_Tn|*m(OnbWO6-fG78y0qt6xA{oLR zrUcUHC#Hl^PX2)^)-2t>a+);6bfwido1ou}o$1rS?UuxJ!#S{mz(sfaNbOl5TK_c1 z%zXt-;B$2SzKnDR_?WdapSZ>F(HM+3Xd^Jcqzj;nL|`eXMoO2pGRwUYbIh84kSAg0BMGCo4GEjRZJ;Ed6d9f77-@+T z)~sYQH)UfH1}6&pc7Vn_uN{L zJiO(!e7Ckc)6>+IxfR}6borW%nLGLbMyX0GxjSlAb}MUw)b4?KY|sW-q^=Abtjt!> zOb>27TD}E6j<)KBWg7@eXzP7+JO72}DTpVS1D?UPpZgHV8jO#)7PA#OoEd-1hCU!j zs#HLvb5hQ!=Dxr**P(9ZtBnMe+(L6PXOl@3IZ5LckT3l)AV2@B6Ks|&+=5a?m3m-D zwUCJlo8*}WmmC7X%A6ZnWqxpy;qfHf?Mt=Gg~6I)U@HU{qQwGFAfJ8s%7ikmX2`Vd z851B7fZ%WUXO8~$0qgto&(~cCX`raJiupGTMHR(u(zZ`xmV(oRl%3spNa?Og zrvNq?=9iR>VB}042`6!k(Msr+{?L+&>%c6ry>k###H%$>b>j(*p6tvx0cr>ge|@3L zz$U?dUvk}5JI7d?^Lg2n^&@3v!)vS9Xi%-f66@MCRd7xRHPinrZp@UiW_6ctkYbh9 zJ#<1oMV$w;ff^eK;am281zk7IQ_^UB9fX3sgMBt->ToweNa>i87)H6wX=wVpYM!ub ze=>;n_*pjrYF66cWP=X6;RR)O+?E-9JM1OZp0UQHwByhCyKBJ0P_FGliOpNwm~t&B@Nsc>t^flRG)6n^T(R~dx`z&!1-p{joMn; z3ktG~ey*;5_O4ELU$up&>CTipy|uG0e|THr@rAoRlUl2;rhbbatjO+9_4n(iV_-+S zpYN}Q`^wCJ=T}Bh0$B-{(d>77ynHnm^jvGE+*h7rKGbu@FE2@O=>fC?WK$~`3eC~J1 z)w5evIe(yxn|QQD=02Cq0Ws$M4Ks(4UxXe_eO8bBd2C!C-t>_acz4Pk^yBX)DBW1) zNwH~y=O~WpNCq~eL;aTR2spVLC8hc2sd;PUr>tD76=^V<(atviIV z+t-nN|CRuD@!8iC3m(gMJi4cEw+`&*tqO4xH4PW-%t**vv$vsN2gQd3TRHo8r0&jm z2Hhb}zht{TG7<2L3|5Q9;N<5S_{JnSM|4r>5ZMK!N&L+wjRL$9xL32uoK+!_(i8i} zElO@Mw#=Zwf)>E%xG_hveM$X)w{tXWCl?8R&&FGyB{?0jX`ih==O~Ew%{%H2lA&1* zC+$@X+vYZMk_=pjQLOWL`fYpden(Mw9OUx*3qn;o+qkbQn{X+<&26+WZW*!E))bm1EtyPlAOhnchNV$S6&y8GKM;IeZa{Ty8+U$({bFI|<$nH?8& z7aBX=>GDYDJzXqBHA=leyNZ<5qgd30{6Z^>xUL~V5Egpuj4%QWpt}0!; z{_C{jvHIFPp6@<)3QE9JMipxeUSv~<>ItBmi;G9%91!MwKj|BYL+ty7uXk z$DyodxtoR3Da=N+t2yjWmwaEbo||uwOt(dL_eI~UuJ?~L4&3(!-5XdhTek=#*j#Qs zrY%D0A9%9g8!JQ658uL(Jf2;Wl@uiFKhe@A1jB+VRtz3l()tQ;s=@6z+An?mpykWV zb|~$jMja9m^9m6-XiA>H>M+%g25Hhh`Hxs!8e1}-vgTjkz68)(2<3^ z{Et#JQU)n;W={$g>x5>#SUaa%%4D za$lTL@S{hbgolfWxB{JSLiZ$UCdIl`sw%_Cd2Xp(u(P{kOgeu~?NRLzd89&=G4y@6{O<>1SlQbS0OX>Ubn(muoL%=aGp zbVmdFgEzf=9F8P>+(YBxMUKnR!rO0`LK$r%NR@kY-8NklWxgd_z+1{ve){rr%zRq3 zw>)iUz=F!4D^=9oGB{inls2Tip~Viz`50BH{gg?`gg|>BX00fDUUq%?#|N3Yy)lp& zs1lM*;x=N$j9mnl*e8#37G;KzC7QcB%`bZvsyySEK^3EtQ;sLIE9Vm1j(bx=-kSDP1*O znZ2yL+m*$K-7h3a`C z?Z4y|Em;#O2n;Tn@E=%1P(3b4-0lzYbp1wS88N=Qozu4W^MWrWMrgD!w2<#=rNT`i zO+Et9e6!Lah5pwCf*BL<8|}Z1n^#H~%$X`HR|;E&T7j(jbBd(oL#QSW6u^UW{g{BN zbDUn!v{4)7VWihyFC65*@^Hs8qEJWJ5eOh42$L^sLpubdn_{6@IPRv~Q*6@91-D4% zVmntM=HXZ-w7Yty#%pLK&#PkN1oAvkDT9mt7NK5=q`~=Z!esw*RH+n0O&O4UI*dk$ zA}c+7)MZ7FBt{;i;x|QdhX+8Q4i$+>sFUH0;Br&)qmU2pPtyAR1AZ-|Qz$~lUKsrK zamNx(fiDQn@V89^E~t;>OC{X=A&i>pD1-Ag2ePuPMl?&xdblc+hUkZSJ?GrhISR9- zD**s;H7c^@hpsjTF{PzXYd#9SG0l;f4mF)GmTp*pEvAOBt?nYD(p)BJ`4cJC(_lhl z;TKjj#G)5sBDfp2S9hnQB6vMQ{R7USjE*YH(s_t(xfv^^)I~TreTsWL-hQ=Cabf>< z*Uwp0JYg4HEw!3xD^;Ez=|zVW*nF@xjE|qAtX=@CN_d;XU@g)DEE!IjQM^mcdx|p8 zazGsB&5G6c)#DX@NeO;NS@@KSry_!A<86cL#Y{1joz_UlV2#J63tyGLIa(2%o;Lh!)tNXvePg?) zaPh-}nWilui)eU0460RMC9gfq3(DtflV7b%GMx97y$ULUiRmG;-zX8FWWUCN5E< zy-twa>5W4ryBWL4z71C~T{C%5zT$O}QSTLl8!@HwK^gjx3 z&;5t!#%dhF|E=%@{vU;>HNdYZdK{=6lHR9g6~7JT0*_}@{vUsL{y+ZiLFGUG&MLH7 z#D@Cs~WhT`e$hyhFh#}A;O~~eoSCpsOj4-Q_OOCLN zQ`6=1BO318&`H=T;e1lfqdZrgR5`_T`hT(A)lb?5XYP*K5c%lXCMGGA+*+a3s=qqN z4^n05vN@!7_$UwvyRpg-yNbZZ4euyDf}LINjQDla&{_i4%92O43a2#GE@|u1^w6)EpH$EV1jI(^AdRI<%z3;JEm8ai(r|_Scn0h5!lUR#7jjc#=^a|6 zWiVi~PFe@=3`d%G5*(ew&Qd@A$JLqs$JKHF*VSdO*(qd~UQ6v_fO|hM4dNv&h~lXA zgC59KoT!8+JL;xv$S*5}Xnsx`E#=No;5)3&qa+I;2qQ^DP`t9xoG+k3nU&(kD_=jC z6x6d~=azFde3c%CHtG_<&9bLv1w?<#m{u>dO5KHIX^>|dZnbUFCajg&Yi5bSz)GP; zHk;4kJx_;;BPr74|F9B}mNm(S+AN->T3UKG>7(b?#^($v6O>|3_lkyGOl)}P?-y2{ z(_?De4eR$4gaqHn$sZH7V;V%i2)e43?GymBNF?w05c|6+6jb1ng;Ar>6V~*iJ*lf* zLM~xdc}y7^^iYO+<(c3iI{2SE6<5hJW{3j_5SFt|>s<^ZV8ji&@> zOcPyb`}b5w2eRSC&1QE|#^4atp`=3m`6Yvwllwoo9m}x4W*1o2(!k4E&n(SiL&g%@ zAQC!vl$X007px-f?}W(o`9L|4MtYQw91Y4~6Y9E;Q&t1qX6RC+HdJ19WD0aznakl; z1!;!vwZQFJO?UjzHUY@p&>+=;D*wfDF9NJ?43kNVQX-jf*BDNnGvSNKY{~(!Jj1i) z0t_`++c<)!_tmbHR>naul;xO}q{-ZwaIlvM28&AMo;>TMLVA5HJZPPLN=M^4bG|(+ zQJtV2BN;E)zqT#q)83sO$Q-f|cbG>5#(5NZ!MqzR+CZiz;VGoN*6uTILl97oBE87l zw9@6TOLr|Q$9%2}BL)aT1sH!2&Rg=fs2e?%X}6k2K0L?d4?Xs5KPsh<=S~O^dxk*m zJpauWGha>s_gRe1Hs@z8H^<_tvRcV_WDt4Gc-LRqWko+!qe!1iGY{a@zNLPAM>enH{n9a6X%PfAS7l}|T+ zK2;jIBy+M2)3To5n^@i0`pkc_?F0!D%I>T=W}IPdj~P=qc2~3SxQ(KiGIV!E7k-Fm zfXCvt8A=lG5%vC#*wi}+UbGeJ?&BTnsxA5G+z-kG8?&^m2|CS1E#T~IM79? zYP+5fn|{0sF2T*kN~Bm*L4a@qM5KH7lEs6&zD#UN5Zd0E`MnGeZ6?OWf=siD%E!(64?T1LkpdXu~u zM{JuJjM^n`QS7(^H`FfxfN!hH^0vxa2{LSQ*jrijZm`UU9Az7HDQ^48Pa{vWXlx}U zaJ5&R;jR|(Uq2_LcfI+~&qWDZC-%Qv|MPQ`k31*!|CgUT`M2tQ=l9JO44nf)r-TPg Jflz>g{2%o}nxOyy literal 42200 zcmV)HK)t^oiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMX7ToXzAIF4OoZ>VP*!GcIaLa4DIO+}>$h+?_8$!?OB&2HS? z08tc0?AUwndiLJ+?ES=k-r0Na?d<$N+fpDwK+k)>|DXHb$2aWGJoC)VGf$rx13}SR z5~0L|0U;rpAU#8L5J^i+kREgTPLEV7mHK*nv;QxZN^}0d)K})`B$NAk$$jK+Fqr?8NxUUe5sXl3 z5;f55t7x8DKon0cKusbLjZgr>VW5X_NQ+QHsn|{(ff_SXG(u8P(XDX#eo}&%OSWAk2tR-;)94z5czteZ1v4*1wO; zTlRDP|A+_GLs~>30)RxcC`FSdh547nvLYlb0ZU+lEodMplqQ&>kQ#;U5H$caq-FRC zOVa=tj2IS8V5r)p050L0NP>&nM6t{T!Y~t{42T-lm;i(Wh$JBsAT)r* zFxrS=v51;PXi6e-1^iE-H&QfEAq?j+vH&=C1OpTYv<_j{E@gn!2vdnfG!{;iXl6kY zl<~z!fr*EEr&bNJx(`%$Kj-mMg~Lq}CAxg#c6oV8nnCCOv}F0yFnC z8eo>Ghs1Yy%d^AzY16 z456BZDlaT0Kr90-0Zwp3BLYB=;!1NB1#l6|IBfMO{tZB(6uUqT2#FFfP$6_Og5b>f zQi$5f4jYLWAe00o(#MEUG{rFk2ql$V9R={we+>j>!qv6_m^Wqsaw!Kv4dK8*B1tS& zA|$gMff{0FkFfU8Y#=t?o6?|3_JGL-75EOGhdYnpg2myDiPoc zT0t`_Y&_-`Rggm;Ml_HSqd{n7EPx3{Q)7my{!Qlu>du6=vec z2sMPU@F5sRIdKCpmcIgsqEIc4z)UBYnG@O}RJ{>3OQQl*jAzJ@!8}e1PNQTXvtMnX zS^6dbvmPRADzR#W878hJGx}oY9f!R|N&twaA+?Tc34zo)8}7<2A|pi;dZmGYm5{Y~ zaY;l>kb>3oAdf`Eq~d<7c#7tjF2&Aah*=ae&5%YMKx(de6`~I>!<08M5`Y;PGnE?%n6kj+xowdhJ+}ZG^%MMi2#ydP~mtLV{jQDiegP67L;va zC0kg@!BpBVzCM`Pwy>Bj%uPRrQZ$1RzdUFiM6>Kzg#Z-SAtcJsuZAS_HmgMs;iv|o zXdV;3oV_LEf#69fN#HEg=H?C6Ba9}%@F>ctW|*YLhygvSUG@>_P5RGtZQfM%6izZ+YYF-2&m|X--IWl9E zcUZ+SB@IF9A!c=$75;ww?DrQNU=>Htj8G<-#Xu7V0wc60kcgPrwsNg?26m41tQ2Y~ z6A@-{+iK~Irp92Bd*zt*VNP4MqA-vIktnx>nL#BCMz-?=4v-{-G?{rNgNao&B`2Nq zkijyCf^ot$$aIEtcd(5ZSmTCOC}@aA80w`M`-ITsQp6~}+m_l&gJ2pFBuiB+8JS!h zNR?dd5x6l_OCe+uszxYJ1cwa-iqlk6z9nXD3-rjTrpz0DEwGgUq*fC~oaP9Hm5%_T zK>OIJNY8|zh>n70&5=K=PpC{5WkBEoJg?>klLxKjK`SkwbFYe=XymUQd#!K_!9t-1 z0_Mh%=`v8EIL!2;fZc?1BIvM_>`?Qq*zK|H#6cvp9!Wilvr2)6Ak_@*U?u=*n2ypq zloGfj-(hocrIv)$NKVb$W_6nt0#;Gu%~jr7w6BC%u@j{LN&%Ym6ZE(K67!Oms7VAy zn6-;hP1v@zBw;jw@KA0ICNly+46)pt8BxxDqktQu{-PvD%Z&Wwq$WfGoWR8-LC}Jt z+>~E+C`KqX6boGeTx3!?KbFZlgx0a6B#2(Ka2EJB10Q892{Z}C5G?`&JtrtQsFwG( ziiBCYOonK+4xxDd#ZR4}%khF5& z)f@G|npzDcLl~HeQ~PppcHU688bmK2zTYfZ9^t}6wjBvC9|mcWS$$#*ON?FDjP6V5 zt!yM-XBWIke0hb9@|CPZaF7h4wBSO?EoV5s2qPH6Df+DT%>2dZ&P-K{wk6{`?hrb| zRau_Q$S#4gb2=Fhnf=27SQMubG6}-$x)>muVO=zpqdh@40wFP>wgxL}`a-aYYfONx z0CFmqddjN2gx6whuOt7{k-tAZ%TWS?wy; zw}-$=2BZSODhZMTF>@zT6Z%~CG{ck)2F%2nj-0&D*&=I`QM^HCV`Q5(ft-eR3TIsR z^{_I!c>yNat3qqE$$;eOAFIjnItepl7BfgteJ%av%}m|`V&EtRIQH%20<_D!hVpEf zFOyfx)`p0AS_Q4kT6MANt1V(y9FglMj8nGeNfV2T(vLBm`b+w8WWZ2C|0V^;L+HCB zM1+3lfr*$YV;>z6qi$Fz5QPDZsEk@Ig9<;=7GD=lA()0=J)ul7h0TN}A*70+5Do&s z2rXqX)UC8U_OlWr*hcL7Wy~#VjJTR}m!Y(YGnRR|1uUwajldO@P77TKZ)L-h7!;g#LG>IZ+{Z3#*I+UVWg@eIOWz=X8YquT$RmoRo6;)|2&=^Z~JYXw8 zoUy?$9-1I6O)@jc5Ex@kV1!o5R_8#w*z#JI?{i|FQ@Q;&#YM^msel`lOffaIY6GxR`sgD9?~-6mTfD4-hk`Ap04|IW z6LasxoGs7UMG&pCKx05bDk2FH5a7XUPz<3=6piT3Oh{1g+<0@rLCZ=5MC%m5#gl2@ z6GlnCU1lgqM9U1D?G%K?1`8$&Lovyek}IYFh*7L&W*Nf7G7FIKpa@Q%VIUY;Il?22 zLV5|Lhx!sYluWVN0KDEK2nri7f!T&mG^!9K3h7y!(qcfy%7MzLPUKvhMvCK7lnyl* z7;3Uwg(zkc=*N}$p`_$Wu4aCLbqVSSC`7AAaGH+-b_wbz4&!VuW_)b{T9YH|uO|r{ z^d?lipCq@rX&@wF6AQ%;dG1pNT0SGQ68rg?KF%uUQ7VG8X%hl%xkoN;4 zW`@lW9w&qC0OAWPfQwuTq*CC{{EsDH$Vd}fw9`arro~Z)}bdKq0&6*%1tz%>lAH#Grhd3Ekfpv&;Lcjp&l{U!` zmb?!X*AUibfG6us_wr(j9s zVSZW#<%4sNu>l6AEW@sJDu`7H%t*5n%lR<*lrg@sj(kvtqrPMvs@9q35l0xor9-Ty znx#1u=i-8Pya|>_u4QVuNwIRCT0T)Owpj2T<0C^1^H3qI=RcW5X&S+qiHC5Tpbs}N zj1HSSq{Sgb5;US_tTpo>Fvze3&o|HZN{Bae)LlTD&ao_Wz_)f9X;7`Tc4}=LvV)p) zo~^(eP_7F(DA0u!!Zbv)PJac2VSKX;k>^@E0&@|-va3aDBr%SZ6`r0NC=n5}pTv*> zmGHhT36-Rl*oM7DqzY0i`1BS;jcOR>R&bui1P6s!916k;wDFB_zZ8IBG9rL7XeyQ5z75wJG;?zzm!?J{L?F=NvFN79(MchHF@9NFg-mnqomQ zMn41013(-c)Y^~ENlsQlHDVY?pu7<&;JP^s$A**G0IVyBDL`7Bg`Pb*7-9nf zi*W)*JY9tto*2SmF@)2o*c|AygzNkDXAp`bFofjYF5eR;MrzP(`?R?QPm$#}r2yS~ zG6+NxFd{^N3xxo~6~$U33L~CO@5K~?aY<;NBIfT{_D_OjJN#d#c;@-!&i@rim-38^ ziVKUC&?)pcw(+(6e=lFJ9QnV#azCG+`M*Eni6cM~LULhkh|ZnZ4s?(?x5tCAvlNWb zVpfEj#(-FCj$yUz*AGZy5DbARL=q{y1^xR2EkbjKoEQTxkij6aT~7=Irn1@Z=8_8u z3F|2U641ZD%m4Or`ak#Nwf>bl1T!EcC7})9v=!hh>)%VBYyJCq`TflQ{~=GmexB|i z3DqlDjYorG2yHSTE%nT>sC9?}xO?{RFJelG!cq*Zos~nEQnUNTM0`0h?^}1gip7?4 zY^JA#+i=FP3?PZ;HlHvhql8SNR7Ul})xcNI{zUb$MvVqd0WM+-Fa(F$-yEhPoYDrF z+u)Ka`WPXM(I#O=muLGR5p_X0=xlkKX@ltx1yqO{V$4o1=w@-GvOU+J2*#&xuqvC7 zn1fQljkkIEr~;!HQU*lbR3Zx30Fq;m91z~hrMLm))k>PPCWZ79hyoCcJdbTY23b%+ zEn|af%;PG2G>@%N#$kbc4L(L$AT?w=qG*}_`L)8+JSH3z8+l^E)#h!15j2n}PHTWm zBTC$ea>)UWYqua2=jeA1O3az!l<)bpHH#vxX4X8yZYZwCjPO^;LDJMb{I)RJ*#LQ= zZ$+UD2jKUtG1Ms}Ntqb|r&Z^xXKZ;Fhx1}hHr#K1XCg>tLvDi6JmVq(^87hG%XPAd>~q_v)V~*4%AR0F@OwcTaqvu*dY^hAru?uk=Uv@U{eA3 z*W7%vKe^m3tDkBD;~f|5OD!8i60?DG_J@H%iT%ax31ok=cslL?8^X*N!hEpP3fF=m zKobB%XfqyaLD=dL2*wb~oZ*r`Y=jJIyQAkSq^nxxlG^Vml0qXm#pc`D?J%M?k|=Fr zlfF@#94^*s<_nrzvBkznA&lf_WZCx-W%=a)-?Sb4EBxQrTb5)0x0h7*^ZbV&^ZYpe z&$Q%NPqnA!^1S({q<}DtV*<=CB3a0E# zG(;P@*FsA+PWK-rY-mE@?*z$Is@kP0uY z;#UqkDRd$i?wr6tAUMaid^R#^-mjETL22HNWJg@G4dqlA_z*EfA^^fD!fdBSn~y^I zAJivk0))hL4mIyovDw4Ih2`@r$yosTL0iDkde*p^EqT1pZMfmnW@Su8^5Q`ce`IYlEl&84-eF$mS? zz|GFMaA;bvv%<+5B})_*CUAts5H{hE9#sPaNhG1nvO{rg;ZXhB2=bf&xB>#lO#D`N z3hm41Gnw=FTrD~(w_k|cgoVQZCICrb7{@%=voF4RxO1tt4hJ_M%Yy9=k~0Sq#jX3n z@<0@4$efKwbG-&k#r&h}iKHNE#|kjNJUKm?+fa#e@$;NXCRPUve7o}PVEF9wToy+* z$Zwf5o*k+QoF)m3vfbN~4~!v$!HzaB?DeDy$ZK*;y9H|@Ws+k%wh51Ruu}2;&nYZO z0%w4H14=o^REyA7yVdNd9PAEpZns@NIiqFon|+DK`wgQ*gDDB<%&l-8WH2B&#jzK* z6P|pxa<$3rskKYwmJR>he*b(Al(iCKn?XXElda7Cvd!G*Hxd?TE%2^l2i(tkS@Jkp z?{UqQm@l@sk2M!M4z>K;STd~%jeQ+OE?vuP*8cz0n*P7{6x9?}2nociB+^5p-5*Wnuv1N|ER_wkeE%>VKE>HqyP51)8}^fAi- z>!T*EPnygr5G-t8C}N-6=fzX4vJ?IvXY$;zG1w6akRHo18~o9oAQhlEjNmlz;u)gd z((Gw~`7CRd+LXTv!s!wDEN;EWKc7{E-KhC)!D<6xKI^c1r}GcrYO8U6M~UAJmaDk< ze@uqu$^S|CyPO04)%&inozjoQt8$PWUXc(WQN zVT^2#{I{=4<|t!UCH<{oS~X$2IKL+u6^4-PcbX$}hyCS0=k98*&I${Q{hiCP^bM{k zmQXCmR`YMfmep^anQk>g!qFm33BE(@D~WK{2OwzuN_1z3bXeeXGw77@IZU$hmjk z(u|#L^;S#5Va1=e2mj=y|2vj?1#9Cg`oC9h{ol*W=coPuL!Pfm1L2~#5NjF;i3!c+ z4tBB0G4)K6x$t&`T?jv2#f3LSY|oV`pwAd^XIVL}mcx1M%~eXYxuyj;zJs7(vse(X zG>?h-bRza>`@E#Hs1IXMs>>whTcNY64S$xR7Y}4b5s{(bmOmU5ABn%Y6In{nteM^Z9O1KKMVM zEL?ao@U{4_uUwWh{^RTY^Zc(L^4P|Ia~up}KNB>km=KZxf986AyRiA;IgYv4zl0eo zRErZN@}1iF+WPmA<<$RtWYVAhpC9tDr#R3E$=#B`O*^-hOl3qd_O>(wq)vpi2z&8% zoDO9#e`S>zKI?~(URlBp0HGUxiT8{`E?TBv!0ydB~1fP``$MUWp+pNmLEb}|uM z1QD+zh(s3!P?NBj6}A{=w#N8N1asnm3tPp6y%Y9*)@+0wrRWZh%XH-SQ37DVj3k6P z8U-$bjpAA(29XX5ab}H69dA)m12g#eAh`d$zA{xLSRNcZneWw0mAt^>;+4d zyX1&{!CfkD;7%v!?=EI<-mtp=lD#CB+sX{si!!6xdsHO0AtCF_HLTj^tzI!3)XxVF zY&o_`n_JGTFU__hfIRTAqM_W^YFi=5D{ErQHkPlU82Qy3OY@DHqw7np@QXlkak$k5 z?E-epWNQ&JH+m8L-GY`Ihs>LrSz&7DDu6tY)uXu5`hIu;tsonU;`giDz!A8CM7T;L z4%H(R4e1T6!^1Q!o6Vg7aOaM&2WSJ~#6i~fU?cn}%tt@iV1xm|gQCOZ z<*{lVqBrL%ap-Y3ABlu$K6mG$!Ya*xS@BkXS{S0$%iIeS@OL)SC}Ndw!*UO+T!dW2 z#61XZ)LR#0nsCf=IL>Hh(}e?{GdgF%xay%4%WAM+%X(c}Rpb_>0*GY*xHA8? z1TaboR|wh^{FhrG5z2(Cg-e6;+j9~zKqy(zS`?D=qE#hL@Ryh7dz$z=^KGtgaxgTx zD3y7vMWlVY&9e{51es`+1ZG?QD{~>U`M^hA^NqNpWd?vbmz2GBRk#8tcd5Mk#XRio zE`_Y0GaUTQT@u7JOz0sr7YF!iUxey8^B?nt7sxlA)uH zAtXv8@K>oBB1y<(=PODh`oc^#f1@fhvU(y3F<r9fj3~|LWqoZO{M!WVC{?AP zy)wWmo9kT*Tt5fpD`~<&V1(Ah-icQjXb7H!k_3BYjnbSJUnoeNH=6~-9h_5ODE72Y z;SmEDf`kzgfmy0Bxi~6ifJg|#5G;2A21y~AQ>n%viYnwJ68!yQg=5b4Y&Od&f!_sO zp}Un65Zg`u+?H=el14~9ibFJ_)RK@Iai9u*6_nNTQ`+t`DF_e)iV;c;#rbj^*0!&m z3Dqqp0u`L&TT++*JEq2+Cj~1l=VlaS7#-!%<+~?&sdD_S1WL8RXkXj^YUh9u08oG? zzq<8+o9R<~z|GuxNP58iWFGSVO#_=YY?5cYyutB5GqzBxrX`mT_W=#Y6 zSKY&+Fm3+#vth|9wbyci7Z$TvS+S)|hETfD+8L`j zBN)P}&EEux73LfW*8Q9W6pU@*FgfUki?RihL_H9w!*5zDd~^u=y4^)>+~9suY-# z9No%a(9Y#~$9{ozhFGVHecUfwE|CVUbo#BVqo&T z(*E1w6>`k;xRw6fK^2k)^K;+*cYsNK`!os{-+!|zl-k3+shcOK0`;Jp<#^S!pLhQp zdJw<2#9o_)ur5?at&-n7QfOfzoGQ=HQo-O@?RxnxUCQff|H5`K$F2R}0>*AvzYZby z9sC7WRBbNAFSwfP;DURHfWEgo)$Wb_vIPvH(1~Wnyh*9u4N1~j?axI0gvojm` z|87-Z%u(i|q0&o7_*P7$6bKK;RUr!Vo@pmrwW%>1a3M zlNlokqg92&RxMg5K<3>j4?nh3Me*j!-gcrq>L~Sryrxmk;x=^@qGX7i9;h=oKjAC;>w{TbC*E$qpBuNCP z0cFC~ywqnV*FvV|U_h=ev4tf<=lK+ObHo1!bDNn-;{W{4Da|=pAoy4Fn61wPbTC5p zM~Q=Lk4ad6H}lA^bkMPh#9!SR%MwBiggZ=RwdV=IjTi>0Fhl`p(um~v=)gtr73Yez zuASOlCJyM?15TCofP28HUfrc)Z{|-gpYBqzPtT@3BtN#0Ctt{mc`NTO6?^r}sp+dR zJCUb6ELZyz&&{K?^!WNj0*m!taJ%greAeIWi2p9mX-clt5TqJmqdiKBHWk9RM9G!O z5LUQ&5iRijZ!|EwOT_^_n>7`?Nm*h{m35cOyn0&x?k@A{$(CX0Pu8<(_c*$z8}o~M z$NcdI)-Q2o&>)W$ws$2JFVtZh$px85M0dNIzIs}l2 z7D+L1n`#m68bd}{ikU$HT!<04#k2_la=APJ#Bw*W0w0NQQ$Xt=nq`ux21O_Uu>;wc zz!3mZ0Kx&G(NG9&*}Y2>pmSV^T};AKAiaU%MY4bZU$M+fEDMN}`6}cA3LihQ)L$W$ z+N~Bz&wWrMU00SLD zvNK@5=axujG%27rQZ##=Dx$_95`h7Vv(wFn1wbNV5|mK}6!W*lwx4VOFuV|(acqtW zvtQToR-6bndTxEzSwSJhaDnr00THXy#}{xnP|oajX3k*5Kq6|0MsgC^zaga^IL_^o zUT)wF`XLT;^Z)no#jI&8)Qm2!R-Oua|~ZD zUD^KVePw@?t&lKq7n6UdPz19eVAX#Yr!r`jIu-Qef_)T5e{*%QCPB$o8cFAE%o* z+E$Sd`DY(mbgtvL^Iuu8rN5I=mz2ZP!8en~FeT;oOnen>?#XlSOMpb~aK}r4#5<2~ zK6!leb@)vtb^J-@_>-5NT{jR3=@B~5M3jG%m6CRp!F-UWJq@ogyBzYOF2R>+=Mx)^ z&G{kE6P$6o_NOAYN zTb)lG65ZKMTtEj=K#yV=N-;Jat;=oC3Q&2m0T@C?8LlG%tc4G$X(NPjmsn5$A`$M= z8qNSz6R=~5*7%VK8-)T$ih)EZ20Tb6j2O&Vlscmp0T2!Y9YK<8pa{b4+0YzRoS+%L zXaYI}GiZz$gNDUN4G@wfNT5e33eqA}bI!Uaa31U2t3v=w;FyWUly_S2#;g`iLU3Nc zEci_6Xxcz2JUwlri7=sNud-Dm23ok&mQQ5jPw^A03BAEcBVsd?5Swj5Pggwx;V7;Z z6B@Cj^;Eb75CLt$FM@Ehp3^zNdIFefP$8f%LK4gxQ6Ye#_1xNs0F%~1oWDR#kX%fV zJ<5&eTK0#Q;|brix)TT^P(!#tcS5)bq5uR87=+@2_QQ07eajvJCuSX@4xtKlj1U|tppf1Ls{nZc{0M~&t8=+6`S#S=-0WwsPJzQF4kj!wHBAf>5qqmvYCAXM5J;^91g%5J+zw=i@vK7n)5Sn#GC&kXV)kfT41yD) zgJNT2I>y@liW7O)j*QB?K7S6e%I?RIn6RL@u+U%!E$3M`GAhs7TwRi|OVdKjF;N}E z@@(5-eW7)S5cm`6?0yT6Y!?<29_OItylO^9tg_VcbC8D+nIUT5YO zTxH>s-L0QIbrrPlci4(vc!j*c%6M+$kD>_@(jsiAv*0O^Mpblv^t69_|4;7yUnmYE zDc`>7%j)^c{V(29dCvP^y#2i8KlgwAh$qLzPFA~+b5RDjCn-k-!0q5NzmiG3C0u3? zFYy$>#Y^fX6-&LuUeY+3pTf&a;VqZ=`gwZ?$bIDg&7?Agl#f{3+ge1KPe$h zv+^nH4M==WHU&r{j$76oid(rIxn@u;LeUB!ll%L~Ah}wN$o&wNzs4U@!*V|#janv! z<*>iXTdkJM{Cw0>4FV$p-rh1;6#xbJdBMJrm&(_h>!yy-BMP&dLW|Nmqe`MC^q%Y{ z1(@)(w~vjrDUt~iw(JJuOYwi&lobOtMi^l+&SvHc6}W^c{x8;OWq-4|ZPtl^Vu0u& zl)H)o=X@cpdE6vsw%i4#7E`ht?`k{eGqR% z1xcmeGQ?jB`T0s=Umsr|xu4w2=O+pNOC%__PlEFNlAu>UNl@nFFY^!hFG&!VN&RF} zwVzre3($BYUcSDFpP$rQ;{(f(0DoAmQTxhN2qF)V1^9S*$-R8lav!yqmm2bey)<4b z&Ht}S&|EE1Ya$4LFS%E1E%Soun@y!eRG&g<`OSy_CEqN9{hhTcA$1PE+_Ld3nTkKvi zweK2LzY`sXJwZD6p6g-idG+0?_K`6kH*a3Dr`EVo?_wRD$|2oOe|ftK?UJN6o~-}ib~x7^dRgC5Pf+;z$B+LyPjx|8*JVhfL| zfs{UDg6p)3#ik&4=I+0W6O|5J4($d{`E_;*KB(fH?%>R<1bpDpnlX+4o-}j8xxdqP z4RJd<`p(6HS&OTYqTrHFfp^ub8*RBj8@zju){c0=-z0;?QSF^iy%F0$ut`=0IW8*5- z7jbx2r3!<*GJ%k=uY+MO~l z8nvF%ozeYl*`z($8Qy-*hw3znigcRgKQgG@T6tvEQ?SR)uHmo38+Fg>Q2AWd<@Hw_ zdF|TBbxy*E8zqA0{^Iw!?$~ML&PsOet5<7SXqTgsM;iAEN!_-(K*{qSl?dAI|7OwW z@t!?K{Nr&aXxy@Q(e061RrY0Ge<@P#Lez7%?oMxabi}j)aNlE3N-NsFzSn$8K0a$!;?+_H7w>&X{eDH4_hc)#@|2PF(NiYYz53Rd-)SFxfT2X`HCm zl+JBVJ*ZH3^RjWtPRClUDn6r+IIf%;_w=9IPPu>KtRqWIhcZf@NISl5-JugPjp_Rj zGp~bPWik?u{8q;4+9ap9B`X9UXb8_Zxou;mKBHe1cNx0yx5jf?wmz|KOxz67OcJfT zqfhPaYvvQK8{HSx8#_-9jkqy;&)Bv@=VcBz9oRT?Z?BmlM4gUZ8m^=UH5fYoWcAOR zj;$J3T63twey5IyBn<~;Z5+8(hCCZD>9pEss&9~Hcl3uktBT!!d^UB3#@YXoF7tBF z3K+gT;Gc;NHl*v6*<-Hkt1pWXo!Af>ieGeA$C3kU|8cWl*K_r5RGac5<#ciUWoDvB z$fW&ON1K{fKU}s_%wH*W1~%(DIR0W}y;sNDQ1=g~?t6J)-;0m(;njbMXm_!`a#Ciw z8uHbNud6RUU88lU#}a<1}d~RP|Sn z%&hR)LvOW6+P(gO4LPyn!LUR94`nt8nc3Q> z?65e0TDre+;O8my%pp7C{yK3wwC1Rq?<;SSYr^+s29yrC)$RJYjvL$8E;r7rY2(XX z$ByvqLIyQ(TG=k(p==h^c&uS$)XU;AvNyj}Y2@rW+}~^U>>e|8hiMUF=c7OK9A(~U*0upq39&5Gxbwx&k^zW>v z`mlWFLG|J7zr}1E-f14<-sxv?tj^EFWtPf z+nriF1Cl)Y3|g@Cc)P8j`fM3?z4FB>!OhP+?9un*H2%^pnKs<$32X^xVGW4hc!3O7S%7IkJm2jqN%a@ zTKu%tZC*SLJ-BpdxoN!;+cpUdIJ>QAbkc&Cw@-C9ICrdxm{8NxfSAyPU*{F6mc6xv zo02HLf6$ikyK1F2TeP#!;}^gD?rdzQOz(E~V&lhG%hV|nc=pug2wnO-)BH0t?i@Oj z*vu{a^wj!3lO7(-?69@hKIbt#yASYR<@_q6d%ce%BGTrY(mOX3D_&3PUbb!t*O4{O zEE-bxI=yOG$F!A2nk~B1H*3P=F%>ts6(C>EWpmeUi@v?C(a3|+ zHrHA%PhGZUy^~9It-h$igDJ?-eF^s#HS8{)kf7f==xMohH^2DB^Vd}V6Kt0hp|jrK zJUe{JhkHx!rS82<#CP3vRpxeawfwI<%k1vmxl2a- z;dLvd1r%F+=txc_SnGX*S@3WjlM@g!$ zj;&tF{caRr-j+*=5+#lQ zy8HQ?9jgZ{jYvD3T(9sEITx*@*)F=m!(*|>wt#vMpEG?-9agJtb{J|k<^dBypT-RhH>aqZhQX+M49 z1!V0>+59f0s<(@qR_);G2N~_lc3!(;tG|fM(uor@uQYYpy?DyUze%DKdC&3_<&!4!|Dl9wcV_faPFFRs4O6t8Ub~`Q>jQ|-O`0O_x(Qj^oh93 zlilxro~>Fu?>KxJIWrR9HYO1FE@#FdnDdO9C!W@^yX6L^|UWF zDyOVb{kl@#zj9Hx*r1vha3iP^kb(To3xvY`Z z3k;j9$+-B-koIuZ`$@;o&#o`=oS%HGYF4>tyF9P%J72xyHe+P0-=GS|*Z&&ZutD*a zJq~3l#0~r2z7tY$^=d`;5%ijq##e!}UNpRQ@imsTbk!ceKeDgv`twuT?;k(?o%AK} z0A>0+ttd5q%#oq9hG$#~JCxRLyvOOxvDd@1AG+^}*}EdDR=~6lf1;uH!ip+aKe~16 zk+#nk_qMf<^?kah%*b;0KR@=Jt3F!s;FPh~PEVU$xy7Z0b6r{{D$fr%#Z+%j7_Z^R*k1g6X8T#Q=*#S2yC?;?DtK{3md#1H5_gTMXWPFLBW%sJ? zt-5>NpvTu#rG`kKA8Gz^0OAy(pZMZ_AI}tdktYWh7Q1<+->uU3%NrkEzQ68Tog?L> z!B^Egii|J2@b?9yr%j^8t)^@gA6qG2Tlq!R*q{$(jqvOJcW-syzhTjc!Dmi2slIyj zmMv|hXM-=6p}+Lrp5`;Vk#6;sONPtauXUK!b>&Rivjh4Tb0XTl>aT}8hyts4wSW1p zWYYbXlB3r$!WR~~Fkg8qdXm$Ce_Y8XeU@oWT}IZKIJ$3}JIxahFV6CPx+QRW@!e|Jzv+ebk}8FQeWf!52}`?ZMfLabCPk+)`f#Ss;=2|Ozsjqb6zhucjULfAQ&Mn=Mb0YYa`7o)4(JW%i$YN<>XK@!{5%{i~mh zZt^tduy5n3aWZAl!#3H=F3cPhqMf+8d)F;ky+$1ub$CAYl*h~u3rgbqu8bcxWgEQy zxz?EZE>rXQhNt$eW(CiU^*L-nl+~ueXKIQIQ(iQ8mqH~vx8A*S2J<|2a zgA3Ra%^R`L;L#V3#g04K|9$YLtHJc+=>uvtT6KF(Z*2LSlnoo-+?4#$b>iTuQ*Sp; z4f%E7(6_tq?eL_E;cK&pef)TG>f`Jid%yhoC2icjUms_0evp0U#qmFjZLL_bb)!#{ zqT1jW9wa6Q>z){jJL%g#9&^BN+oca9@ZrU-UY=0t@xZu^L*{S3FnRN`lx;!IgwX|d zX*hOp-StZ^o3_PV@SJ`6*O4*sRO7bj8;AFEy^7tLr&UHA7Li(73)Kc{8C z-_Yn|8u1QmmR`stXTua}qIxlu! ztJsG-<>~DX_bfW0;h>}b`h>R^ih4}!SaITx_u_?FtK zEgSdP_4~RzhfCc&d~?RBg^Kr6yHdz>=c%iE{L@b1u??7dH4e#q{Xvi5v-#TVXTgl4(!YM+1)<+pt(KkArgAGGX<2Ngfe ztlaZ&%}7`0jPPTacfalnANB9_wu2(!*WtILSB!44xBH*YPFqHSfob=BD@|P5>c}|* zQU7$j^j(K%3lA^;XF{#W!#kRH2wIk>pAGp8M?@ zOP8y<@VAHU4gW0Ze&WlAtOLIc8L7V9bzk!EU*C+Rt{9Bxl`qz9ji|MHb(>Wur$jtB z6|!*r#EZsbtA07(^SJT$)>+r|zdL^p{P6rs;MJWKuGO8B@vP#tUGBRF@0j)QZROoY zYLo8y?frirzOZyn*1n2O)~>MiEpUD0u{Y0}6**T? zvidjW$DM7~p1*VGC3LWdonYoUYN_rOq=KPfS zzHfUjesxHHeeIX^*DtNO@}}9ZcZM|Gl7PM}J#|a+sm_}&G=1^rpHZ6!UVQ2P$BC)y zQ_tTmk-VkHrI*sbhLo9HVQHt$^;-8^ydreofKsQ7HCOid_%d3R3|$^Ns#l4>k0s*x zx~-k(x2#T%4AHGAwGQueqL^3Z4$c>%zhwUI+4g3xg zZ?dody*EBP@W|QA*;{C-;Z5Woui&j4%V0Y$JbSwR^WNlHo9?Am3~#g~1|1NkzwDh| zacHSkpOk~2yw5IWx_`0b@l88!oGZU$$KjM&qcYy&8V`jwHgd$4R$0aGOglPhbICt4 zo0RETb^E;2ZR8D>MqSx-Q~ofO~iBwXk5z}vBDi9MIR4^6!(xmNAd@muv$#~m#? zY0-?~RbV=M`R!fvl2e`MBrG8^>u+AAC9mn9eRA)s8%f#OgIiqsl$M?O zKKsu$$ioLttB!3iGxt@XLe|T)=Ejy8&TE_R{Aba?dbj8LHUd|M_VX-Lq`T<1&0<4csD==CqY$n`+{*q+`qL#O*!`n~tv*4bm8p4f5k^&mX!a^lN9F?3q( zWyN-FUfn0;jb>)7LHO@^d_aljes`)p?s>7}+_JTlr#&xr>3`m|HFWU)y;m=^lXqRV zKtExx^?zxjd!4(aURjhb*>t@A+?eMFuOGa=qG7c<6-pi-_^iq7>jvpxjjLW3 z{`)ksVO4tj^?I>oR);tIYs|%~4Mz1U@wnODN*;|Leb#*GKJ}kx-P?WK;`Lzejv>`e zuYO(sWfS?O^So!%od31lnPnU}9Ue={%>)NJnV+VIzT*YsATJp-f z&a<1|8meqpZe+{-i(;d$PI>`}CIp-c>okLUJ+e}g3_9t5c+=!#CF}+vmC6?Rr2qb=Al2n`<@Q!u@yh?cc2~-K#e@yP}g*cJ^KQ(_u~v z{_9}a|ASBd;=l5LG5+iABlFE2|MmVE|NSA)x5R%-rB)03$88=7N|zqKvBCU|aXW64 z;h?R&VUePX#%28S{I4CecXVIBrQhSHjRLBa?x|?^40K-9q11Ru#Eju?yZo9KO)nk% zW%ukWyDlX*D0AeW{<{NL1nzpAwbLYjb@xO+;_6QA;)JXNt>eSHHyGT%gET89oR7V%8?hfcXb&SrrS4a(6|djS`QdI`i^u-*CWlg z+_}0fdwi)4Rc3?zRlFCMsMWiKW|QJe+zRjIl}c1dC>GK+;I(`BBIj{#F=eIFO1D5+ zsi>55=Dv>OCbwTq>nfI8H2MzMU-VF$PQ{U;#TPrxn;lWHmWUkJc^X>wTv7Sqgao}| z?TG7t`0hH^x_kP^UJcvapx2ffj|`y^g!nJZZiC_`YUC zo=K2O_9y~t+9a!yfuSGFp=h1`u&K%)6 z>{TG$Q14ZFQh3=<{)b9_uIAKQbarC&Fm;_Jhr9l^zg)>9>&Fx=I_b}G>E+tE_-|}b z*GsW4efm7-=i&`@)LjBDED?ztRgWyCE?KlLO%2}H7Rea%w;Gy&Sh)Zvom%MVem;Y5X_}$d$-V?Jn)W2S%@#yr4OZ#b67_?peI<9!#Kcebaj9=LV>*(CBkD^&bm6AK0PnD}s zYxtv@!*3+5s~IwJOrI*HN2R$AxDC_|+%g9%st&F-?@^r%r}ngtykDl2IKIlQdCuof zZeBO;hVpIafb9#tq?LxHwr#h$m2xuCPKyjLm!8ph`0-(WRd1Ek40qkSvby}awAiCj zug}am^nA)Em9%w9aov^+x($Ey;Z(JAMSHACPndSFeei&?WLA2Ks-=e=GR*#U`Llqf zn)LQxMac|#JvJT45n+Mhy8-MM?KKGz`ua+&peLQ82=bL)#X23_t zPVq(8xi_1?;Y_)xAf_1=?z z7l-y7nKG@xm7&ubG~IdX!RgvPA=BipJ)6e;lX19r=-VT)8^`Pk`*JIxJt1H8>d|pk z7g6A3-=VL&Rf*X;xv^VT6Swh(*-t)5js}Z|kBlf&e(dv?vzx@$E@OIIGB~(3$R6vT zIpL3Kp|_5nYS8#a_o0;36+E9u%Td<0VT!+h<>w-xEf4by`<#jk5ATm4B~Pt<>~#ndgd@fAryCr?ypHQY)A2 zq?Lw*Y}dA&x<>M(%>86|{N+bohn6;u=!6#8z4%7oIgQr)HiKbho%Nds?LF;0Bs5KY z-Lx&ZYl+lh`i9dRdj0LME_$MoY}}A)!+xnYrQMfNVS6$)Wdr4qq*7H zxuI{YcZFC`rhJ_fApB7KTb-(fQNJaxe7$_v)`XX7vf?Ky&1yO0Kv&-)Z%&?!+IwF% zxb20SrOv;so^S-NRI(-_J>@&{m7B+dvXe$PtX?PL<$aOcqBkk8&U`*zKkB{m;_kn$ zXB;^9SGDR>5AIUc_B@*%;u?JAc6RB}ZC=HHNvj&=eqvNc#_0~l) zNfa zd2Jc>L8a&n1?Z?e$G5@vhXs0Qz-%l8ENy`@d!^8GBLPXA1sDSXl0YF2I2v$ff&~G3 zI~tXkgcQCkL74z+g@y(yMVt{Z0(%sIh&neb4509i;6Nrpl0XlEjpgxV0fvwlu`R$L z^QBz}nlL)XRr=@{Td9oC0tFg;l669=gF2#ZR`B_i;prF?0gB6h;>6YEZJ`mgG6cmL zfnQx#S(o*5gC?|F7>e`t+l>-Mu@PW`+*t7Wbd@&~WHaj+;Q)d%kT(1!_#PBy}C@nizzAk~GJF8mt$0Y9Bz zp(gA<&2j&`p8WQI(R{+tBz&dgfS20;H5%9bKQu(GiFmvJzlkS1TfRT=`majRXL*Jo zKO97#FdJv48KMu;X;tZskpQ2e)YUjcw<=T-m!fOj1N0WX*mfuNWnX;HD}V8n)AT3KXPj@q6$2u5aHskb9wj(sIX zO2-JU)e=Oe%-BfbSP`=ltr^;8MT4M3b`e*biOja%Vx#-j~*EK`FUv%b;E*L-5TJb76^c3F!hZfvi8Y;8fcP)22MCsadv5O%m|^z3>Pt>!kq>O9;O&Nd5-cwvTIT=70=(&rrd+?c(z#EWuzg2D;P^lvsJ z6CB-|;TxBgM|MKtOA#dl4@C9ix^YK|p{ZW9p5@oFaKOV#hgd*3#CjECl(Ew6Y@})Y zuP7P?l>bGdDViGa68T>p7Ut&v7pe|x6#7>Fe-qDZk^enK|CR#1yww0+hXz2CFc}&} z9|y$i@|g|=0G^82D-J$u)Axb~4yhpm@D!J#3V%S+=m^vdCFYOz8X)owB=NFf|KzsquAk^}oi;i}K{l|0Qke6`es} z#{V0s!`#n*!Zl%U`Tv`EM8iAm?brk4jxxZ{;V9$hz9Bf-zQdQsE4R;&r^>9;|IeAc zG;CHH*NI$9CvrXAD{(r$!*o~7Lb^?`=yVHm072o9;y`GArIFK60n5rx%L-OOa;Pd` zak=AT0jo$Z%>`~j=J{8^I>;5e;P7S6iv=Ae(M8RpxbjstseJiAgTI!0059SHA)z4= z?*2dF>bK|rZ{%^6{|yW-6aRVAVlc`CK^-f`nS($by`5jx+Gar2iXb7YYE9C5;K$QR za3G=-Kaqm#NChA#R7SeGQYk0bSESsx&+B;dTmLw;lC<3dsaL!X__FiAun-UbpGIMC z&;Q=YBbVqftCe+X^D&OucaiJ<-%4mj6KPp1G{~H-SuIeb`y!&RGQz}yaRzgH*|@&0 zku6Fm9oJU`)MPV{;K`h-#aD&XAb|KF_jJ}~5tU3A@iqXtf`B@zR1p_dON@m{{4V0> zrj+_M04k6JSThzH9>Lly&L$jwKd+-vug9bXK|t8vWWywlk*kBcce&54j6?2gEQcv; z!gX=JPBJPAA$jk#K;}OCD}MIusz+y~g@t~5C_k@Y7(#^NybmY7Q94(@+%riwLgGCj zKfZ+6A@oWmU3LI{`v!aYC|z{`efw%XK697}IX$~Hk|eug;ZehR|3Z!&q%vM@y-NL$ zPE2SO*IE}Jm846EijQ@C2LRn0F1*I?YpqHRCMBk%woZ;s(IqCuCP$?vChG(gIwnV@ z#w8}`5?i*7jZS@yc*xh2%Q2EC!8$%V3058AllmQVmm(a>I}5sh@z$0{?Eo@%Cny7P}`w zN0AKj-SU0kY4!lhIqnG;-0&?{b-i?7NK3c~1IuJ}G@~;atiAzw+Ql6R$nSUOhk;j1 zxd;Up!h;mhWG)SxAey#VXbNes9+}`LB>IMpIJqqip{ht0#$k}T@k3_ zELKHeKZQT&gfRqdV?kU@EUQpi8M+rVa0(THLo8S*$I!k0cXoZ}$JR!S255%a~T2TgO7}cJYyir5EO+qr6j?=9K{4N@P)f$ zvUA7^jF~UHhYY`Xf>OpwK_+F`@sn+Wj3;{W1VvbE7C_l7ddTpII$~6!&D0UUR~(l7 zk#&i2DMKsDg`&g@gf4I-ZXjq71Wlz97RYj##fo%mny=^Zw8l}aF}=%$9RP$&swm_Z z02wmf1D)_lAw30Sloe7@NpYPtDF!pNIL@jC^l0R|u8>Z1dfNyF;#%1;Q@U^^N&3^t z5M?ht7bXB!h9^djfU@&e#64tZQ69u3JdmVjLX7fOT^7c*;8h(_UJHt=YaCdwwdCtkR#8}# zTA)+|;LrcJ8NdmaM;%wV75tZ5aE(P<(4M5D4^o)kqRfTb<%5m&3CAnqzLpsPJ_0v-l*+GP{Nonhu~7Bhj{{!0 zOK#zgy&#isU)-W0`>gI2FHm4QDKH+gh|7iU3X-tJsLLMCR&TybKEg!%#K+qzmcj9e zCjBaaJY!B56`CV7b)f+el##($jHr%^1UAb}wHE z4qA#b)nY_v6fv95fR5m?h3x@F;}8YbCMd|VFIxxyHUaxOsw`};+(VTYx_-U^%!>jI zQW*q8Qz)*v!!5E%kURsV7Z!>)sZ)BCz|SSzx}UF7R*b<&5|W+;cqAof9-V>2Shk3h z=o;p|DAnC{HfL0u*8;A{-Es$nx+&hjeN`lV1ivpVD6e7i3k?JX%r~-vEyrhV@6wr< zc>9Is_)gcg{~S~E%nVSNVf5whP~e{AC(3!obuTgp#J9p-SLK&obGtTfbOd9 zc(yu7)3-s326gJ^+pZM*{mr)Q5|sXxn+|~8@rDgrI1p&qpoRD`3RAi>0VKjQ5N!g@)!MuS?-+$$OHQ!oZQcuz9OeXi9QUzN89R zq%01F>YNWaIxEJR3lGehW2>-L(`|aFBQOgR_6l8_q)oDPGKH<3$#=d;wcVWyy;Upp z)aI9*rw9X+Z(^QThgZZg&nLsrtAp~)1@szVvade+Qf{GFmSA$a2Q~-@79^MMY(uwK z4YI@fx`{8il-|y>%_4}!@(g)J=2`n%x+LO+;TD1t)J-ETSd@!yALMb= zA=5Mu=as;C?dr=RSTT#v=b@M@2w(E0$dHSw%ZtjHV-$qmyg56dm7zIk;AI;}FWzA5 zhyt86Rp{{(>RSl``2x=5zb!60=uu~53MZF)y0jM<|9mAODO;XTTznZ&d?nQ@G$6O! zdxfem`RBiuu@Hcv*(u7dCsu#c_6}LIZ`Rx)He58(O04|fVJdM&=09pFk)gxuxsWLo zqY#~C6z;`5mcq-~nG5{?w0Gj$=ih#O`2Wh%1-yC$u>AdhHR`Z1kNA)3xA%YF%=7R0 z|H?_|f5Pe4i72-tV>Bws>s0(i@#4HN0tl!1=lP?As$XaQN zCG;c|eRoBN&*=7E9X|OnQt`rR94>27?f{v8qays24((-`#XVba4u;BE%n{*3@-Kdg zb8CXbsg?Pb!w^BF zf#6^DW0uE*@-af*M@fL|jY-JAJ9y+vI_O)|#7L6Pgm?-fgk+5J>B#EMphDS564R5; zEYtGGho$8uHY&Nwl9Q%ZM)v5A=cv2m-SM2zv=C(&|7U1;T8J{dM}zLFH(MxJEELMW z)ue?eLwmR@Oa{`+xm!L40Phno%%gnwc=<#ErxB#^>hCL9>|gm9;AiU- zNf-&p0t^jgADV(F$t(qEqmhN2^Y$)BgN~`uUNMQy!Yo#v7pWs7Bb4e;r8+WI9ii1k zYQq~TLmF#CLcCT>5K}}DtdKxgT9R^PJSy6})46z<3~V+6FAN<>=O=IQ4uP}llCM4M zmLJZKEyd(uIHkK2TWf(||fjA&2G~MWQ1*Ai}B#iQ)_`g-I{Y1x~aYeX{ z;}X%udtKjGapIBhfIKWsVId?7ZUe6ZBE@X!RLI*v?%wVEoZ-;QKm*1>cLMu2c4;5y z4_-;n6F854v`^ps|2}+4IgO?GIN!}@LA~m^t_9qyL`qbZ#cBS<!Lce|Fnq#BA+HD)=%E3# zu|k4ESt!xyq>c{a0%nW_7J?)Rl%|_Ad+M_iRB_t?JRuW2*HHna;bR8QhLJ)La~5C> z6p|JQ3ZQ|;eM6Mv$3S!n1z?tyfh-RmWYRVg=QT>R%>)5P;lNBY3_2*nC~rEn0F|OS zfHE|iA!#+*NFEKRjT%6Pp&4L-EQ^^S+eFarX-dS}b9G2yQ95lNQ_*N4Dzhdc15%=V zS@@YUbDWjc1_!&ICgQXKr7|=?D<|bN6#c=(D1J%+;s8Ssp%lCn& zE`eNF2w0nuhrntz+ zCS)c|=E8F!B%;6FHs5ZW|AB4uwNb3EAUFSg2LvccL#ve5-aiJ3f{ z1suBszn?J}P8c`-- ziq5}wLSp{)#aO_4uOFk6W1~`IV_N!XIp4YoiTTzR>N2!fnwDBlPHZ2WZ`(fWORf8a zAV!7r`YkS@O>A;ps*jfQtC^6PUtN+m5#HoLC1{`6IxfLSd%kNWB>Ju=i)rcgdva`2 z`?zS7^3g}TFN2Vf_%axd3%6dsrgTh-P41Wyo9v_g{A(v9das{f+}gs6EP}(i{K;2V zA^NCiSqT|NGyCK^2Nd=M9{?m7K{>{hp{ue>vFzeMXeaW7e zrFgviJfGZxjiUDxRc~o0SGs^9Iw=>P?g}>Bks0Z;1?RA zj#P#;R;o3r>PA|PMym-`HPVEKgsQ_rH4Q^TwISk(wYRH9tRphI2$4lbI6=#20c9ff8H6J$9pz||p*)FFmO2BSLC7zRTlB4DFNAz{XFTn!@|;|8N4 zLam2T6RD024-M6XMi?~VhR{$0)(8(ZhU$$77^F-D1+~CLb5=nYFw+*Obr>#81ZTGC zRR-D;jIJ%z-QNQFkfk++-N zHBbql1tWyy6qH~c`Ao!3ulCh;PU z@5_7uxE#qV3qNEwhV&RNM+qe&MHOf2gGyw$Qm6@K$IOpD!!(+3O=M$_hFoxC1-( zniFL2aqh{{wh774)~{c*z3RA_u;T5Dm4V%ky?DHw>ZvN*pu^;4-&J9gE2U^Yn}5vw z=aBgtcsrFn-m&V*9|O;}n69>Dm!4ej{MqB0Ps+S^q+Hn>=Xdm6_0PQve_QWuxp;2J z-4{D{ytwf9g)KX*d4KnOQlre>(gS{>>qvgRR6;Z zKV}py-z(PQnqfuV4QEPpPZ)B=e`?^2x;xr0%qgKs*Jy72wRq>@2fc%ni9AiasiQh> z+p}Uvq^^5FjTs$|4!M+mbjXA3D)VBl_U;_o@LKiK>ZOxDj;hkWUWG~v+P674_{#L2 ze@q;X4}P?;!=^D$&gY!oI`#cx<7*Gi&0P7-rR&FsBnM~wm5~>*r^+z4>yz#a=KLJi zXuz{IXG@+eyRP5T6-Q?HKZ?D4es&%^`O29yi;HdS_u}5&vX9?y_b{*L#tqwM<(0cX zY-0T10mY8y1dci}vcum?J0BWQQ;h|8N)1VA)n!O**E*LfjVjgiw~w!ux?lIR$@TXS zZyU4rlc?*5u5A6b{`6`pOZ^WrE)*|SWpbU;Ayvoj#@}iG?(yl$&*J-sG~GYp%Ba@U z(~g(U*q%2ytWo73s@F|SC^ob4h^RKJGzk@t;z5_X#666wn|8Qe`JXF(Q)}tIKmF_a z&+hv4V#$_sK5g{8#@MOjj;prrtXcJon9loEzZ(O~W^~)=hh@(Fy=2tp#*Y>}|1!Ay z@V|ntMvYtYB&jW|T4Cp?3l9{!t zScfq2J%M5R!s@d;mrGyXqp4nN)}xQU8-2;TsLbON1DZy6{c!b&!<#bjvfJ~jYHNN!nSQo= zNu58o?7C?FwaE=*UaS5ok4xdR$84%+obW?v)!u`ur}k>Ha!;S5HFlN*lladwJ*~dH_>J4gbCwz_H~!r` z>P(NaB=t??UlVGt8EDq!jXArsmO5T>XiZEEb*i!U1ih=EDQ6*c*Fuc0Sm>^Zqlecy zH*r*%Dw-APe^y#}tjcE{>T%(%>prX>s4um{)H`Tq&HjT+)Cw`|PJQr6lL7vn*UtZ8 z+k;_QYx_^nirfF~_ym8`ncDc%Q~f(VT3)PE{SLF*^%@PPb#LeY$K)ylHuTcQZ>)W6 z|AI=}>s4Iv;N2nhb_Q1UPg%VLH!ZsK(ZMQvmfDlP=)?R|K^<}CQj2?oo6gh>=(4@o z#GzQl&x1xCj+-^~a?^~>^C}$NUA|Y#Hj!6LetshIV9va0tF>dv6_Ms4hqm1O;)lLJ zjH(?yeWu^~PF9Bt9sStbX)qg}RmfhcymeF{}G@^ADGs zMok}>ivM}<2@`WU^^pArHnwNxxc7z(PP<#G;ntdUz{oa-{n~_RCe>H}v~TW(`)g9; zJ|?E`of|pw?Cu{f)qXMj-5)>xr}Kf0>8(FL->>nHcgQNswAaVuHP|~B8y)Q0>%t#b zKHQNUv|-IS#flpHnQkBZef9Cu$&0)049aZ#;z`}6+79aKQ~voNczCT^O~$mG5mIwY zgYXVoxbN$QHS9OvfBMagJ%+uTzfN8|tixOw*x`7|nGZ{@J25@ugKVPLkl5va%)Hz* z{nK2t3Dmn2mpr{IUh$nN>{pXw7QV+WD*5fO6cBiD$-}nq&SAHHR_43zV|I00cXauBh~FS9?KzO~m+q5h-pWpr%vqhe$laVbzgKh`w5 zJsB66HMo2F#@Qo>&1gQQ_oBgTudSv_E$+Qz?BP{A8I(udH%k#$A!8mlxJO*& z?e!Y}eY^cs{kMI7h4-!BblpAW(A4+VOj_TNXhu)Wo{_aKW@YcpF8x2<@z<0gq2Jw_ zRQ=bHVPhk|P1b+Ec>UsTSF3&(nGw``z`Vr=+iV1tW~s>w;+2p5N-TOS7RCQe2 zjCINDS7~cz*ZXL=qW;jiW9MqB-?l&AIJI{Z|G?hm7I29Q^}s($PKSC`h2~e)3lgZv zuTMN%G@?QHqmpZmjjGmS);~upjX0K_oT9$|?SsWf1Lt+?-R1jgQP<-ZkGqv{YE_*j zw?0`rOHr$&rITrXXJeK1=Q~YZ(dz!)m_3WXD>Jocdh7ZvB9Cu+Hz{M@gDXeVtd-k; z0_}wTZe(&y*U#s^^KstBk^wrp#I6AwzT8?hr{RL{df&eP>GzdwZFB>>9Y0m?_PJ8k z-)V9D=$Ux)z`6FXewu#uhkfY{1M-edsTDr))}B%AHul_Ec}$PAevOw`emFR-=Ck4P zxnJ1_c5JBB{y8zNbd8e!BdYwgU`UM%-10Bl=PrAv;exAu4v(KSrrer<;>)`BT6xrN z<@%fno4+5J?(Yomw7uqz*~GLO_`0vdqAR3c9`aGC$(q{f+uQ#5*L#bGZ|XSs{G(>8 z6!WT%Jv?jwAj62qcJ;CEI=9{y9k`+9-;;LD=y0^2-=lqP@2xynqUoirM7hJu>?2xj zznn30^lDIZx~w~=4mh=Tpf-&v7YVfm${vsaUWSJr0U%zC%$rCZNVUrssNvqsCy8Bw?NfzH&X@TSwAG_Z}@(`xiPv)N{yhBoDCz!4*_y4-g;-M`Yw%zATJAyRarDFegHQUGd69mj;e@J<%>xpKJ!^X5Q&1(LC4o;`F@1OW zm{t|X4{f;MR#|RwgZ^<<8=t*5DlM#I=fQ1<)hL@AS$yHeN5_UQzIJiQ-*a`Xr_H~N zRXWsr0F|{{Rpytet*ZV!>ek+2y}x0Ik*Z27QYw8Ec&0;O#K5w>*00DuX&W=+p?2!5 zjz>TGxZ2Xl;E(>eP~Y#O5erIe_f_iD#IfJ0zh87U#E=m&Frn7zdwW|x%lPu+>4{&G zhiYfp9)D~7V4OkK{>F&W{jZ<>p!Kx6XR(r34qa?wx&j8zoH(*|w^eKRCO_^Pmw}l( z9<3HVI;Nb%yg z9ksstt#pq=@dKVSPb!z0Gp|Lj>M75eC+nXbut&5xUvIm%NwT>^!&2M6S)DoJ^x^K2 z6JmFdpAw`!(yaKHn~k3IEE`AG^Bq zyKHv7oqb)1?EsoFKn_08kq4?QJ zGr3K3!sjg>x#HQ`L$;Rw+nJ@wKVw6NkC-*9dA z{X96RV*JRxTONI`dS}p_g!8|csvNviX3t`sIxAHbFn9g@VzVb0sn_B8uwjJXgm_tMH`S1Y_fd;imKcBi!d zBy88IV;NtJ>omFbjUf|bKEsDy?ECQNrFz@m-hV0WeTZ(V9V_=%s;BSPXw|KRa^-FI znduia7dNlkJ)p)<{*${ZlE-uz|HSy^{ZoVShOLZzwYU9|O&J&f|Z?Yw%G#XI$;J2Tr)Tv!!S@$0fYn65S zT)7%uZp8|1+Hh{=5!=HSGw;{AeCkg!WAXCsjeg8KyY-iUa=(A}&ub)Vb zb+rS1?rtwNqRfrwww3T(7vJ;-46Yk&W9h{|k=g#i=#V?)BzWl+B4{g7nxv~0u^?hYRTAnj( zdFRX0^S_@rdg??@*?jU^<$-0&RpswjOo@70+J^tR>)PeCU27H$AN14F`ju9U-msxn z$nln^N^viGZO#p!RoA@Y>}l(n&F9-U#GQ@{5TlfCnZ>JvuyX?3+p`rd_yBkpc!F|EY575qx%AC{j@<2;nf>A(Hv8bg z&HauVDnH8k=;p1!uWOZEkQlTUXv4s9I^C9NjVZ>f;f2*0uW({5-nY#4X)V`rUZE=0v|%f%h-2 zY4v^uY;tzXCiMvQoQG5QHopJQgH{{vWL6nEFyusJ`3ae zqCej`^zpXqTY}l*)T+EMo;^D<<#yi19WQ=)kvs1C=eP6L-^}~z{=r|0Z!A~tv%3FG zOl(D+yqTWa(tO8SqL`)i?J>I>Z94sQI5n*JxijNGy4^o@?U1k5pPaOQN!F&Q%CyZ7 z_p38@P>pXFpRsRBJ{dgg*ykgr*VQLv>#{RVi%+dyXlvN{{6Ou-PA{sQ`?*H&piOZv zTGYB5yfUw8wcD1fdC^Uq|8Y3Cq3yHh-+cKr-v6NDs|wFzzdZ5N3+=RSBNK8jxBMQY zExL5eI<8{c#aVS~WORAnsNIBx?>yeHWBWaT|G03-+27k9Z&tb55 ze;T>!!=)d8Q89x0YW>u#q^{J{VpEhdQxEOV42VHIU zNpcvqp;}I}dfm5vzxwLl_b=_eH2vs&?cY{f*}KftKm^Mw^>Fd~xij zXLpa-p0>EwbzbELQ!>9R`{aj5c{}btc$(KUYt@Ooa`(exY2!EMbG;*?)5a1e_fci|M)kO%CkO{@4o-H2?hF1KYau8&Vhj`Rwzbv0tW}6AnMa=Lcg8PRz<6zhA%Y1-oVG z^23!Yj_kK#V6Ds#%2b^H^{uwnzZRt(dhzt|?oWq|Fr4YKGjrJIk4CU(t+o@o`>Qv` zS6#89)$${g<8K~~p8w^9Q?>)kKRwaopzX@WnddFvSAO2&=^rmzocpfq`5Lna-z#^1 zYv8s)TV~#RTmeTUxZ8f4ja_H{nVo1n-8rzG2qyE?2AF`&TPwx5gPSBEs%(3N=b!ITY| zM?0=N+2H=8zecX_f9gTtkB6pwn{(n?$;=JiPd^Cxbx5g6Wfyl?U-Prr`A8y&QH*4m| z!H+3pkk*uvFnmMv!zHdx-9K^tdq0k?z+&vs(9w@wS>SM=t+W@@Cm>jkoqR+4}6Lr_Igjwxe?AoU;oCs@5H> zH7EIxJs0*|SX$@f>SfH}#Kuo4MS(%X0lM z*JAeXs`Jn3ceX9*{K;q4?A^u=>b9^#qhYz3%dSFl2)KFUyIM;h;4H|fBKx2pJ^IjM?HZ1?4~%EfeBYuZ1}X!}v0j@)(aCwImj z-_mU#5&hXx|Ac!hMm9cM?3_5)V)OpfhhhH@AJ6k&;R=<~z~HaT z2`K;b-_WoyO{m-XZ+Jvl=-cz(H}N>4_%qN%upDF8I)15~?jiEH$qcm8EWy!yQOrQ# zUVSq-%%lYwYLx~#16ge(nIz(-9@jf>AaEKy*5iqNO>~WAc?4Xw(+;K9T4z6D%a05B3)k zBPf#+z2bW$#8PCHG^NCpRvK6Md*I3stiK~Xt;L=!4sAPV7fK{x7N}6Tg?|Crefd}I zg&^7dUkc%hQ!2)oAg2X>l1o&y{N_`1ats^xb1(^hA28mL9+D?wkjS5E=9s`5xWC3j@ zzjuGYuQ?82lpRno(;55DO5+GBZR3RdOTvHvngXoZ;=Ft-CG9p_bfW*P6Ex%uINxDP z41Os1a0(HN1qUe=h_zTbJFu~k1??KMq8Am2iB3`Rz&Qhsa{Fn4pC1C5VkJmg45=qf zu|kTlCuYIOcg+N&Rf7Dnlm^_YREKKSstBc39ib3!w2C)35w%M(wRjSC4{xmkN;PN( zC|VdD1&XGnb5zj+rCI<Cz;f0T}~ru@D?jxdOF8 z_iU&-juvx|03$}SP$ABQl?NnVlM|Q+;9O^(6(9;lT(ZmwUjUSXWBQ4|1PT%V;4qV) z(3o?67}~}Wlo-xa;-P@GVwn_b-pbGh$g+;{;YZFO1h6Lg)sRUu=_qJn7{B_Rt6OxZ z)br%;n09pSMO^nCr7JN04(~Zfj7t6oj#iD#uii0tge*$3?7L2?t(ETLxdL~%$fL34 z`=*BumaVsoVcSs1Y%64B&>X&yBIZtqP62%}=PigyNC7x);pu`l@@rcnbC8J~${nLi z$h}0Rj7CCDtz&7Bv;kf3@|Ruy5mOfcjAaQE1##iNJan`p`0TSXsBXCqq_NUEJT(0U zm@Xh?7v)8W@^O+AimWA+W>ZLPF1=Io@7%i}%c8)yh2f}4h_0*4d9kqOgygG2_Ij5P zW&VQGDy9{j_c7-yz4%Hmkh;LGS+_8ie4C7o0+>O-L=TNL14U*_u)-dJE_8|P-Xln- zgXeR!$;3Cn?;jY&5kShZ&+i=kU^&R&D&!gvA-bpJm$-o(W>QQ`9#a-_{Qu$!Dk=jb z_(29CfM$T&6&g;kj19>%I=u}?o3(Ri5{E(JF9y${AZg0Pf0jl;z@Ft7i!dM3&L`dE zbwWogN{R`KWDBVQe*F`ak-=Dwu^BiU0|7(xhzJZ0qi|rwSQa@SD};;V^1>*O8mp7# zZkH#wP`VaYqQb(olLU(b6o@MZ1t8`*xE@NknRpJv&=!{kWWgxHD5NFucb1c_B7fwB zF2C!;zkyT+!O#@q`NI4O0sRmQV{Itjwb4cb3t7GXED(f-SQOG0`0z&Bm&;b87G0Mi7LjgypLkjIJn1BHP4dtiJK_cTfd4^vn9BrjZ+GIx&k@>bfSb#!^awuQsI;j>4fDZgzbpc-V<1z8` z7*avnj>oKm=qn>RkbyCTu)6s{rL8v9d72Wrs>DorOk~x>?1UD~ieycJl0+3qh^`=A zbU=+*kqU)G6b|DcPtGi_1)+>6u3{Z4cH3E7XO$E%nn7fS=*sTrV*~QwnjnBDV=)Ie z8!Sb!XB&tup=^ud<|D|u;$=sFH#iu>lLuoVD72lSVDd&Aue@PKel?;{DB=neVLOWJ zOblj#?t-NYu{ky{Bxi^^8gRJ#d)Yo^)UA_1X@Ot1B!2W&IVezNU?2{8QX*MZHfLqh z2@3CvexWQweTCsfzd>A#Kqr|z2U7|m_xYg}R2CKlNLTqJ_AfAfj0F@;DH)pPB+;*d zNErl4>x`_J&qy7j5hy0p3^_9zIBDN->?{(i;h__}>`Y`3Bs4)BSOnfCz;wQs?|=?r zDN_tLFRU+dE+w%<+@r(@G?Ko}i>0U*u5-N80>3~%X=4!B4@Q#CIxH`~N~$V!1_yG; zy?pzgySY$9dY44w$?{N-{wUCp5Mj{)!0I)1jVMIQ5;+1tjWD4Gt4%^fys21lP*Pe! zpbin$7*SIx6jCk;6M?!lA%|2H)ZCID^4P@_{0xXoj}=k@2-L~a7MbuYN^Fu*LM*R! z*>*F{kOG+)!AZ)uCqohTJsgry5c81y#Q()hPkdG9KC2c-&L?!9Q}X-LiTYI^Z(5j(_9C^+*{#s6uW0*5a-p!>JB49M-f{L_R69gD zn1`d&JK}G6>P&@bRkR-D+pJi|ce`@Q(=olG;m<`lkoD6QEoYlzA90FxlSiFGKsGNa z_f=&J z809oO2`3&P0Lx_g8YY7k$i(ReJiZp%MsZSyB&Eq|xsnu7R7?x}_$^E+yi*D)o1kxC zoY@JD2ZibB3@9NWf;SQ*WbG^mEe^&b$w>jC#ON|AJ6b;bZ4C#Vt%f<@4i8T{eD^E+Z)klzaiF*9` zA39c}6KWO|Sm&trfQ2Sr6BSR6I!Cov#tI4yKfbO2<>ZLWxC~8!UbJ3RhdtFe#~OA` zUL+pLNirfUO|_VT2Ix^q1UW$5&3R>s1y+*|w^^(hN%Ev17S{>Ib*^VWelI`#5$h|3 z`hL=QMG4%l9kheE7|&)r623VOq~t>_keJvJ2C_C1nV&sjImXvwR|>zCHimRSlr(x^ zV@N?#p=eGF9vSRidz4hgnFk^&Ey>|!Ac{#j1M8r4&jUuG*$vEtmpmM1P@=54RNq_O zUCzC?E>(4BXxWLPqOQjkf_rq0vZ5O`@nl`r$FApSVjz0Z;2Mo)qq1gQAJLphqVBGb zBWK<7Rn>j;bPs}?v-yYS54vwx{l2fh`rg%N>KF5$IBVI`+=8zZUprHL@Gt$Ve%3A9 zCUhPzou;wdcXW1cy5=YE&zW%Yt&>jq>gg{`UD`SKjgAw3`R1%;FLwTL)DAvj^ukpq zo;YswmO~4#{Zi|xOV@uiw0_5vl}T*}W>1^>>(@FTe`oj3<@ayBH zf82gXuKk4Ln(vKYHR@i(Y?2=ld+UAj#N37DQ-A%? z%5Oh*-9T@v+}g6{;NtZUFFNCv;@%Ad*UumO@elWoC(mE;y(@QJ^|1fV9n%;5?1kGV zpML$F&gmZ>%3bv4%$1M-?O!gbv_3g$=jdBjUj6F4W$!((bq!y!d3#ISqd$JT@||6a z?znqi`PGwW>}$HV@`H&(cmC^|ec%1b!Y$KIT6~3n*0Qs{kUeK`+PEA4e8OMx8>jtXy=_e_i`oTjsLug&!|l(7U?g?U`}TI5Brs=EkNamrQ*0 zl%DH1ytjSX8{K0jtZ7GQL`GI-B>YY=%a$oO^X3!AKR*4Q_a{$1XWEz#*WGdF#w9au-@SdZ`>hkFJho)h zhK^B_U!B08e+=zMFa7(H9b=|WdWdhG_Pu3y?7r05-TA_ApZ?km=imo7lY{R~ef^UK z-`_v+q}SM~1fxW+nb^`ZZH=8g|`?!i}{{QiV} z=>b^Yf0hnAiA?kBr`?lcC|p8oKgPj&VNlTW(f-OKLTwU8bkezfr5EgyeS z`Q*`|Uw!@4-~H!hqeqSU^was?1Di%=M*p$@>eaDV^s4>)Z|XK=k+r$=hq?s#@PGfU zt-XEj@W21oeopJ$V}Jkc|KmkHTv8Wtb5f#0@=88_mrJfn%&9vGbLvt=P68&VWH1!l z)Tw(Z_Z*ee4ktV4J`sPKwDDmHqK$ zm^sRt9cGJWoB1{gqE5v#FBGMRum9lp<`x%+GWNBL9~b*61c455no1k zjy8vZqwXH<>g^V2kjgSTFCczL&+-fAT!3BbL^t3Q9~WF=+1#-P;^%oJc}NryusMma zUqL<@L=Kj?fCWJTa}+8IC32}pjX)6dktln}BOc?G1YVRq%iLj~Koe;@uwb9qYPI?v zbyQ>=_2|n&b7-*_$O;Mt5opCcVQv&f-Ccb-0GGC7m6-rJHz>%yfVvDh%g;7JTHr$r;!*hc{;14 zD;klO{?;-yEXgdj-aIN|SCEX^i2#5Eb%>3*_8Ul6Z@SQkdWesbeCdKA3^LwNxX7cq zVEBn)n>&OHp!xB(haxfg^M$z?w=up%R(%f(hc}Kg7)}D0+S+SGdl&16-t6dmE1I1L z0pT!4$rjdDAhEYDR1o#4bP5@y2d2>CD}fE?$t}j3eTNir=t|VRx(_+b z1HE8HFzZOo(O{YIlyqtOYGgCU(lhh1X}HYBuENKTLv?&Yu5!o-3qeahabOX!ZxJ0< zxP;hBha*%yAs<~29my&mUsT+vF{>bx1xAQzIi(dPMG0i3$;E5P!n5H^z)6?^TG&w} z$pGP%DmSccE)&m zq{uiiewGRy9*C28dYEn{VL9dQB`cMH0E7BINx;ueWK_vXQiuxS`n1YZ|3t}9#R--vo@KB{PRv~m$oKAw05kur(>{?N{%rC*$ zkrXud0*poPM-=olL|O*QqF7N+P-#U(0PLUoMUI8!p)EsB5NE)Mc5v8tP=JL{cW^7H z>cN&8Y-voZ+8#H$+3LYrHCS)EON9iEjK#<@mdb+{2%>(Oa0=R7@1 z)X}qCVKE%avZV%&xc9LSikxs9#w76q7fYg3uI^6M)WaOq9a{(LRMs&|>mXDIPC@&x zBI6#0v@5PCHXGF#$_7qBCu0!gIoOG00drZY0$xYas&N5;J1gKRK&6>llT7sk1Yt4y zc7H`AP}P%4Q6!cG)LB3Q#ELa(oiNKzW?Mm2doixmOzBmnVT1rw<26wesIee?fZ9Yf z6W<9K^`)3)D1qTe!AGaL@WdF0nuM@zv(T4XQB*KOSfF$J&RgAlVaKYKQI@XlLFFHX zN|HQ84#K!_EldM#8E7kkP9z#_;czyNj7~i&+7J>A4Va_l2unr<>N~KGqgtm>HW{ADS6O$QF{s4O z)WfF!te>x>+=&%In~ zi7YF@MS(gF<&kd(bPB}^9iT2L5eIoXm}t!8c{=5Yv(rH=?J^O@p_#pm@<5Xml;{BQ zlimi@0cw_G#n1An;mn385^H~tT{oE{pk9dD0#L{4nl&jjfPQWIq-R9WdYIE|CYtcv z)pNcsCKG_fxZ<1#s%M?GTXqme)mTGoLyTeA(5d86K$&^4VWw_0rQuOAWeP^aS9?p# zDw@XnJj_+spo%{OTE^T_Yxby5z0gBxO$|6GhFj4oVvVaeU4WDj+uF^#6|7uztL`o6Xb$~bQ1q)DJAk}Ks z@c?D2wc;}DpaOQWZzpY!l?*$b=9yN;?$v$hQs!$rjo(0xk zP>n<4SOA$vP9bNSKRKIeJsMts2Z3ABMW#*whEr4(oG`^-RaqXV26apGO{NaNv|hDd zRpmyMWhEfP)~eDxld!<ebdqLDh7b4olXI8JTXnLW#FG zO2^VUO=_LhY^6M-UZdlSVRcKAyKIJvYACl^s{?3*Nmq`Fk7(s1F%qr%Fq~vzOnr7T zkai@RNFC&mLRcyRD@M*q(@YBD7L65k!?1kSb`B6;V1j4_7fdp>OM!6_piTRGa99lH{%`>_)k0q7%^*OV2Rms{5q4SR}SmHp7&P4!w)})oIXbE;C?MZzohMMr8fEgt_K&dKV7Vzt- z0?sTbDz1|bXVkDz!}6#BLReNm%j(izmq}Qb6EuLr1vWq;1`K?WxZ4P6igCnXSD^#&~N>x_yO(3L@9>0x<9!mJHDjNNQ&0k%F!3ScwEFdBm6IaB*G7%IsoYcY%DX0n_TJ z+WnmQ=n_^iZIjySGqz!BW49CA{h?7nbxafGh$7l*R}Vx5HBuornP?74vp_+~azZb_ zt_!MQhFf~LRee0z@Y&~sW`0)BZ;0g8p}pM-QbEhRYsrW?#=BYlq+(GgIkQZQTC12@ z2qz+R)iFgaRxDqylGX3CA$Ma$qq(eTuf+BUl~U7ho~b8CIMYtWEWt4K=lj7@}U<7z!{|W1%rZokmj%QCK*jX)x2MF}@sO zbbjy3v`50BEJMgKC!bC6!6@Vgsx7X4e}Fx!>ZcWnr%U=duo2jCRQn|*5VHqSj-b0( z>a+mHf^R7ou!`PLoEo=mx{u0nI81NUN^{hr^<=P2?Q$%0pFoHylhn2-?Vvs)*e=(r z`jaEuWFxc`qXU@Q@6Dl4#EDs%Ir9NiFA&wXALLX@;=^uXKYfrhTT2G=%-X{$SU+cy z*icu+Qc&z}bpDtw;gm>+Z*hzqP|bX`!l!7$XEK^jk_K};^!vphbJQ9V*kAex}8Pd!p> zN6uMS;;Eln9Krl3vYOd(=W$qDt_HwNAF1(JueEnLjM|Rls#M4E2!TdwIIcQe!|^k~ zV#ajSju)_+`n-fVlCpn`y5@4g< z9s0CmBpMj?RgL>>V8B^xeh(Pg)G!R~|8r{S@}kO61d+N~sGz_~0BknG8i%HjI&?Op ziiXg2wI4F&d2k3UtMy4zLng!1O?2I<){emrLZuF#`lL5D(T}KC zpBmdbd~DpZ?TCIB5ucQ*6F+rm-iUgS;-`^`_iEM{*?>7&pE;ztF*;_=V&BPPU(#%R z7MuLgV)GC%w3fMqn?~3lk|zl1Qqh+&h2~81UgXjGi-CdF%puXK{pVr;LSMENQin9d w>a&8l`ZtHo8Tj0%J~FNr{Kx*8@3Ggh*Rj{JSIz5x0{{U3|GrFW?*Nbi0N<|8z5oCK diff --git a/charts/postgres-operator/postgres-operator-1.5.0.tgz b/charts/postgres-operator/postgres-operator-1.5.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..f575f7cfd04c55f39bf421afd6ce0a63ddb654db GIT binary patch literal 15843 zcmZ|WQ*^RvM%~m! z)tv8B#BnefApaRaS`d0u88sGj8AUDyUtUf#b}beQbxvD7bzUxIEiEoZZ3jD3CktN< zRcApNO9w}gKmE4e$6d`d=L_%jEVP5}WaSdfRLIb;kzDX2p z=vcCgAT*lmY=ePHCgfcwRPED5aWBS+=U$t`x~Ja+F95%AB?*a1TdK8McyY$_@M0_sw=hanm`YSR zmso3wO*L)cGghQ46U718y?~-EpE$=g$&bs?lW1 zH#NZ#huIfisF%~^A>dvno)5Iem><+_9~@(xAu$X?-cP;=B_prA4V@%;Da#X0LH&oV zTvwPVn!*w4sxst^ugrI+Z7+5*k{B%%0#JYO@_g`8<{}4fPb$BgG4Z8zAvo}B|B#3J zkQ8_5;W?OPlNf>S3TWfwa@Y1vd{hiukgN(#s$``hGl0`=+;1lNgaP_WRsmB4IQX53psHQ_9H&Rqep~C zGRwVB#32?V3fq^!I5`QO)-RvXBB+D{8%+_$rN97m@ehp{hoqa*>NX}W60F9dg`P_e z{zRBx+A}#;Y|!L#l7-qA%b(&#JnS#Ra2!SJP(MgPG`t@n<)BDxSe{;XT_Zy`MMHN< z7Ou8YFVZcGlzT@J9(=fQO3P`o2;ihr=a1Sx&sfSbb->FAhWzC%&Qo?W=_>!4?|^7_ z&Jgw#G=O;-{E0!RDDEi_=}>^;HFrR8H8cnsDgG?lbgo~V61i6y;wVaD;;8QgQS{t9&-2+DP$OqUNCZO3T2BGstAs${taS zrjaXsC!>=_o(5$-G&it|Nl($Cm?cM9g4~UR^5EFOkSeK;Pu1&1MSufENuktC)W!2+ zcF~AhD2mavYN@=)CO)n<+gLLk>ii65TeA}pL}2lfQQYTnL!&;cvGFRH*j}ndXxual zXIRn9fDr@oI+q76Umytb9vbU$tq>igHF4zu!hs*?Mx(ZxWb2 zr#+yz*vt*xk7I&3t=p4%bn&vPM9k&~4vjmTLPdMKlVKyEo+B6TEm1Mqo!lZ86QmDv$@q_#vYq z!&)W8w3?kllj^_~P9?%}sG$~7sZKKpk__X1f@L&LraH7*N?*-rltGPkIkrX5DIt-t z%AZrPV+Pk64@EK&W$A;8DWGll+b2a@Z{j|p0T*GWcjP1FCqw?QH9uAlG+0Ir6%zeA zXUu~P4%)-{{+ zlN@O}5?r%OAc1?Cz#1(0sC zc4KKToYg7-F!>DugEbnd3%4>Y8^JT2Y_+2jfDjxOf|fDJgWGWvKl%%q zx;yWde9B3t<|p2|;wZPQ3)$k-SU1>=3)7joW|L zQ7jbZ5@Qb|TFbch64PQA$!A8C#8dv>ptT;Dl1a68?bh9R9f2n70gHt%9Va8LZeo)N z@a%H*@AtmaUVl4RtPz78cMb#G*ti?Q=$*4)T03+hw;{`O-4s1Z$l6VQf|WBM@|Z8zocUV?hKLGc!hOJaMu&s54|%(j zo!On&d3Zi-JeY?l4+j-@a!|3O6-&T_Ah!D$uVd!P`A1vwP>-;XvlnWGbF>6VIm_xb zVX+Wv)Z6^oOHg_3ZAj0~(LUl~x`bh2t4oE5=XadNdVy^--U#qO{Cg87nfcS#zqoxR zuCOgZ7lcMDQ(beYZk@r7z^a>lDIO7y1p!ayq5?mJ?ODo(hh@C18Jk;dS|=H*v2^%1 z;j^h@xWN{S<{NI2`z-m|SwscL6#B6`ebrvX1bqG0nopA}vQKzdu*FCH5MD%Qv;onN zFLIHdqg65%YxDL^(SFMx*4eo2JA79P@2axZJ-MkEyQXyYzp9u}Ia=do6xx^?F)ZHT zSIw*RY*fWp?zPT`>@Z8AXw7x|*V_>+o~7Y+a^yywU3bnmhtS@#G16b^O2U?+kPbyJ2>FeIQ*Fd41`pF((Nk)cZByDi`bPXvI-278x$*J};(YEQmb*s_oRt$wkA@P2wFW zSMaqkiz4S3W>p#;(SFD{I8XPj{Bi&13-jZrT0ekMb4b3tp;{=7B-HT&FaA@nX`6bw zP_~3MgrL}FnVgt)){xmkb}kmWBG2+(Og~x=wyLAbfzi~VBgOW1HN3_bsG9p!Sm0YR zfNHW%(Oy3#DMk0C&X=jGVKNT)I8N6n8Js|A%YYxk-r9}wX!%nBQ1utLyu`u&Q@Xe_0E+(H{ELz5&$LzWuA1+FnGgH6OmDu@S|7nbX;DLChem_2Hu|CgTmyN}+05HmMpil6uJrIPYtNxnZnJw1)B+An9QYsC$I@l;h zYQRDN=9;QhKjep>+i0$?!&nnV^r)+v2YCUpg_rQX!C}scz=Z$L3pkz3MgN!}N)Wo<{Dauhq->0Tx$1k4C(;iLMc8kpkJd z?UfBXNB#%z99t2UGv>07>4i0PQ2f}i&V~W@CNs5jGD&MhnUy@o9z#)Ol*@ zHsx;y$zW^rqot}}MNX6}C=-n?KB@R%8uNW8 z1QQ}~6>mD&#F^emy|tZ4ieDld#J0BR1pVjR$^1*7x6&;lXV%Y9W1$D$NBAu`D^;6X z&(MbGxff{EWQl7+(u5iK7Z#}J`=aya&`XOfxp4YB>nq=q zc9c`dKW3FE$E3q_9VTGA&HkoJx?N1ZiHdPG)JQUx*c*08-68Gv8Fk<*zk?q_Vp->H zD7D1?IwSZ?<9vC%uUo6}WV$DbSs{CFI5vkZ7k%#N5m@lxW|k<&is0ucWjmuhrKbiA zHrkC&4C07sG6#w9YC3)LycZ+mk?EuKqN{bqnK-#((B1;LN38Vf@k$6S?chPD!H#m2 zo79j9mhH^i6Zd|eXOHjYIASXXdb{Jz?+R2vNNnc$sMp&7D*TLFh8tD>P++UqQja}u zV;4$v#z$UcZT$s3WhI?14xXq^qlo-5VQZ@|jf`N^`m9KwfMT5zVba+x3SxmV{Flk7 zcfY>cUp%LQVFoT>KaiuG-B|f~PJ**)&G%)!qkON89$VfP2 zz@XEvH_n^LNt^ibMLR$m`hoE~{v|%MmfC65$oE%2MQOiDaeQ;uo!^jV?yQifCbNbj zItz;`0?#lD2VD*+Ky?hMR&4`@9cO`T$UoUObuRr9~F-#h>+^Trn#&qU~5 zH5jNg$WW8K4MY{4ve97|x;9Leb^!#wpd?wscqSpM6*ZpaAY9!2zZ%d>=Rm>b@BNo$ zN2INjJNbso^dUuC#GX-V9)fHUcC=+$oB%KJgNL;#+a*Q*NXp>iZ%#21C;t2-X~(8E z|Ejx6On%9^5bs6iq9|KT2c^T~-`QArcR<}~o*&=pb7aHGDN(y5DBK+IVnjb0)M%Y; z)@zGxx+Z_@_1leNmDZ|tuLVN$Y)}=gJv6`}^qcwDoWE<>W^u7dhKCJ+Hd({i^Q`Xo zEQ=S<4hjG`SGbw*@F5z(G#l~!y6{@ty@&$qxJOl^yn7#aU(tbz=QH&lF^m;;P;vk8nVUst+71hc!f>MUpNRRw$T)lVEjVr3TCNR z^HOeKyiXexhx+mCE%IA1E(Y;y6Am;yg1BO)BUa!P;a>dsVYBzn05Eq z`ckf&CU$1MH@!MPJNZs8NQuu!W@C_MqE*kR zl%g&w*-^r_upniHH|QmQ5{?@Tdu$C+1m~q@+B**RWsuYbNuclCd6zb!-T(L;9sH{< z3^cEldD;)|3JP3HB|Z`SrQ`YS4D=av`#yVfae4FjK4UGgk8*Qs= zKv`Hp-BcO+A&y&0;xddW9Pl0Qmq|gTleFjm7c1~9)=I(-*&L;mX}-lMI}seB>W_tQ zXOx6;cath|;MaEGArA5fgFf(x_Xqokq_Wj3A6%E&7coMi_6G1VeE7HRm+<)GyE~BI zLq0V*<&QcZdiUfca#8DV7bU6l-tUb@LYg(-FIIJ@!J6r!`8aQd?xAzp>slU~X;2klUOM1g*P> za7mLgT^m<<%BUNERKZ!Nix$CcR;jD=bJq<FFrWo^4M8 zg;hlsT^bXmT$Ylm{*v7fT5W%436KSIGO2t*|JsmkcPzdf-6;3lp8Ug_5Ty_oseP9W zYr9D>!WY#C2WTVLGBDgZd%=s#N`Sy?P!vg#7&Gs$N2Ar#bR3A8OMaemNnz6gW@4 zqUKSGS>AMb&#w3kvc+`!F$|CTwB{*D&~gZfYdPLtD&XLTrwIKU@c!QPnxeHX*R{%yiI&Y9V@|fFfAwR$C<=AbX4oC^WF1-zxjOIy61B~1*{dX~ zHv&4~Tw&s$O$`8ps z%!MJVRg-zAJpdSC7iV$)^iav}J>Kc@p)JkOP|HFCa#iHW`a~;JC@-&T0hp+h!zY{+u$Fy@b{0yM;J71Ohv!@o{Fx&i2Pqv|PAyNe+t5$6e z9K~5F3)3I%vf+ULBaj!6*2n0dffi}nCgUn5p62Q$9}2nX(Lq>KE4AoC*;aZhu+oVF z`!N;a*E@>DA4yk-ta){GWg9`J)&243d(?2HNFUle!LqnVyarm*=JHki-aH2^yA}sz z*B%0ML%Lg8D5c48fon#+_UK~|rY1b0=DyR2XYb4NGvmPi6bL7ciK;o9Z5loPNIeZ5 zvN42XtsB1d+qll!rA@U0|G88QN|N~xFZx{6bfpAdQ(o466&%~V8i*TmT8+5?`7;0E zei2gjmD>|0q`lkp;Yf&;t$jc>wsoTf@>iMmCxJDtY@`);4SL{8yb$xcxte|%r%Gkq6fJQV!jr98Gy3Q z3SU2=1mNJk{xn#C8k}gY_;(-S)XOPE^Lzj6s5JIVSb&c}XbYHl|0((cq*r8Cb_$7h z0dLAONV$+9t_~_?KGz*bxZ2C|M;e%X?mkteW*M3xJN-_*+C5mol77M2x!b?hod(u# z928^@H|_xagA5XYd_+Wo_rM7sh2Mzp(Zngg8}MG8@fzVG!mfXj&Fuu?DnFZ%ea~h0 z5q4DY-*xN?M-H?M9#AyQ)_21Y1&VSc$j~i)w=7@QoN&(gg}Pm7Hf37W>+TFr+sx6u zEGM+<-`&XSv;um_tODQm66me~DZG~hR`-|POM2}^C}r~R!ihu_$dl7=K+$v{F|lB9 zEHE&wvJMz6@UR+8f&o~;6%C$m$`ck3E?_XRO$-w0A6(9g*~`sRRKY$k1Ll{cH5AVX z{{oY^oW2u+0V{ly^LTp-$hF@3U^a+(Jp!X&2OiuX6!idwH-o2vk2HymO9u?-k(|L= zJ65+>p1=;gZ7;g|Nf{nGr$%&g&*%Q8GqqU~c5bD{dRV z6vxmw$mj3N^NEh3ZYQqQ2{_eE6TKtfTBdq4=w-%=%i?yY-DOXSiD0#1f4$x{rxuIb zJ5>%zj{}EdlAJd`hsMhwH`v{6iA~jQpp$8@kaq1JjT*;_%T`RMi9WqhiDjDt&^gmD zx4l^mxG!8Y2E4fy4(0>i(=5>0nxKCbd~mq<`D(dCYdcYO6M7{OF2Nj3Y4-J#stuev zi8C`VYuk5d*LL|MdzR;_t@lKGs)P5f&&uYij!^~KuCRN*mrQrGDim~_fmVa2Grf*} z0G!p^1M2@q6}W(d@Bd&uI?~N*2e0}&MhN!(RX@XG^OS~9lRUr`sF>MJ$mK160o8B^ zc8#lO%@XOx8^eX&2ddF~&_JlxTl#fpJT}~|@STh4f6Ex6`FAkf?sMVpefT%+_nB(J zIH4hVQNd@>>+9ct{S5<#`S(aP9)lhd(et;s;kZz&nmk#o7N3X)W#(1<>b55ML7gqn zt_1zl@7CA)A1@caP1cCt&QHD9V?#Ax`$yrXp1`xW^Na6JY2WF;gXwXmIpm_XW+PCb zLSue=#+w3u@#cYFuO?MbgpLp-n$=m;38lnRd)UT z26+9asF1ai0sH*(Py6{3+x)kZ*w)2T%%8j@7n~n- zZbhrWcwc!E6T7v+0P`X|%8Ums4p?Tyby)4JMMH*jkJuR%WpulQ=ZoYs85!9z z$I6u2(}7_=+H7y5ae2wcM0B#;RbVk%-}AJ-SkO9iqoesVl%0AoB7p|)ng#7wO(EqN zc?jj34~GRhbWMD6v*=GUo3!>Kp79pAvDeeXl}p2zv84J0-CX$H(OW{raChePm@yfres8J^*W{$1gcMpMZKUi=Xcp)>fG-6qtol?m$!zvRN}de-CkLxW`+xbIYl>5nq7r%Q-(oIgwv+;cJYaxJdFroCL8Hh5 zrK!#4zbAaSPm{k|qcLy4bXg>BEdx!-QWf;n4#$4c+#FoLZN$h5Q_P_c;&3d@yKz{1 zIQa43t?hP`QPuS2YCsy{4Rkos^F6Tx(+AoPFIrL~1@N?~+x7fMbQD~+H3Y{CvCQ-B zgchs_q#0otKuJ%hbd1%SL^W}x=$`MlcNw*SJlI^unqlnCL8Y+Y=az_eb*JvojU`ry zLqi)r|iFd%L|D|R}@h;OP?a+y-2Rn ztjw034!AknexVcCt8=y1zQi7njErXU_azUMb4NJiyOzig#B{RD+X3X?jDl6*gXW)6 zZKOS6c2VWXR+)Z|7{4~r#F$>R951#IqMp@etU!p;^PFcs)~+A@y{Bq8QRP|cU1!q+ zo7ZaZOY9FK*DKE3?`hgv=Xy)fN5i?+Q1U+C3}ZJ9>BI=o2Z)3eXRzBFm(#EDL!rF3 zx`@X0pjUq03h?+DP%?@=-+)$i?ON0|a&O`3;maE!7UdzSQvxu`lTsC7lPcUtswtMH z*Pa&D9n~I3C#n8q#4D~&jSK#(YsYVtnl-QP5{*P0i$!u@=;Eky8qa5c=592qeCzyI z2BJcn-WmFzgeP0hQR_TNUv#+_87^r+!N#0MQ7bGEY}HbLEKeeq4r{kV?O$FDkTqZi zk=edmcf~O$?Z}g`T3&IYCdhI4bEq`|4ch71SrekcJtxTjDyfE6u!tegR^UpIf>rKI zN#p#UFmo2h1zLj8TD>^XxxPKOU0iRplhVdJ!5r2-NT|Nz?I3w{Sdgk1)yyd6dR@Sv zPI*;r#C=9-HEI>{eFWV4V*C(#f6#yWx^eE`FIgD1hBkivZh1We>TEa6$^7kN%w${N z>%B1i&yqc30o`h424?4J_VbFX1#5#3P59q>-~r)X>8&4PG2p$TiQvww@!!deme(oL=^ z^m21gw@Ew?zY0fa@@b?ge~cIE^mM~;OJz25XCUF}ZG^EaW&agp=aH=L%4~6N?nWjM zz8t8Y9#7Ly7~qUR%XRlJoo^q)wbq&EhW4T!&ztR7Quv=1siybQHfwIe*sxDZH1`Yv zbUmC-pXK$XZ%l<9zOp0kxzE<=6d(A}C>peRpfK-dwRunXx^Z;pN!AAFmi`Vg$Z7_M z4zxvv1gjM?sQOMY-muYRrw%oz=J|~C{`K`sJKR(5+rWd}?c>2lERn_h4M%%-q$lf? zwYfWwwY{mS^B#9=8!o}E#!|S))b9xLMZNSjAxl4V6Mt^{-#l9U2q0%hCV0I)cw$kh zu2)7bxDvS~c07MOm{84+dQ=zteFePj#Rc^@?N1f2T05mk2>$f4lum?cH<3k6lbEBdirjT< zQUG>9#3E9x%(!ffrKNM=Rq|5Nnt>xZxZ?AK7lF^5Py_N5M_>N96lEygxK{@4B{t{@hIhP~ywU|9ZG5q?LajfB&Jz=SH`?U(**a4sU>opgh zTo(}{)D+eEx4TT{g9|d$?KxK9+o_nXF0(?BTV<@e;&|P9wS2yB-Xk&NS#Tdh((53o z-}`&v=z+Ji(L@Y<~dwo zt4G?W0al4Gp|8Q?1w58iNn%`0l$w>*+Iu!Maiuc>TF4~c&!oIi+9>WDH0C4$@9iJn zSua?!aVW-#H&UJ5y|Y=sUTaT$*uwX&g0!piERf`E@GL}Ld5kJ$5U$Mva`O2}3Ri~< zyD6dAG$c>iZf=%*uc}`l^&hEV3qxk=rPMDK(aMrlX&20p)wMQzVv{VhyQx2oFuW#W z(pL9iGU<`*V-q5t94KLQKN@gp-`61JpXQ}%kTuJQlMk7?!Lxbh&a^XCj|l$qb^kfX z`N{0l0ZTRY`936_L&t#fReYB9`8Pygs>(NwV~2|BAlO^LO%nRoE(~PthJtnY5y7v# zO`DslD<-9y39%>Wc6Nley*Ewvj#@Lzw*@xf4^=q4>pe`mzEPXn?GLE(mGaQLT7wVz z`IU3)AN#?SDc?;@;!SIY%nwsopO+nL7E!!!HAaRUg! zaY1nde}H^2NCyjFrN+?1OSk@m&qSRn`xx8NozR0wVPE^TQD!*YKi3(uyIwHy(w=ag zW3&okws)`t6Aj<}qwxeLUwV0PlyQ-N#9HIRck6WcuXlXWB8_|V>@iVD>1iGpAjSUM zcxHcaAKnW(7H<$Cm=NJLnECtLvkTZu2M)pW8F;&4|NfWhbD25R203US|C%mE^Hwf6 zfS2gc_4hh^{imh(>jv=q>F-|VuJIsp&XA9ao&njppArf+`vdFBYadGf9NtqW2o&T6 z4PBf6`-ZxQi_0CFp{+17-zDKKDer{Zlp|lc1W6s#1G+ry2-pEe^MwYAt^o}>?#IPP z8*1tXv1#TIvd(VgSdu1S)cQU=MsZhYAeT3yPCPAm^g5+_?f=KtgRE& zJ-yPu)X?ER5OVOCxld+KJG{UwKX45hjIy)8=$(F`U){o_uAN__#}&ZFz$;HomxG2S z!SU{OP`>3LP{@E9jZ9cdZ?;~iF6iQ&Nn>>tu6NRwBu{p=tx%NJtU&nvi|kLj%z#dy zOye`o8yVtRUO59cJz1-spa3R}IZl>K9K8(MCu+tz&v+}V{DiuKxmT~ku-&CPZ|Bib zwXP#JdmU+PZn5F2T3)^J)=hyx(#lguOQ&UyE}KHDh_x9Awx zj6m|5Qm$+L z>*b=9KV#w_3*%U@hB)95b06=9Gat2G@rsSgVS{B02tB2kukb~%!9eARahd0li$U>` z`WbwgK$L-3m-}@Al2LvyW@`7+7>{iGPfys*w#-@EF8PGQb`Va zI4m6Fz=~y9bW?m_gt2at zqJjXe=e{no$#Mg<=Q3FsNHf$(phwwrB44{{|Iw@c>&zfcs>HavWx~aPBBdK|Yscw% z9^qGurU7B+-iKLJp9+5tu98WGEt)i02_hQ5cx8Qlh}#nswI?8cTTq-^AX$OQ0Du(t z8na-@W0`mWkebwX@=#Sn3_aoY6fG=r21e~ITAvdJMg^4t1AcY~u>U^r68hGa@!CR~ zP2CAd$yIi9-_8E$Ov!cU%XOZQOpE=d;41ui%8inXEY-|K_`CBi_WNq5sj)%)>uN2x zVn!o#2mc?}Q?zq1k-rVYrtc0VHvI)7Pv|??p{={_{Clq>myYc@P*5$%mf5g8A%B{5 zC~p}>{z1EE43P!5vQsC8c*?Byl60?Sd(F#`so~_Sf|}Gx2HhWVI=iBVc*>)v`)?Oz zhi^GXN0x)8@0Jqe)A6&g8$oCAmrU*vQhLND9I2p3u#E9u>jtrG)(n>;p4=u&%sLr% zo&v0~UQ^KyJz5Kg-le#+*+e%9@&V@_hx9a+Sc~sfz!-1VGH&k;WgKkIxQ`b6v0DuU zjUa(EM_SBp7eH1X_b$wGkMZCMm0<`~90L6C5!E)*V(p+|o`Ip^K)>B*PRmLAez1##~)4k8TqRLU{hO1!;CJu;;#&5;l9BBw(Iqej%O_xDpjeY&c|2a z>($KTRYpeUS^&^gcQnCZ`Bpa>=np)Xgad!-)m(&BQI{)tHDdYUP5|msJpl zfRK<#K$u8jvRP~Ol!27|3?@gFcY(S;i89brM$qi5!e1ec)n&p*V+YDiE?ZdM^1lJJ z24Q$BrUymkyL>m~sxzj?l{5`>3`N$Z#?Mox4xppU74JkV0Q9f(SC4w1t<(TU9g4%= zd)`DlpcgHathgXAP>s^5Hmw~BmZ(8COd1-RxqvuO9xZPmtB*iImIo1eWGe2fbZSN% z4Y4ql#t)Y;(gKf&z}!tBfE7n7T6%&RU?fyXaCpW{ZTj?S(xPl6l>D1mA}o520w#@< zoEAf0OdFaRP(Vy{t>5#pE)cY(o(c3zrGxQ(Qjr^~`2>z06p;ZF3y`CfAAqbx-dmOZ zE$$H+hI`fjEv%~srdv%z{{ibg2OY1BiTlWT_2{0nl~)BL7cv>T-u!u_k0jeli9VL; zN3a78ei98R_n|woMDuHFjDLMBHsRg-Mu^I8FcbB$TyqnqU95kenorMrV!WwwK#wk$2 zo=;TE?LD&=5j_c0Bg)JRt3Gz$801qeBvhRgOlNN|W&wzs+&y7ZT>o!09ozp8P0M}e z{|}m`C>1XZYI_<)Kf}JEihp`_jO=H3SkKe5jY}HDrt_6wHO6QLUD^xz)_p3cY~}u zBcd5(t0FLKegw!ixU-L1QL%woa0zmzLTukt8`{Ju{#aPUmiUu^Z}=#W9{nzUsH@Gb zHH&QBT~^&q{&O1cKd8Cplzm=|ECQbxr(4B%6>F5NZE9?Z)TKKt4mUp#tae_p_aOoQ z`;=_lJFn&p33oJ6hEZ$b{qz*z*BQ!bh1$tqYx#RVv0@zSgNsy&LlNtA$DlGpy-#Hr z@l-hDA?kLs$kXHH8}4FtpQTB7z{LV^leRIH!1Sz`EVt~^$zg_7BUnV}96e;P^y9zi zvW&D5>IT5;&krScCfB^~W<1_e6X{Wg&?m751c~6h0I>@WEBMY^F!y5JYWO|J;zcZg zMydp)&^wt(WB0niA^*142>JEF=veI!U(Tg6BK2akf=@5@c{Txc!W_4Zu6ubGxk?F{ zp~=*s=_Uq<#)7By_(T5b0NU7EdGFZQ1|uMULJ^jEiCYq`%5UG;r!`tI#mO{RGYC*j zJ+B(a>WrPkNQSh3N*PL|;h$2rNTm*!h}`nHVqeJ&>afIpg-D6?b$yXT#8NeX0aQ?T zW`SWAfyNA*T1(crDx2k+HX*RoAxq9K;|6q;7Byoxj?ObgwfyHsP^&h#I7h<3Xa!DY&>);W2Cqs!#N(mrI_i#=oa zQfh_E3huuH9;V-NvG2xbyzBM3D}^z8F}r*2?QX z+|uz$*YiFJ{^O4^qt_bDW)Rn4z=!wkl4NEzDJ;h$KyWQ|xa3X4cz<~8$qNoLZY_KF z#R+;|%uRys1@TOs1Q1srpUF{}CBO%$cYFL7J^qzJxl0W`!B_vsWqE_jeO>Qzi@wc& zn9SpJ9l@`BeYRkHKwY^4&@+Ge*P-gzK1mY2elW~w#PoSH?c$OO$)cIgRz&BK(2O~mPRg- zV*u(nAQACW)x5)N^~+&xrctm{!=IEIHIx%OZqGXV9zRFQc#YXp|3h0*Ad z$z8?P$b-+(Xj-|J?{uEwO8>VFw!%z>f;Hdj)9T}^|FKA@lMC9nWEn6y)^7 zd>_APb1owF&fp#*%keH#0|Po9IlVRaTmHiEx;Al8;&I z{pppKDQ0*Ehoe^$k}0RD9!Mh!X1>CS^WM(aj&Vwp^Az7$jm@Sgt?4g3a~R2m+YYAU zsETVklI2FpEF%BlW&RS(VFj$DIun_)YG<7JM}Gq~doopZOkUu6C+M0b_cKq7otx6Q zZPs|&Gq`@`RqxtwbTaHb>3Pm>*!BfwEIsg;y9;@*GJ0e#n7{^J2d*E3teUK*C)_m& zI*XSvlJ`Z+f>`D(U4(3_s*ojkkR509U2!n_i&SG!6y#`nSh+J}ulW97;*v2L$dPa*L2yL{Fg2AhD{` zc4YCQ?g2ge-N$7*r&fs8O@ty*q6|?4%P0?06lBt4s<&P)C`@+>8I)~oDv{D1b;coo zafw3wy7`#IREoFPjiL~#x!tlk_RH}a=yLXRaj}8T9>pogOh`uGn)Fd*c>&?u!O5{P z`v0QdaLoUp-rfHP^&0*U>Yb_5FLXl8Wai}LKh%=`$9d=G(qzthw3ECOIH1dQiyXKB z4-})W2xTg=3C4Q~2IN~+)x_r?T^s{Sv<`z`bT zt9XO|qj)v{DPH%!m*dP@DH^87aOeq!6*Cz;`}Gwr9}n+p>Ug90?BFABgB|&@LEQj;0(7f zab5S|s+1#>3Op!Bf(xo373Ed^R_Bt7V<)B2Zv4lzkfv;OXFUwhMGay*yCz^<|DhS# z6V3K++J|>*xH&7UAUi(KX#{95`qEnb31yECq#ufg6qr zxwvFu1l^vyos-o`A}k$~<3Z%FQUPuLdrq12Y&B`UQge8oYC=G^$!bAcN-moGi99iA zd6iWtoqd{9c8K9qY?mgR8>R@?UksKvh8(qW_7b=cr814g9|7(ocg? zXhC*g`hOIcMWnVOTxgO{UM=ZwcR!|#WSHLZh=1!jo~R9y3X}@cm+cJlfNF{}y<4HT zA&m;hy$F1%E1d`8QC2IDox>n0e%JiD;ZJE1+21|m{5vT-y%}sjwQmsFf*>aSY}+;6 zlE5(c`&*5iY(2^d>+CBfy@(b(TO9C>?{a)g5zmndf4f*5jyenwT{R8{c)IC%tSA{z zkyZxlCSyHRA!fV`Vj$I;;aA|^A$>q{5Z0gkb94j(D|JNQ!t%J(OTjg~6g+AvmG}^c zQC6apBX zK{nYVI0-Iq;V+6+u(XIKay0SxqZQ|B!Ib{T*>*e!{I9dk(Kj*RJ1`r?9q{7Qz1}k| z$4YZrerjw$Xx*u2@MZ-33ZXIHD4#~EwakzE{t4CIMd;XmU2{m_wRi$$I=(l?^1rf{ zU*Laa?R7S`DcgU^+A&Qy`<>!oqguxe^RM;YY2C7c{AP)br}r4~{eTa!I}iuVg*C zO#%*KJK(K%Rxq%b--rOKkpdkMTa@e%@ literal 0 HcmV?d00001 diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 98a399c8b..4f57dd642 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.4.0 + tag: v1.5.0 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. @@ -28,7 +28,7 @@ configGeneral: # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) # kubernetes_use_configmaps: false # Spilo docker image - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 # max number of instances in Postgres cluster. -1 = no limit min_instances: -1 # min number of instances in Postgres cluster. -1 = no limit @@ -67,7 +67,7 @@ configKubernetes: # keya: valuea # keyb: valueb - # list of annotations propagated from cluster manifest to statefulset and deployment + # list of annotations propagated from cluster manifest to statefulset and deployment # downscaler_annotations: # - deployment-time # - downscaler/* @@ -214,7 +214,7 @@ configAwsOrGcp: # configure K8s cron job managed by the operator configLogicalBackup: # image for pods of the logical backup job (example runs pg_dumpall) - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-58" # S3 Access Key ID logical_backup_s3_access_key_id: "" # S3 bucket to store backup results @@ -265,7 +265,7 @@ configConnectionPooler: # db user for pooler to use connection_pooler_user: "pooler" # docker image - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7" # max db connections the pooler should hold connection_pooler_max_db_connections: 60 # default pooling mode diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index cb8e29081..2a6a181f5 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.4.0 + tag: v1.5.0 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. @@ -28,7 +28,7 @@ configGeneral: # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) # kubernetes_use_configmaps: "false" # Spilo docker image - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 # max number of instances in Postgres cluster. -1 = no limit min_instances: "-1" # min number of instances in Postgres cluster. -1 = no limit @@ -203,7 +203,7 @@ configAwsOrGcp: # configure K8s cron job managed by the operator configLogicalBackup: # image for pods of the logical backup job (example runs pg_dumpall) - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-58" # S3 Access Key ID logical_backup_s3_access_key_id: "" # S3 bucket to store backup results @@ -257,7 +257,7 @@ configConnectionPooler: # db user for pooler to use connection_pooler_user: "pooler" # docker image - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-7" # max db connections the pooler should hold connection_pooler_max_db_connections: "60" # default pooling mode diff --git a/docs/index.md b/docs/index.md index 87b08deb2..d0b4e4940 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,9 +37,10 @@ in some overarching orchestration, like rolling updates to improve the user experience. Monitoring or tuning Postgres is not in scope of the operator in the current -state. Other tools like [ZMON](https://opensource.zalando.com/zmon/), -[Prometheus](https://prometheus.io/) or more Postgres specific options can be -used to complement it. +state. However, with globally configurable sidecars we provide enough +flexibility to complement it with other tools like [ZMON](https://opensource.zalando.com/zmon/), +[Prometheus](https://prometheus.io/) or more Postgres specific options. + ## Overview of involved entities @@ -70,12 +71,26 @@ Please, report any issues discovered to https://github.com/zalando/postgres-oper ## Talks -1. "Building your own PostgreSQL-as-a-Service on Kubernetes" talk by Alexander Kukushkin, KubeCon NA 2018: [video](https://www.youtube.com/watch?v=G8MnpkbhClc) | [slides](https://static.sched.com/hosted_files/kccna18/1d/Building%20your%20own%20PostgreSQL-as-a-Service%20on%20Kubernetes.pdf) +- "PostgreSQL on K8S at Zalando: Two years in production" talk by Alexander Kukushkin, FOSSDEM 2020: [video](https://fosdem.org/2020/schedule/event/postgresql_postgresql_on_k8s_at_zalando_two_years_in_production/) | [slides](https://fosdem.org/2020/schedule/event/postgresql_postgresql_on_k8s_at_zalando_two_years_in_production/attachments/slides/3883/export/events/attachments/postgresql_postgresql_on_k8s_at_zalando_two_years_in_production/slides/3883/PostgreSQL_on_K8s_at_Zalando_Two_years_in_production.pdf) -2. "PostgreSQL and Kubernetes: DBaaS without a vendor-lock" talk by Oleksii Kliukin, PostgreSQL Sessions 2018: [video](https://www.youtube.com/watch?v=q26U2rQcqMw) | [slides](https://speakerdeck.com/alexeyklyukin/postgresql-and-kubernetes-dbaas-without-a-vendor-lock) +- "Postgres as a Service at Zalando" talk by Jan Mußler, DevOpsDays Poznań 2019: [video](https://www.youtube.com/watch?v=FiWS5m72XI8) -3. "PostgreSQL High Availability on Kubernetes with Patroni" talk by Oleksii Kliukin, Atmosphere 2018: [video](https://www.youtube.com/watch?v=cFlwQOPPkeg) | [slides](https://speakerdeck.com/alexeyklyukin/postgresql-high-availability-on-kubernetes-with-patroni) +- "Building your own PostgreSQL-as-a-Service on Kubernetes" talk by Alexander Kukushkin, KubeCon NA 2018: [video](https://www.youtube.com/watch?v=G8MnpkbhClc) | [slides](https://static.sched.com/hosted_files/kccna18/1d/Building%20your%20own%20PostgreSQL-as-a-Service%20on%20Kubernetes.pdf) -4. "Blue elephant on-demand: Postgres + Kubernetes" talk by Oleksii Kliukin and Jan Mussler, FOSDEM 2018: [video](https://fosdem.org/2018/schedule/event/blue_elephant_on_demand_postgres_kubernetes/) | [slides (pdf)](https://www.postgresql.eu/events/fosdem2018/sessions/session/1735/slides/59/FOSDEM%202018_%20Blue_Elephant_On_Demand.pdf) +- "PostgreSQL and Kubernetes: DBaaS without a vendor-lock" talk by Oleksii Kliukin, PostgreSQL Sessions 2018: [video](https://www.youtube.com/watch?v=q26U2rQcqMw) | [slides](https://speakerdeck.com/alexeyklyukin/postgresql-and-kubernetes-dbaas-without-a-vendor-lock) -5. "Kube-Native Postgres" talk by Josh Berkus, KubeCon 2017: [video](https://www.youtube.com/watch?v=Zn1vd7sQ_bc) +- "PostgreSQL High Availability on Kubernetes with Patroni" talk by Oleksii Kliukin, Atmosphere 2018: [video](https://www.youtube.com/watch?v=cFlwQOPPkeg) | [slides](https://speakerdeck.com/alexeyklyukin/postgresql-high-availability-on-kubernetes-with-patroni) + +- "Blue elephant on-demand: Postgres + Kubernetes" talk by Oleksii Kliukin and Jan Mussler, FOSDEM 2018: [video](https://fosdem.org/2018/schedule/event/blue_elephant_on_demand_postgres_kubernetes/) | [slides (pdf)](https://www.postgresql.eu/events/fosdem2018/sessions/session/1735/slides/59/FOSDEM%202018_%20Blue_Elephant_On_Demand.pdf) + +- "Kube-Native Postgres" talk by Josh Berkus, KubeCon 2017: [video](https://www.youtube.com/watch?v=Zn1vd7sQ_bc) + +## Posts + +- "How to set up continuous backups and monitoring" by Pål Kristensen on [GitHub](https://github.com/zalando/postgres-operator/issues/858#issuecomment-608136253), Mar. 2020. + +- "Postgres on Kubernetes with the Zalando operator" by Vito Botta on [has_many :code](https://vitobotta.com/2020/02/05/postgres-kubernetes-zalando-operator/), Feb. 2020. + +- "Running PostgreSQL in Google Kubernetes Engine" by Kenneth Rørvik on [Repill Linpro](https://www.redpill-linpro.com/techblog/2019/09/28/postgres-in-kubernetes.html), Sep. 2019. + +- "Zalando Postgres Operator: One Year Later" by Sergey Dudoladov on [Open Source Zalando](https://opensource.zalando.com/blog/2018/11/postgres-operator/), Nov. 2018 diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index d436695e8..e626d6b26 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -7,7 +7,7 @@ metadata: # annotations: # "acid.zalan.do/controller": "second-operator" spec: - dockerImage: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 teamId: "acid" numberOfInstances: 2 users: # Application/Robot users diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 0a740e198..4314b41d3 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -29,7 +29,7 @@ data: # default_cpu_request: 100m # default_memory_limit: 500Mi # default_memory_request: 100Mi - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 # downscaler_annotations: "deployment-time,downscaler/*" # enable_admin_role_for_users: "true" # enable_crd_validation: "true" diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index 4b254822c..e7a604a2d 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -15,7 +15,7 @@ spec: serviceAccountName: postgres-operator containers: - name: postgres-operator - image: registry.opensource.zalan.do/acid/postgres-operator:v1.4.0 + image: registry.opensource.zalan.do/acid/postgres-operator:v1.5.0 imagePullPolicy: IfNotPresent resources: requests: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 9ae1b3b26..049e917f6 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -3,7 +3,7 @@ kind: OperatorConfiguration metadata: name: postgresql-operator-default-configuration configuration: - docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 # enable_crd_validation: true # enable_lazy_spilo_upgrade: false # enable_shm_volume: true @@ -92,7 +92,7 @@ configuration: # log_s3_bucket: "" # wal_s3_bucket: "" logical_backup: - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-58" # logical_backup_s3_access_key_id: "" logical_backup_s3_bucket: "my-bucket-url" # logical_backup_s3_endpoint: "" diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 389240c09..41d701fe2 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -37,7 +37,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps - result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115") + result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-12:1.6-p3") result.Workers = fromCRD.Workers result.MinInstances = fromCRD.MinInstances result.MaxInstances = fromCRD.MaxInstances diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index d8c92ba3e..348452193 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -112,7 +112,7 @@ type Config struct { WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115"` + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p3"` // deprecated in favour of SidecarContainers SidecarImages map[string]string `name:"sidecar_docker_images"` SidecarContainers []v1.Container `name:"sidecars"` diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index ccaecd312..8da564322 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: postgres-operator-ui containers: - name: "service" - image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.4.0 + image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.5.0 ports: - containerPort: 8081 protocol: "TCP" From 62bde6faa24b75973781d012983eafdb9a6dd863 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 8 May 2020 13:02:27 +0200 Subject: [PATCH 099/104] fix env var in UI chart (#967) * fix env var in UI chart * re-include 1.4.0 in helm chart index * fix import in UI main.py and updated images --- charts/postgres-operator-ui/index.yaml | 8 ++++---- .../postgres-operator-ui-1.5.0.tgz | Bin 3786 -> 3799 bytes .../templates/clusterrole.yaml | 1 + .../templates/deployment.yaml | 6 +++--- charts/postgres-operator-ui/values.yaml | 2 +- ui/manifests/deployment.yaml | 2 +- ui/operator_ui/main.py | 1 + 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/charts/postgres-operator-ui/index.yaml b/charts/postgres-operator-ui/index.yaml index 114e6a4d7..2da53497d 100644 --- a/charts/postgres-operator-ui/index.yaml +++ b/charts/postgres-operator-ui/index.yaml @@ -3,10 +3,10 @@ entries: postgres-operator-ui: - apiVersion: v1 appVersion: 1.5.0 - created: "2020-05-04T16:36:04.770110276+02:00" + created: "2020-05-08T12:07:31.651762139+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience - digest: ff373185f9d125f918935b226eaed0a245cc4dd561f884424d92f094b279afe9 + digest: d7a36de8a3716f7b7954e2e51ed07863ea764dbb129a2fd3ac6a453f9e98a115 home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -26,7 +26,7 @@ entries: version: 1.5.0 - apiVersion: v1 appVersion: 1.4.0 - created: "2020-05-04T16:36:04.769604808+02:00" + created: "2020-05-08T12:07:31.651223951+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience digest: 00e0eff7056d56467cd5c975657fbb76c8d01accd25a4b7aca81bc42aeac961d @@ -49,4 +49,4 @@ entries: urls: - postgres-operator-ui-1.4.0.tgz version: 1.4.0 -generated: "2020-05-04T16:36:04.768922456+02:00" +generated: "2020-05-08T12:07:31.650495247+02:00" diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz index 6d64ee3b55582f3420da6c1d78ad883affe7a09e..4443e15bc739899f370f7c71a2d45e78172c09a1 100644 GIT binary patch delta 3770 zcmV;r4n^_G9oHR@L4WXk=CA0bpRJn74k?n>n^~3T*5i0nu1y@5?PP0jYsv+YEeT@^ zU;t2#6dNz5QVS2Y>J$rS~l*lZuERysz%7 zJ-I(fAt`-_LQ>8`nEMt<)9R=1_FT`3QAUEM%Dg%&xCP!6+yZY;AQgO0W0XK(B1k%= z5lMhDgqs zm0_ndp=z3q+<%DAIv+_wSj;;sylY+K?F|>P3@r!BTFZf|-hqr-4nzshVuuk$=cplx z5}fG@nbL$?GeVgnl%X()a7JhnY9vhZOhowiVgPQ8762o9nE#!>vhrW+c9oSkHYgu7 zsd;lj9(VoM?*BPSGL(-d0JiM^eSgqv@BdzZxBs7{?0>-v9Fr_jkT1H~uIre>wf5k0 zN+nRe(f@t;`js>0Vn&q07{)X~4e$b^gb0K=5tNJ)l%P0_5K`h=MuE=A1TFJ7v;YK7s8r%Mte)KE)^yD_a${}) zQ!c6E8h?~TG+y0WdRGD{GSRY6Z!RaMLfu*bX_h2sJfYET2=>YNl&dp=5}C3sON>aQ z2eJnjDMoZmQNqm>)f5Gs|8jT)0*kE}zL))a^Y5{0n^#Zw4MWN2B1@H>nmGyM%{ z30ySbwjl@g1_s6I2q=jrQb2^{AK6`V1d3bsm!oQ|8!Aeg+qzxcLiMJ=^4}m~q zWPdTH`VUh-gDSRzromF*jZobnGSkD`6N99}dw6!@!U>ZolxbC}zWOTFfnkaQ|6)P) z@-U@uMG}IoHAvW5W$AF}4L!To;wZ_aLUBrF_==2Bw~g{wj_%bq1TEI#9hG#HU3M6| zk_IJRsDx7<;~CcrU|K>%m}Yxf?2$SjvJ!MmF^TKAn>8A(;r0^8&%g!HF;E3lqU&;D zs8<*n5D;23&+`7M)Vn6f&bTsYL)Z^`fxki#%JkYu*K9Au7~>FZWLh%D`98I+0)IwR ztEw4m>RSMWF|LR$kp4P_mFNVPGmT3&@Qk6o*Pb0Ees zWuM>0&JxtPo^Bp+)KUZ7cX{wZL%x}#5Hv>kq>UvOL4V&o(%D5e9@7tCJ4yJc1ubw}vn;PA@d9wobLB`IBp%j6viQGDi||6K9fzi;??S3T|kUXd74C znLxC=tp{I>_I`9OU9u&4=(r+nSy_>{gxLsiFZw5PiRu5DxrEQ}zkkHFaaLr>u{5;G za;%NF4<2ADC{tsw|0g3K#q#OWKSSoYoe%BJz)vMM)NK zXA1YHY7v@f&6>dHCQ)NmEe(p`q-ImYt_APsP>FUp?>j%!#NN0-b^ghpi$FDe0)c6w zjSc&6HrQA8gI4Za3xD8mT(;={B2*xFf?pmE-lG5gUcVLp4ZMEP-|7FSD0g>t%^*f3 za*N>KVqw(}hfEd8)RYVQ(ZnR&>!C5==e~7KSsX$U<~ipHS~FB6CW?gB35o9Mv*Kx; zfYbKA1-T$8A{FaB2!sFLE`&ie#boBnX~zhj`?N14RE1!5B!4r)$OL0&bZhWcTBV4* zZf;1KRD?y?Y<5}*W058Kxe-a}OTn|$kis!h?PkQg7EoQDfW0>%nK_D4QIW5(d5kf^ z;&Fn?{7uMST|T1fNihqHEwmcg80TDz8AKFDL#jBl`Ls0S24wyCj54W zOvxOxA_~V@B7adm2>ZHGR$lPuFx}l5rSpC1ocqiF5oIxDlP^#M+vNYg*NXoHf#36X z^8YEy?_qC=1P& z=S`c!ir=R={lk?l@_(r`zakrCi~R2e!G7ER@9yvH|9>YbbrG-arlmB0Q7wH#ivP)F za80^x8DcHkV&rCuQJ%hHvRz~dz6BsL!ANnD({rZn;#V~SKO%|0WU6Hk#pr5Wpi9yi z=%WtxO9q=4g=7|jtvw?f_TY>NGDC%ee#HMWK;=4^=M@d~b!yAxqwrdt< zYgaXpXMZ<^VCzX0R+xF^mf20PSajA@mbp!^ScEIpicDa!Sn1}hmesV*K3ehrb*@w4-)T(#9^&k1yW5 zJ%2wszPNgKa&hv@tK&^nYWLJmX=?3;S{Ijx=P!>h&0H?d4v#kV(C(m>7qxK)m&b>% zFE-%(_q%ny$2^i(31!!IXqS1*D}vG0 zn_5j#18ZT8tYZvF)vGcL=vq!#C_T^qOK-H;SIU1_hWLh);1_K`onvwnDjotnM83fz-83$ zsyLjHcDR4OKDx>Z>;e))f~9}%~ics7H%`KE<1PbNy`@hugos|s!sv7<$r(d z?|UoxA3=BL|2;)%r3RGiU8JHe&-1l6b6T9~-}{t#*<8*elYbjNZy!9nCabn~`Y&-d zM{&ewDG>i@0nGs!C|Ft&eecj@7%*5dnywA9**@Ct4 zs4|;d_o1Iwhi2T0Y65wJclEQ@i>YZ^w|`FIj+r^P z?A{THu93xU|E<{TryF+28qnOY)S#v4s6jSpvz}((TCV<4$`<`!3hCcd|NC99{r&G? z;QKrM{}iS5{cqz$XBYQ<#JF!QjOG*(t6EH^>{>m^HkXlYxlpa z=|m@t3w(?|w(b93&|kU#A=rKY^CV>t&WKVdn7OJUUvs#bB7;#zlYf}938W;tCKHs^ z4G=O*jWq*uib(>Kgpcy0Ysw}+gTTaGtYW;x+IzxcYY!MsjDHV5rUJ+G1IBr1;4eS9 z@P;L~z?o^OhXg4KNGL%PFzB`8zrFpz&i{Lo(o*cm zl%RR2X&q;j`PGd!g?ek<8>QbCcawx*2VUSgUeED{m%bkcL4VjCx(D81c+l_n{l9rZ z=z00^(C=~Gl)-ziE2j)3ouE`947y%$FdPiJ-2*=w9r)v5)Ef_nez&*p6EckYBY#BV zcswMrf6yhP;9%q*_|dQ*ce`FR*v}1)hu>kPn=S9LD*eUcF01@!1J>MWl|MA@wW>mO z=5DL}SAM@0n156w+RRhgde4>d6B~D3E#GXoSqGqzLh-uzJfFOwu`hNW9VAiUcy#BkS*A7F~1~ZrRI_rKXoZVf|*_vUiYnV2& kYk%N-{y`;MzqOv(m0j7D@2~uC00030|6@Qfpa50?0FnrN&j0`b delta 3757 zcmV;e4pQ;g9m*Y$L4R;Q^H=oJ&sNQ3LyFX!ZC2&E^*G*?YZFIhJK5UXnsPy8OTw4} z7yy**IJ)0{1@J{AB}H=LaWXf;2b&@q4WQBJH#CuOsm21O!xI!lap9!2bvz+Lxp!n5 zKYNVgd7jtrcFlj!^P2y?e!u<9@AP|Juif>!ooAlk>-gR?@P8hq_YEbLiil_4H}}wuIZ)O+kW$NmFy?9GFrw%TH6&4j zGhHDQ8gpw(C{u(o6b2Da35^4dgh`%?5dWSJz>UxXU_=k|zq40X_G{Iyva-eo<%1?Q zt1rmouK&vYKO=F9^3epqhW+31`}@uP-|P9i{r?nY4}V_ah@`QCY|+hjUBwixwFj3I zDuL>a{_mr=Z=4YqQ=$|`FrqPPfL9pCL?FzFpkx@M1jS*9kPs;`(n5j59Zv-)oF*|* zC|%1En8Y+B$2?^!0N=9qw6qw}aVii}3Uo@wXqmr(1t4%tr4n~R`Q$FPrfas58*u}e za7h){pnoK!(el>PyAnW=v6g*ia{(l291nO*!@B_N)6p4M1A!8mvMoytNvMag2Nwy3 zbVO0Y%>>m11)TqKbPNLXO%8Ix(>T&|4+Rp1kp=lv07|52S-IFd3PUsTHE0Q3)ZjKF z)_5)_Xyg*`5vU0^aVV~Zt!qF}N5HrOi3$jda(`5Mku|axKr%d3!WfyR8b**x%Ekuo zD2-!NU#w>#59^Dll?0C#)UnDZJ?)rIsWMLhVUh;W_PnY2HN~k}y|(AQrDjDxrYLK$ zeiPO=u+8pI&OS&B*-U11K7BiQcYb+vb{W9GCN#!ULqMoaA47@vr^OF}KqF)kCi)Lk zFMmMg8$r`xq3MRGZV;L2;r*#WQsO-toVswzBno9(6-uwVN>yN(puoRaP`*4$=z9?d zU~AnGw3eAUIPeZUyVBw~PNhO|MyB|N3{kfYvsaGp)iwmp*Wm+|bQog*_CH80O`RfQ zj7qnb2?=#olt~JEiMsh@3YOM1R(FYRl7Elkbf7UNT&Mu-1Mk4M3*OW7*02Q)3c64U zXFS3I*9%};LP(fqdztT%Dj%`}bVM0>=yB0_X^+f(g-enHbb7j0^|} zt&^u&|5WN-lVN9E8MFa(+P${FL=npL+DO-IFT@C=0BmGhCPvvlwXGaRQ>&~QX@BZl z0E98Fh&H-p0QYml;d^kM4pA`FVw3V#(MK2)DFa|*%5GNzs#rQC)OLxT^$alKuoNi^EDMZ$2~`Y^(IiGcyu7~$_Z(v+65Ud3z`Sf2nNP|5p=?6BN=_$=ADr8}@&@>-o+7-+$ll?)Lvv zl>7VEbC}U-U_1cr$7yTq1F{(Ej7UXkOdz!i8JGoB#D`ZYk&OHnFz1TVK=;>tKnvVUY)YFcG6*4o<# z4=@pwsS((JmCj$K-N2Uh`w*sE_uOE{!n+~4BmR&jt;Y+_DvMhZ`6tT4IE^+lh5K{4 z2z9h(P2fwDsIaP*26=E&v#DX%ocA-RSi7Cu&d)Tl*Dg?%f3oL1Pz|3!U=nL%!~UBM z_LcphmHXZT_&b*k`hPzU7ziHY*N1~Q=zqW0?bY>v+w1$g`0taH`}?Y95JM8Wc?fVm zx2lIjri!F$!Ug?gA`|ZQfid7`zI9Dm6hI#AIp;B2Q&c1(iUj2eiSFr(;%SwD)Aqgv znIOsI6{|f6gMVum!XTPpJay%yWdzSQ?Q;oLB3K^DlrS>J$bT8$8GNNy$>Xo<8&W0} zVIkI=o#nz}+zA=cm!j?0|moP4qB}wE8Oo}KRrLjcyAZ)8f zReHT&!gPOclz+^Rtz)*A|3k_m%En)z1~$q6j@O9)wA+5Cx0C-*QGO@+|4YU!J2mjA zYJkUhjw3z)LPmXIT`YiQwfya?Ak*P*7%HQVGW_+!#Yb6awm5ZK7gqdZh}R#!Y>@vE zCNaO8BKwwfunqFR*KRM_|K0XZ{y#~nig;TaPOX`5=YR6tepRl%U6bU0ayeX)a2tkL ziMSZqnP8Zuu$YV&8GvsANQ^O5Tx9f|YTNitg}{$U;;)%%(L+AE3K!@S*9Q8iL;ae; zWHYHki`WWyc|h#*r`DCj5rF9TGpgLz)kLf@vkOlAsSWps5PXPr%oC-O*QkOfGNL8{*sTeYSWWC^ z?dma5MinLTD~!qA1%{kOGJswUMA0eoRF$=UZGV^*w{KDI?;TM5-((suEax)w>oYKF z(uBZo*QtnX)}&$16Gwx=)%)`|%}JSDlts!<%tI2JWQ`()W(BHugOl^4%XjBjM}yNZ z;H)e?vYB-2X-4&Nufl%(=KaOx$@$gU(c6=&H%GsmJhY=`cGB7|&rdGiy+1!bxw!gp zdVg{H%bSxeJ8I@Ot?l6Q==}A`rJ2da;OKZ;{Ehsm_2^HI-d?Q1`Ppf_2l&s@8^}f3 zQpK+u*697|)$zNtf1JMFzW4i8y+=HhS21PRc3>Bo%S&R>)mvIkPy=gGjjUx%NY%1m ztVGJ>RhB?{C6%CTY-<7N&z*a3?%ucOYkykhnd9UP>uL_&{`#ukG7C4ESeKnc_k?AG|5xM~e$%G_oAN(89j}@HvA;Y2d79El4Sy)s zJ5NYmoabv#<}5$c-};n!(Ok|#lZ6|+Y#uzjCbzb7`Y&-hLvhTf2@weP|9|Pbe%JAA zaF0nshBT&%qICb64l(y!-PcqVhrv!GY{^jFaB)r9coD7~q9T4hC7{{_Zxu2lHS4fh zuqktmPcu5Lj&ClAwToKnHkx`jRDVsDxU7=poIzI8o=s;}OS@uOcNd8BWyxH+HQcP? z9KXueL4ygx#<(7`$crv5|4JM4zH09|X5#P}KBin%EWyh7vKQ5iY29S;duc+ow)p|z zmquP2RFTuI`_NCzLo+T#Ie{$AyZl+{#njYoTPJtL%$yrm?~sJo$fBn2mVfW{vo$+p z1!(3|YS2P-R3K}#SxvL=Em!|2WrO}Ng!J#J|NXAl{Qh^pzu({K|EDO8?|*A2I=h(f zBgT9yAv7nCR#hSjHDjohbXkaDDZG|zIl?0N2|q63FrQP|#n)5zQmUU@eiK!8RJ>U^ zre*LYy|xJST^#*dYJCezzh2 z+kU&h`~K%iN#6Qrpoii7`MuE!(lo2SZdSh@dQO(!~LT;OB$v1$MJ+Wn>b zAN<|-KTlHjU_g{Y!OT?+*_y-61Q`rd8b_3kAtB*48KW$3fRJfotbZAh6O3aR$9$L_ zTT?cE0Rm%lv5N5$EAI)5tUX{jHvT>Qln5NrTa2>Mz+ZlL;T?u8wFk$1I^_&L9A7|0g|yr;RW0*38@}ZZKZ%z4TfCTzTl!D&Q_fgxfk^wT zX=1*8lGbxq-Xzv@cYjE(t>>%Pd%wGM|4XO8+y75e8j2m65;O}nt>TO#zq-~YS8t7b zqx9SShLQm6w%7I?uj_QYOaCC~dO_#V^*yiGIrQ6yfAiXb=YM5~L%+v$QwHx=S56s7 zI!37iIK%_e3)|r^>V${Ac0>-m(Cds2{QhCL(;InhG8|#QPkNmZX^(nd)bI87!`^-; zI_!~QL^6Zp;dfZ+W{Z2Q3V$)b%PRX>gEeG`t;$fHx!WrHmECUzCe4V}b5u6o zb7lO*+Fe(RH-Bqx)&Z!cPdxD6E8Qd&@e&cLoj8WsTs8h=fM!h|+=5xVTQRG5czD?9 zc6&c2uX-MOI2!GHy?)f|cl-Nc)C&*yd;Q-2XgJ*OhX;}8lQ4|hJ<=WSlaL(v!?4>X zNW#OuAN@IawQMG@|1RmZ!%#KBCB06&KL}%YS2MQe*Cz5VruE#~?sof!{eoS;x0>0N XUD=f%u>5ZT00960EC=W;09F717N1$~ diff --git a/charts/postgres-operator-ui/templates/clusterrole.yaml b/charts/postgres-operator-ui/templates/clusterrole.yaml index 4f76400ec..57a1a6365 100644 --- a/charts/postgres-operator-ui/templates/clusterrole.yaml +++ b/charts/postgres-operator-ui/templates/clusterrole.yaml @@ -38,6 +38,7 @@ rules: - apiGroups: - apps resources: + - deployments - statefulsets verbs: - get diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml index 6247ec933..00610c799 100644 --- a/charts/postgres-operator-ui/templates/deployment.yaml +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -1,5 +1,5 @@ -apiVersion: "apps/v1" -kind: "Deployment" +apiVersion: apps/v1 +kind: Deployment metadata: labels: app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} @@ -44,7 +44,7 @@ spec: - name: "OPERATOR_CLUSTER_NAME_LABEL" value: {{ .Values.envs.operatorClusterNameLabel }} - name: "RESOURCES_VISIBLE" - value: {{ .Values.envs.resourcesVisible }} + value: "{{ .Values.envs.resourcesVisible }}" - name: "TARGET_NAMESPACE" value: {{ .Values.envs.targetNamespace }} - name: "TEAMS" diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml index 90e9daa66..2fdb8f894 100644 --- a/charts/postgres-operator-ui/values.yaml +++ b/charts/postgres-operator-ui/values.yaml @@ -8,7 +8,7 @@ replicaCount: 1 image: registry: registry.opensource.zalan.do repository: acid/postgres-operator-ui - tag: v1.5.0 + tag: v1.5.0-dirty pullPolicy: "IfNotPresent" rbac: diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index 8da564322..4161b4fc1 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: postgres-operator-ui containers: - name: "service" - image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.5.0 + image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.5.0-dirty ports: - containerPort: 8081 protocol: "TCP" diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index a294ae081..dc2450b9f 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -7,6 +7,7 @@ gevent.monkey.patch_all() import requests import tokens +import sys from backoff import expo, on_exception from click import ParamType, command, echo, option From a5bb8d913c149e6b16d43e910f52bc9e52ffdba1 Mon Sep 17 00:00:00 2001 From: Damiano Albani Date: Tue, 12 May 2020 09:20:09 +0200 Subject: [PATCH 100/104] Fix typo (#965) --- pkg/util/retryutil/retry_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/retryutil/retry_util.go b/pkg/util/retryutil/retry_util.go index f8b61fc39..868ba6e98 100644 --- a/pkg/util/retryutil/retry_util.go +++ b/pkg/util/retryutil/retry_util.go @@ -27,7 +27,7 @@ func (t *Ticker) Tick() { <-t.ticker.C } func Retry(interval time.Duration, timeout time.Duration, f func() (bool, error)) error { //TODO: make the retry exponential if timeout < interval { - return fmt.Errorf("timout(%s) should be greater than interval(%v)", timeout, interval) + return fmt.Errorf("timeout(%s) should be greater than interval(%v)", timeout, interval) } tick := &Ticker{time.NewTicker(interval)} return RetryWorker(interval, timeout, tick, f) From 852f29274ab0fa9ce79354798393fe7ab9176a05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ask=20Bj=C3=B8rn=20Hansen?= Date: Tue, 12 May 2020 01:05:42 -0700 Subject: [PATCH 101/104] Fix typo in error message (#969) --- pkg/cluster/pod.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index a734e4835..44b2222e0 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -331,7 +331,7 @@ func (c *Cluster) recreatePods() error { c.logger.Infof("there are %d pods in the cluster to recreate", len(pods.Items)) if !c.isSafeToRecreatePods(pods) { - return fmt.Errorf("postpone pod recreation until next Sync: recreation is unsafe because pods are being initilalized") + return fmt.Errorf("postpone pod recreation until next Sync: recreation is unsafe because pods are being initialized") } var ( From 8ff7658ed3caf1d93605a8310bda309d1e622753 Mon Sep 17 00:00:00 2001 From: Christian Rohmann Date: Wed, 13 May 2020 14:55:54 +0200 Subject: [PATCH 102/104] Fix pooler delete (#960) deleteConnectionPooler function incorrectly checks that the delete api response is ResourceNotFound. Looks like the only consequence is a confusing log message, but obviously it's wrong. Remove negation, since having ResourceNotFound as error is the good case. Co-authored-by: Christian Rohmann --- pkg/cluster/cluster.go | 2 +- pkg/cluster/resources.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 31b8fa155..de6578f69 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -1345,7 +1345,7 @@ func (c *Cluster) deleteClusterObject( objType, namespacedName) if err = del(name); err != nil { - return fmt.Errorf("could not Patroni delete cluster object %q with name %q: %v", + return fmt.Errorf("could not delete Patroni cluster object %q with name %q: %v", objType, namespacedName, err) } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 3528c46f4..5c35058c2 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -191,7 +191,7 @@ func (c *Cluster) deleteConnectionPooler() (err error) { Deployments(c.Namespace). Delete(context.TODO(), deploymentName, options) - if !k8sutil.ResourceNotFound(err) { + if k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pooler deployment was already deleted") } else if err != nil { return fmt.Errorf("could not delete deployment: %v", err) @@ -213,7 +213,7 @@ func (c *Cluster) deleteConnectionPooler() (err error) { Services(c.Namespace). Delete(context.TODO(), serviceName, options) - if !k8sutil.ResourceNotFound(err) { + if k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pooler service was already deleted") } else if err != nil { return fmt.Errorf("could not delete service: %v", err) From 3a49b485e5c407d0dfa1d3f6cb97421db54b8cef Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 14 May 2020 11:34:02 +0200 Subject: [PATCH 103/104] delete secrets of system users too (#974) --- pkg/cluster/cluster.go | 9 --------- pkg/cluster/cluster_test.go | 31 ------------------------------- 2 files changed, 40 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index de6578f69..9538b4ab1 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -822,10 +822,6 @@ func (c *Cluster) Delete() { } for _, obj := range c.Secrets { - if doDelete, user := c.shouldDeleteSecret(obj); !doDelete { - c.logger.Warningf("not removing secret %q for the system user %q", obj.GetName(), user) - continue - } if err := c.deleteSecret(obj); err != nil { c.logger.Warningf("could not delete secret: %v", err) } @@ -1300,11 +1296,6 @@ func (c *Cluster) Unlock() { c.mu.Unlock() } -func (c *Cluster) shouldDeleteSecret(secret *v1.Secret) (delete bool, userName string) { - secretUser := string(secret.Data["username"]) - return (secretUser != c.OpConfig.ReplicationUsername && secretUser != c.OpConfig.SuperUsername), secretUser -} - type simpleActionWithResult func() error type clusterObjectGet func(name string) (spec.NamespacedName, error) diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 4562d525e..1f6510e65 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -12,7 +12,6 @@ import ( "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/teams" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" ) @@ -334,36 +333,6 @@ func TestInitHumanUsersWithSuperuserTeams(t *testing.T) { } } -func TestShouldDeleteSecret(t *testing.T) { - testName := "TestShouldDeleteSecret" - - tests := []struct { - secret *v1.Secret - outcome bool - }{ - { - secret: &v1.Secret{Data: map[string][]byte{"username": []byte("foobar")}}, - outcome: true, - }, - { - secret: &v1.Secret{Data: map[string][]byte{"username": []byte(superUserName)}}, - - outcome: false, - }, - { - secret: &v1.Secret{Data: map[string][]byte{"username": []byte(replicationUserName)}}, - outcome: false, - }, - } - - for _, tt := range tests { - if outcome, username := cl.shouldDeleteSecret(tt.secret); outcome != tt.outcome { - t.Errorf("%s expects the check for deletion of the username %q secret to return %t, got %t", - testName, username, tt.outcome, outcome) - } - } -} - func TestPodAnnotations(t *testing.T) { testName := "TestPodAnnotations" tests := []struct { From 0fa61a6ab374dfcce19fd0b1f49a86d9c2a495c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20P=C3=B8hner=20Henriksen?= Date: Mon, 25 May 2020 16:32:33 +0200 Subject: [PATCH 104/104] Changed order of sidecar env vars (#980) * Changed order of sidecar env vars * Cleaned up test code --- pkg/cluster/k8sres.go | 2 +- pkg/cluster/k8sres_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 534ae7b8e..e1093b72b 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -531,7 +531,7 @@ func patchSidecarContainers(in []v1.Container, volumeMounts []v1.VolumeMount, su }, }, } - mergedEnv := append(container.Env, env...) + mergedEnv := append(env, container.Env...) container.Env = deduplicateEnvVars(mergedEnv, container.Name, logger) result = append(result, container) } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index d09a2c0aa..78f088389 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1394,7 +1394,7 @@ func TestSidecars(t *testing.T) { // replaced sidecar // the order in env is important - scalyrEnv := append([]v1.EnvVar{v1.EnvVar{Name: "SCALYR_API_KEY", Value: "abc"}, v1.EnvVar{Name: "SCALYR_SERVER_HOST", Value: ""}}, env...) + scalyrEnv := append(env, v1.EnvVar{Name: "SCALYR_API_KEY", Value: "abc"}, v1.EnvVar{Name: "SCALYR_SERVER_HOST", Value: ""}) assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ Name: "scalyr-sidecar", Image: "scalyr-image",