From c65a9baedf20d225db1dac7758741bf569c40578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Inge=20Bols=C3=B8?= Date: Mon, 17 Jun 2019 14:03:33 +0200 Subject: [PATCH 1/5] specify ReadOnlyRootFilesystem: false for pod security policies (#560) Explicitly specify ReadOnlyRootFilesystem: false so kubernetes can pick a less restrictive policy the operator has access to. --- pkg/cluster/k8sres.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 585cf2bed..0b2cb0c34 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -359,6 +359,8 @@ func generateContainer( volumeMounts []v1.VolumeMount, privilegedMode bool, ) *v1.Container { + falseBool := false + return &v1.Container{ Name: name, Image: *dockerImage, @@ -382,6 +384,7 @@ func generateContainer( Env: envVars, SecurityContext: &v1.SecurityContext{ Privileged: &privilegedMode, + ReadOnlyRootFilesystem: &falseBool, }, } } From 3553144cdad5aa499b72a44459a34eff2b6090a7 Mon Sep 17 00:00:00 2001 From: Maxim Ivanov Date: Mon, 17 Jun 2019 14:49:01 +0100 Subject: [PATCH 2/5] Support subPath in generated container (#452) * mounted volumes now provide a subPath --- docs/reference/cluster_manifest.md | 3 +++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 1 + pkg/apis/acid.zalan.do/v1/util_test.go | 6 ++++-- pkg/cluster/k8sres.go | 5 +++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index d2bbcbe88..0df86260c 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -282,6 +282,9 @@ properties of the persistent storage that stores postgres data. documentation](https://kubernetes.io/docs/concepts/storage/storage-classes/) for the details on storage classes. Optional. +* **subPath** + Subpath to use when mounting volume into Spilo container + ### Sidecar definitions Those parameters are defined under the `sidecars` key. They consist of a list diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 33c3a159b..165139be3 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -82,6 +82,7 @@ type MaintenanceWindow struct { type Volume struct { Size string `json:"size"` StorageClass string `json:"storageClass"` + SubPath string `json:"subPath,omitempty"` } // PostgresqlParam describes PostgreSQL version and pairs of configuration parameter name - values. diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index 02bdcca1c..1f8825090 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -181,7 +181,8 @@ var unmarshalCluster = []struct { "teamId": "ACID", "volume": { "size": "5Gi", - "storageClass": "SSD" + "storageClass": "SSD", + "subPath": "subdir" }, "numberOfInstances": 2, "users": { @@ -263,6 +264,7 @@ var unmarshalCluster = []struct { Volume: Volume{ Size: "5Gi", StorageClass: "SSD", + SubPath: "subdir", }, Patroni: Patroni{ InitDB: map[string]string{ @@ -311,7 +313,7 @@ var unmarshalCluster = []struct { }, Error: "", }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"9.6","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"volume":{"size":"5Gi","storageClass":"SSD"},"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"ACID","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"9.6","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"ACID","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, // example with teamId set in input { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 0b2cb0c34..808105a6d 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -342,11 +342,12 @@ func isBootstrapOnlyParameter(param string) bool { param == "track_commit_timestamp" } -func generateVolumeMounts() []v1.VolumeMount { +func generateVolumeMounts(volume acidv1.Volume) []v1.VolumeMount { return []v1.VolumeMount{ { Name: constants.DataVolumeName, MountPath: constants.PostgresDataMount, //TODO: fetch from manifest + SubPath: volume.SubPath, }, } } @@ -800,7 +801,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State // pickup the docker image for the spilo container effectiveDockerImage := util.Coalesce(spec.DockerImage, c.OpConfig.DockerImage) - volumeMounts := generateVolumeMounts() + volumeMounts := generateVolumeMounts(spec.Volume) // generate the spilo container c.logger.Debugf("Generating Spilo container, environment variables: %v", spiloEnvVars) From 69183945627e0dd582e50b07deb2ff54360943a0 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 18 Jun 2019 10:48:21 +0200 Subject: [PATCH 3/5] Add PDB configuration toggle (#583) * Don't create an impossible disruption budget for smaller clusters. * sync PDB also on update --- charts/postgres-operator/values.yaml | 3 +- docs/administrator.md | 16 +++ docs/reference/operator_parameters.md | 7 ++ ...gresql-operator-default-configuration.yaml | 1 + .../v1/operator_configuration_type.go | 1 + .../acid.zalan.do/v1/zz_generated.deepcopy.go | 5 + pkg/cluster/cluster.go | 9 ++ pkg/cluster/k8sres.go | 24 ++-- pkg/cluster/k8sres_test.go | 119 +++++++++++++++++- pkg/cluster/sync.go | 28 ++--- pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 29 ++--- pkg/util/k8sutil/k8sutil.go | 2 +- 13 files changed, 205 insertions(+), 40 deletions(-) diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index f007240d7..d00aba065 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -114,6 +114,7 @@ configKubernetesCRD: cluster_name_label: cluster-name enable_pod_antiaffinity: false pod_antiaffinity_topology_key: "kubernetes.io/hostname" + enable_pod_disruption_budget: true secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" # inherited_labels: # - application @@ -161,7 +162,7 @@ serviceAccount: # The name of the ServiceAccount to use. # If not set and create is true, a name is generated using the fullname template # When relying solely on the OperatorConfiguration CRD, set this value to "operator" - # Otherwise, the operator tries to use the "default" service account which is forbidden + # Otherwise, the operator tries to use the "default" service account which is forbidden name: "" priorityClassName: "" diff --git a/docs/administrator.md b/docs/administrator.md index 4629d1284..1b51091fe 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -154,6 +154,22 @@ data: pod_antiaffinity_topology_key: "failure-domain.beta.kubernetes.io/zone" ``` +### Pod Disruption Budget + +By default the operator uses a PodDisruptionBudget (PDB) to protect the cluster +from voluntarily disruptions and hence unwanted DB downtime. The `MinAvailable` +parameter of the PDB is set to `1` which prevents killing masters in single-node +clusters and/or the last remaining running instance in a multi-node cluster. + +The PDB is only relaxed in two scenarios: +* If a cluster is scaled down to `0` instances (e.g. for draining nodes) +* If the PDB is disabled in the configuration (`enable_pod_disruption_budget`) + +The PDB is still in place having `MinAvailable` set to `0`. If enabled it will +be automatically set to `1` on scale up. Disabling PDBs helps avoiding blocking +Kubernetes upgrades in managed K8s environments at the cost of prolonged DB +downtime. See PR #384 for the use case. + ### Add cluster-specific labels In some cases, you might want to add `labels` that are specific to a given diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index c9a9db753..e44a0f8cf 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -161,6 +161,13 @@ configuration they are grouped under the `kubernetes` key. replaced by the cluster name. Only the `{cluster}` placeholders is allowed in the template. +* **enable_pod_disruption_budget** + PDB is enabled by default to protect the cluster from voluntarily disruptions + and hence unwanted DB downtime. However, on some cloud providers it could be + necessary to temporarily disabled it, e.g. for node updates. See + [admin docs](../administrator.md#pod-disruption-budget) for more information. + Default is true. + * **secret_name_template** a template for the name of the database user secrets generated by the operator. `{username}` is replaced with name of the secret, `{cluster}` with diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 956ab7f0f..77f2756f6 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -20,6 +20,7 @@ configuration: pod_service_account_name: operator pod_terminate_grace_period: 5m pdb_name_format: "postgres-{cluster}-pdb" + enable_pod_disruption_budget: true secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" cluster_domain: cluster.local oauth_token_secret_name: postgresql-operator diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 136be7843..b6fade388 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -49,6 +49,7 @@ type KubernetesMetaConfiguration struct { SpiloFSGroup *int64 `json:"spilo_fsgroup,omitempty"` WatchedNamespace string `json:"watched_namespace,omitempty"` PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` + EnablePodDisruptionBudget *bool `json:"enable_pod_disruption_budget,omitempty"` SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"` ClusterDomain string `json:"cluster_domain"` OAuthTokenSecretName spec.NamespacedName `json:"oauth_token_secret_name,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 65d8ee925..6a554a2d8 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -76,6 +76,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura *out = new(int64) **out = **in } + if in.EnablePodDisruptionBudget != nil { + in, out := &in.EnablePodDisruptionBudget, &out.EnablePodDisruptionBudget + *out = new(bool) + **out = **in + } out.OAuthTokenSecretName = in.OAuthTokenSecretName out.InfrastructureRolesSecretName = in.InfrastructureRolesSecretName if in.ClusterLabels != nil { diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index b9759ca86..a6644b868 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -579,6 +579,15 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } }() + // pod disruption budget + if oldSpec.Spec.NumberOfInstances != newSpec.Spec.NumberOfInstances { + c.logger.Debug("syncing pod disruption budgets") + if err := c.syncPodDisruptionBudget(true); err != nil { + c.logger.Errorf("could not sync pod disruption budget: %v", err) + updateFailed = true + } + } + // logical backup job func() { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 808105a6d..b7fdbe50e 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1272,26 +1272,26 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_PREFIX", Value: ""}) if description.S3Endpoint != "" { - result = append(result, v1.EnvVar{Name: "CLONE_AWS_ENDPOINT", Value: description.S3Endpoint}) - result = append(result, v1.EnvVar{Name: "CLONE_WALE_S3_ENDPOINT", Value: description.S3Endpoint}) + result = append(result, v1.EnvVar{Name: "CLONE_AWS_ENDPOINT", Value: description.S3Endpoint}) + result = append(result, v1.EnvVar{Name: "CLONE_WALE_S3_ENDPOINT", Value: description.S3Endpoint}) } if description.S3AccessKeyId != "" { - result = append(result, v1.EnvVar{Name: "CLONE_AWS_ACCESS_KEY_ID", Value: description.S3AccessKeyId}) + result = append(result, v1.EnvVar{Name: "CLONE_AWS_ACCESS_KEY_ID", Value: description.S3AccessKeyId}) } if description.S3SecretAccessKey != "" { - result = append(result, v1.EnvVar{Name: "CLONE_AWS_SECRET_ACCESS_KEY", Value: description.S3SecretAccessKey}) + result = append(result, v1.EnvVar{Name: "CLONE_AWS_SECRET_ACCESS_KEY", Value: description.S3SecretAccessKey}) } if description.S3ForcePathStyle != nil { - s3ForcePathStyle := "0" + s3ForcePathStyle := "0" - if *description.S3ForcePathStyle { - s3ForcePathStyle = "1" - } + if *description.S3ForcePathStyle { + s3ForcePathStyle = "1" + } - result = append(result, v1.EnvVar{Name: "CLONE_AWS_S3_FORCE_PATH_STYLE", Value: s3ForcePathStyle}) + result = append(result, v1.EnvVar{Name: "CLONE_AWS_S3_FORCE_PATH_STYLE", Value: s3ForcePathStyle}) } } @@ -1300,6 +1300,12 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) func (c *Cluster) generatePodDisruptionBudget() *policybeta1.PodDisruptionBudget { minAvailable := intstr.FromInt(1) + pdbEnabled := c.OpConfig.EnablePodDisruptionBudget + + // if PodDisruptionBudget is disabled or if there are no DB pods, set the budget to 0. + if (pdbEnabled != nil && !*pdbEnabled) || c.Spec.NumberOfInstances <= 0 { + minAvailable = intstr.FromInt(0) + } return &policybeta1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 7ccbcb51d..0ddb3c55f 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1,6 +1,8 @@ package cluster import ( + "reflect" + "k8s.io/api/core/v1" "testing" @@ -9,6 +11,10 @@ import ( "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + policyv1beta1 "k8s.io/api/policy/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) func True() *bool { @@ -21,6 +27,11 @@ func False() *bool { return &b } +func toIntStr(val int) *intstr.IntOrString { + b := intstr.FromInt(val) + return &b +} + func TestGenerateSpiloJSONConfiguration(t *testing.T) { var cluster = New( Config{ @@ -143,6 +154,113 @@ func TestCreateLoadBalancerLogic(t *testing.T) { } } +func TestGeneratePodDisruptionBudget(t *testing.T) { + tests := []struct { + c *Cluster + out policyv1beta1.PodDisruptionBudget + }{ + // With multiple instances. + { + New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger), + policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres-myapp-database-pdb", + Namespace: "myapp", + Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + MinAvailable: toIntStr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, + }, + }, + }, + }, + // With zero instances. + { + New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 0}}, + logger), + policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres-myapp-database-pdb", + Namespace: "myapp", + Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + MinAvailable: toIntStr(0), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, + }, + }, + }, + }, + // With PodDisruptionBudget disabled. + { + New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb", EnablePodDisruptionBudget: False()}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger), + policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres-myapp-database-pdb", + Namespace: "myapp", + Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + MinAvailable: toIntStr(0), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, + }, + }, + }, + }, + // With non-default PDBNameFormat and PodDisruptionBudget explicitly enabled. + { + New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-databass-budget", EnablePodDisruptionBudget: True()}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger), + policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres-myapp-database-databass-budget", + Namespace: "myapp", + Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + MinAvailable: toIntStr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, + }, + }, + }, + }, + } + + for _, tt := range tests { + result := tt.c.generatePodDisruptionBudget() + if !reflect.DeepEqual(*result, tt.out) { + t.Errorf("Expected PodDisruptionBudget: %#v, got %#v", tt.out, *result) + } + } +} + func TestShmVolume(t *testing.T) { testName := "TestShmVolume" tests := []struct { @@ -269,6 +387,5 @@ func TestCloneEnv(t *testing.T) { t.Errorf("%s %s: Expected env value %s, have %s instead", testName, tt.subTest, tt.env.Value, env.Value) } - } } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index f5ae30b81..f69bdd2d9 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -73,20 +73,6 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } - // create database objects unless we are running without pods or disabled that feature explicitly - if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&newSpec.Spec) <= 0) { - c.logger.Debugf("syncing roles") - if err = c.syncRoles(); err != nil { - err = fmt.Errorf("could not sync roles: %v", err) - return err - } - c.logger.Debugf("syncing databases") - if err = c.syncDatabases(); err != nil { - err = fmt.Errorf("could not sync databases: %v", err) - return err - } - } - c.logger.Debug("syncing pod disruption budgets") if err = c.syncPodDisruptionBudget(false); err != nil { err = fmt.Errorf("could not sync pod disruption budget: %v", err) @@ -103,6 +89,20 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } + // create database objects unless we are running without pods or disabled that feature explicitly + if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&newSpec.Spec) <= 0) { + c.logger.Debugf("syncing roles") + if err = c.syncRoles(); err != nil { + err = fmt.Errorf("could not sync roles: %v", err) + return err + } + c.logger.Debugf("syncing databases") + if err = c.syncDatabases(); err != nil { + err = fmt.Errorf("could not sync databases: %v", err) + return err + } + } + return err } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 66d8bddd6..a6d2173f1 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -46,6 +46,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ClusterDomain = fromCRD.Kubernetes.ClusterDomain result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat + result.EnablePodDisruptionBudget = fromCRD.Kubernetes.EnablePodDisruptionBudget result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate result.OAuthTokenSecretName = fromCRD.Kubernetes.OAuthTokenSecretName result.InfrastructureRolesSecretName = fromCRD.Kubernetes.InfrastructureRolesSecretName diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 82249eb46..b8188e44e 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -110,20 +110,21 @@ type Config struct { EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"` // deprecated and kept for backward compatibility - EnableLoadBalancer *bool `name:"enable_load_balancer"` - MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` - ReplicaDNSNameFormat StringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"` - PDBNameFormat StringTemplate `name:"pdb_name_format" default:"postgres-{cluster}-pdb"` - Workers uint32 `name:"workers" default:"4"` - APIPort int `name:"api_port" default:"8080"` - RingLogLines int `name:"ring_log_lines" default:"100"` - ClusterHistoryEntries int `name:"cluster_history_entries" default:"1000"` - TeamAPIRoleConfiguration map[string]string `name:"team_api_role_configuration" default:"log_statement:all"` - PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` - PodManagementPolicy string `name:"pod_management_policy" default:"ordered_ready"` - ProtectedRoles []string `name:"protected_role_names" default:"admin"` - PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` - SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" defaults:"false"` + EnableLoadBalancer *bool `name:"enable_load_balancer"` + MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` + ReplicaDNSNameFormat StringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"` + PDBNameFormat StringTemplate `name:"pdb_name_format" default:"postgres-{cluster}-pdb"` + EnablePodDisruptionBudget *bool `name:"enable_pod_disruption_budget" default:"true"` + Workers uint32 `name:"workers" default:"4"` + APIPort int `name:"api_port" default:"8080"` + RingLogLines int `name:"ring_log_lines" default:"100"` + ClusterHistoryEntries int `name:"cluster_history_entries" default:"1000"` + TeamAPIRoleConfiguration map[string]string `name:"team_api_role_configuration" default:"log_statement:all"` + PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` + PodManagementPolicy string `name:"pod_management_policy" default:"ordered_ready"` + ProtectedRoles []string `name:"protected_role_names" default:"admin"` + PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` + SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` } // MustMarshal marshals the config or panics diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index bd10256e0..66b51dd1f 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -158,7 +158,7 @@ func SamePDB(cur, new *policybeta1.PodDisruptionBudget) (match bool, reason stri //TODO: improve comparison match = reflect.DeepEqual(new.Spec, cur.Spec) if !match { - reason = "new service spec doesn't match the current one" + reason = "new PDB spec doesn't match the current one" } return From 0ed92ed04e40a5ed45a749b0f89e4615cf44abbf Mon Sep 17 00:00:00 2001 From: Taehyun Kim Date: Wed, 19 Jun 2019 17:47:27 +0900 Subject: [PATCH 4/5] add deletecollection verb (#589) Fixing privileges to execute `patronictl remove`. You could/should have also just used the operator delete cluster flow (remove manifest). It is not really the plan to use patroni inside a pod to remove a existing cluster. --- charts/postgres-operator/templates/clusterrole.yaml | 1 + manifests/operator-service-account-rbac.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 0486d254a..ef421df93 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -43,6 +43,7 @@ rules: verbs: - create - delete + - deletecollection - get - list - patch diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 2057c414f..95221f204 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -40,6 +40,7 @@ rules: verbs: - create - delete + - deletecollection - get - list - patch From 93bfed3e7562f7d6784500d7b523190b6a352e9b Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 19 Jun 2019 12:40:49 +0200 Subject: [PATCH 5/5] Add secret mount to operator (#535) * add secret mount to operator --- charts/postgres-operator/values.yaml | 2 + docs/administrator.md | 16 +++++ docs/reference/operator_parameters.md | 6 ++ manifests/configmap.yaml | 2 + ...gresql-operator-default-configuration.yaml | 2 + .../v1/operator_configuration_type.go | 10 +-- pkg/cluster/k8sres.go | 36 +++++++++- pkg/cluster/k8sres_test.go | 71 +++++++++++++++++++ pkg/controller/operator_config.go | 2 + pkg/util/config/config.go | 2 + 10 files changed, 143 insertions(+), 6 deletions(-) diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index d00aba065..2d8c1122d 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -69,6 +69,8 @@ configAwsOrGcp: # kube_iam_role: "" # log_s3_bucket: "" # wal_s3_bucket: "" + # additional_secret_mount: "some-secret-name" + # additional_secret_mount_path: "/some/dir" configLogicalBackup: logical_backup_schedule: "30 00 * * *" diff --git a/docs/administrator.md b/docs/administrator.md index 1b51091fe..dae0866b6 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -333,3 +333,19 @@ The operator can manage k8s cron jobs to run logical backups of Postgres cluster 4. You may use your own image by overwriting the relevant field in the operator configuration. Any such image must ensure the logical backup is able to finish [in presence of pod restarts](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#handling-pod-and-container-failures) and [simultaneous invocations](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) of the backup cron job. 5. For that feature to work, your RBAC policy must enable operations on the `cronjobs` resource from the `batch` API group for the operator service account. See [example RBAC](../manifests/operator-service-account-rbac.yaml) + +## Access to cloud resources from clusters in non cloud environment + +To access cloud resources like S3 from a cluster in a bare metal setup you can use +`additional_secret_mount` and `additional_secret_mount_path` config parameters. +With this you can provision cloud credentials to the containers in the pods of the StatefulSet. +This works this way that it mounts a volume from the given secret in the pod and this can +then accessed in the container over the configured mount path. Via [Custum Pod Environment Variables](#custom-pod-environment-variables) +you can then point the different cloud sdk's (aws, google etc.) to this mounted secret. +With this credentials the cloud sdk can then access cloud resources to upload logs etc. + +A secret can be pre provisioned in different ways: + +* Generic secret created via `kubectl create secret generic some-cloud-creds --from-file=some-cloud-credentials-file.json` + +* Automaticly provisioned via a Controller like [kube-aws-iam-controller](https://github.com/mikkeloscar/kube-aws-iam-controller). This controller would then also rotate the credentials. Please visit the documention for more information. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index e44a0f8cf..fcd0b2623 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -407,6 +407,12 @@ yet officially supported. * **aws_region** AWS region used to store ESB volumes. The default is `eu-central-1`. +* **additional_secret_mount** + Additional Secret (aws or gcp credentials) to mount in the pod. The default is empty. + +* **additional_secret_mount_path** + Path to mount the above Secret in the filesystem of the container(s). The default is empty. + ## Debugging the operator Options to aid debugging of the operator itself. Grouped under the `debug` key. diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index b9f81cf7e..3a04d3156 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -33,6 +33,8 @@ data: # https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees # inherited_labels: "" aws_region: eu-central-1 + # additional_secret_mount: "some-secret-name" + # additional_secret_mount_path: "/some/dir" db_hosted_zone: db.example.com master_dns_name_format: '{cluster}.{team}.staging.{hostedzone}' replica_dns_name_format: '{cluster}-repl.{team}.staging.{hostedzone}' diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 77f2756f6..e9c46dd8b 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -67,6 +67,8 @@ configuration: # log_s3_bucket: "" # kube_iam_role: "" aws_region: eu-central-1 + # additional_secret_mount: "some-secret-name" + # additional_secret_mount_path: "/some/dir" debug: debug_logging: true enable_database_access: true diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index b6fade388..f69737c37 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -101,10 +101,12 @@ type LoadBalancerConfiguration struct { // AWSGCPConfiguration defines the configuration for AWS // TODO complete Google Cloud Platform (GCP) configuration type AWSGCPConfiguration struct { - WALES3Bucket string `json:"wal_s3_bucket,omitempty"` - AWSRegion string `json:"aws_region,omitempty"` - LogS3Bucket string `json:"log_s3_bucket,omitempty"` - KubeIAMRole string `json:"kube_iam_role,omitempty"` + WALES3Bucket string `json:"wal_s3_bucket,omitempty"` + AWSRegion string `json:"aws_region,omitempty"` + LogS3Bucket string `json:"log_s3_bucket,omitempty"` + KubeIAMRole string `json:"kube_iam_role,omitempty"` + AdditionalSecretMount string `json:"additional_secret_mount,omitempty"` + AdditionalSecretMountPath string `json:"additional_secret_mount_path" default:"/meta/credentials"` } // OperatorDebugConfiguration defines options for the debug mode diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index b7fdbe50e..8cb0ea0a3 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -384,7 +384,7 @@ func generateContainer( VolumeMounts: volumeMounts, Env: envVars, SecurityContext: &v1.SecurityContext{ - Privileged: &privilegedMode, + Privileged: &privilegedMode, ReadOnlyRootFilesystem: &falseBool, }, } @@ -445,6 +445,8 @@ func generatePodTemplate( shmVolume bool, podAntiAffinity bool, podAntiAffinityTopologyKey string, + additionalSecretMount string, + additionalSecretMountPath string, ) (*v1.PodTemplateSpec, error) { terminateGracePeriodSeconds := terminateGracePeriod @@ -479,6 +481,10 @@ func generatePodTemplate( podSpec.PriorityClassName = priorityClassName } + if additionalSecretMount != "" { + addSecretVolume(&podSpec, additionalSecretMount, additionalSecretMountPath) + } + template := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, @@ -864,7 +870,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State effectivePodPriorityClassName, mountShmVolumeNeeded(c.OpConfig, spec), c.OpConfig.EnablePodAntiAffinity, - c.OpConfig.PodAntiAffinityTopologyKey); err != nil { + c.OpConfig.PodAntiAffinityTopologyKey, + c.OpConfig.AdditionalSecretMount, + c.OpConfig.AdditionalSecretMountPath); err != nil { return nil, fmt.Errorf("could not generate pod template: %v", err) } @@ -1013,6 +1021,28 @@ func addShmVolume(podSpec *v1.PodSpec) { podSpec.Volumes = volumes } +func addSecretVolume(podSpec *v1.PodSpec, additionalSecretMount string, additionalSecretMountPath string) { + volumes := append(podSpec.Volumes, v1.Volume{ + Name: additionalSecretMount, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: additionalSecretMount, + }, + }, + }) + + for i := range podSpec.Containers { + mounts := append(podSpec.Containers[i].VolumeMounts, + v1.VolumeMount{ + Name: additionalSecretMount, + MountPath: additionalSecretMountPath, + }) + podSpec.Containers[i].VolumeMounts = mounts + } + + podSpec.Volumes = volumes +} + func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string) (*v1.PersistentVolumeClaim, error) { var storageClassName *string @@ -1395,6 +1425,8 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { "", false, false, + "", + "", ""); err != nil { return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 0ddb3c55f..3980bb4c4 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -389,3 +389,74 @@ func TestCloneEnv(t *testing.T) { } } } + +func TestSecretVolume(t *testing.T) { + testName := "TestSecretVolume" + tests := []struct { + subTest string + podSpec *v1.PodSpec + secretPos int + }{ + { + subTest: "empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{}, + }, + }, + }, + secretPos: 0, + }, + { + subTest: "non empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{{}}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + }, + }, + }, + }, + }, + secretPos: 1, + }, + } + for _, tt := range tests { + additionalSecretMount := "aws-iam-s3-role" + additionalSecretMountPath := "/meta/credentials" + + numMounts := len(tt.podSpec.Containers[0].VolumeMounts) + + addSecretVolume(tt.podSpec, additionalSecretMount, additionalSecretMountPath) + + volumeName := tt.podSpec.Volumes[tt.secretPos].Name + + if volumeName != additionalSecretMount { + t.Errorf("%s %s: Expected volume %s was not created, have %s instead", + testName, tt.subTest, additionalSecretMount, volumeName) + } + + for i := range tt.podSpec.Containers { + volumeMountName := tt.podSpec.Containers[i].VolumeMounts[tt.secretPos].Name + + if volumeMountName != additionalSecretMount { + t.Errorf("%s %s: Expected mount %s was not created, have %s instead", + testName, tt.subTest, additionalSecretMount, volumeMountName) + } + } + + numMountsCheck := len(tt.podSpec.Containers[0].VolumeMounts) + + if numMountsCheck != numMounts+1 { + t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v", + numMountsCheck, numMounts+1) + } + } +} diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index a6d2173f1..c74de8960 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -86,6 +86,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.AWSRegion = fromCRD.AWSGCP.AWSRegion result.LogS3Bucket = fromCRD.AWSGCP.LogS3Bucket result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole + result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount + result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath result.DebugLogging = fromCRD.OperatorDebug.DebugLogging result.EnableDBAccess = fromCRD.OperatorDebug.EnableDBAccess diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index b8188e44e..661574368 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -98,6 +98,8 @@ type Config struct { WALES3Bucket string `name:"wal_s3_bucket"` LogS3Bucket string `name:"log_s3_bucket"` KubeIAMRole string `name:"kube_iam_role"` + AdditionalSecretMount string `name:"additional_secret_mount"` + AdditionalSecretMountPath string `name:"additional_secret_mount_path" default:"/meta/credentials"` DebugLogging bool `name:"debug_logging" default:"true"` EnableDBAccess bool `name:"enable_database_access" default:"true"` EnableTeamsAPI bool `name:"enable_teams_api" default:"true"`