From 93bfed3e7562f7d6784500d7b523190b6a352e9b Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 19 Jun 2019 12:40:49 +0200 Subject: [PATCH] Add secret mount to operator (#535) * add secret mount to operator --- charts/postgres-operator/values.yaml | 2 + docs/administrator.md | 16 +++++ docs/reference/operator_parameters.md | 6 ++ manifests/configmap.yaml | 2 + ...gresql-operator-default-configuration.yaml | 2 + .../v1/operator_configuration_type.go | 10 +-- pkg/cluster/k8sres.go | 36 +++++++++- pkg/cluster/k8sres_test.go | 71 +++++++++++++++++++ pkg/controller/operator_config.go | 2 + pkg/util/config/config.go | 2 + 10 files changed, 143 insertions(+), 6 deletions(-) diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index d00aba065..2d8c1122d 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -69,6 +69,8 @@ configAwsOrGcp: # kube_iam_role: "" # log_s3_bucket: "" # wal_s3_bucket: "" + # additional_secret_mount: "some-secret-name" + # additional_secret_mount_path: "/some/dir" configLogicalBackup: logical_backup_schedule: "30 00 * * *" diff --git a/docs/administrator.md b/docs/administrator.md index 1b51091fe..dae0866b6 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -333,3 +333,19 @@ The operator can manage k8s cron jobs to run logical backups of Postgres cluster 4. You may use your own image by overwriting the relevant field in the operator configuration. Any such image must ensure the logical backup is able to finish [in presence of pod restarts](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#handling-pod-and-container-failures) and [simultaneous invocations](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) of the backup cron job. 5. For that feature to work, your RBAC policy must enable operations on the `cronjobs` resource from the `batch` API group for the operator service account. See [example RBAC](../manifests/operator-service-account-rbac.yaml) + +## Access to cloud resources from clusters in non cloud environment + +To access cloud resources like S3 from a cluster in a bare metal setup you can use +`additional_secret_mount` and `additional_secret_mount_path` config parameters. +With this you can provision cloud credentials to the containers in the pods of the StatefulSet. +This works this way that it mounts a volume from the given secret in the pod and this can +then accessed in the container over the configured mount path. Via [Custum Pod Environment Variables](#custom-pod-environment-variables) +you can then point the different cloud sdk's (aws, google etc.) to this mounted secret. +With this credentials the cloud sdk can then access cloud resources to upload logs etc. + +A secret can be pre provisioned in different ways: + +* Generic secret created via `kubectl create secret generic some-cloud-creds --from-file=some-cloud-credentials-file.json` + +* Automaticly provisioned via a Controller like [kube-aws-iam-controller](https://github.com/mikkeloscar/kube-aws-iam-controller). This controller would then also rotate the credentials. Please visit the documention for more information. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index e44a0f8cf..fcd0b2623 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -407,6 +407,12 @@ yet officially supported. * **aws_region** AWS region used to store ESB volumes. The default is `eu-central-1`. +* **additional_secret_mount** + Additional Secret (aws or gcp credentials) to mount in the pod. The default is empty. + +* **additional_secret_mount_path** + Path to mount the above Secret in the filesystem of the container(s). The default is empty. + ## Debugging the operator Options to aid debugging of the operator itself. Grouped under the `debug` key. diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index b9f81cf7e..3a04d3156 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -33,6 +33,8 @@ data: # https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees # inherited_labels: "" aws_region: eu-central-1 + # additional_secret_mount: "some-secret-name" + # additional_secret_mount_path: "/some/dir" db_hosted_zone: db.example.com master_dns_name_format: '{cluster}.{team}.staging.{hostedzone}' replica_dns_name_format: '{cluster}-repl.{team}.staging.{hostedzone}' diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 77f2756f6..e9c46dd8b 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -67,6 +67,8 @@ configuration: # log_s3_bucket: "" # kube_iam_role: "" aws_region: eu-central-1 + # additional_secret_mount: "some-secret-name" + # additional_secret_mount_path: "/some/dir" debug: debug_logging: true enable_database_access: true diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index b6fade388..f69737c37 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -101,10 +101,12 @@ type LoadBalancerConfiguration struct { // AWSGCPConfiguration defines the configuration for AWS // TODO complete Google Cloud Platform (GCP) configuration type AWSGCPConfiguration struct { - WALES3Bucket string `json:"wal_s3_bucket,omitempty"` - AWSRegion string `json:"aws_region,omitempty"` - LogS3Bucket string `json:"log_s3_bucket,omitempty"` - KubeIAMRole string `json:"kube_iam_role,omitempty"` + WALES3Bucket string `json:"wal_s3_bucket,omitempty"` + AWSRegion string `json:"aws_region,omitempty"` + LogS3Bucket string `json:"log_s3_bucket,omitempty"` + KubeIAMRole string `json:"kube_iam_role,omitempty"` + AdditionalSecretMount string `json:"additional_secret_mount,omitempty"` + AdditionalSecretMountPath string `json:"additional_secret_mount_path" default:"/meta/credentials"` } // OperatorDebugConfiguration defines options for the debug mode diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index b7fdbe50e..8cb0ea0a3 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -384,7 +384,7 @@ func generateContainer( VolumeMounts: volumeMounts, Env: envVars, SecurityContext: &v1.SecurityContext{ - Privileged: &privilegedMode, + Privileged: &privilegedMode, ReadOnlyRootFilesystem: &falseBool, }, } @@ -445,6 +445,8 @@ func generatePodTemplate( shmVolume bool, podAntiAffinity bool, podAntiAffinityTopologyKey string, + additionalSecretMount string, + additionalSecretMountPath string, ) (*v1.PodTemplateSpec, error) { terminateGracePeriodSeconds := terminateGracePeriod @@ -479,6 +481,10 @@ func generatePodTemplate( podSpec.PriorityClassName = priorityClassName } + if additionalSecretMount != "" { + addSecretVolume(&podSpec, additionalSecretMount, additionalSecretMountPath) + } + template := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, @@ -864,7 +870,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State effectivePodPriorityClassName, mountShmVolumeNeeded(c.OpConfig, spec), c.OpConfig.EnablePodAntiAffinity, - c.OpConfig.PodAntiAffinityTopologyKey); err != nil { + c.OpConfig.PodAntiAffinityTopologyKey, + c.OpConfig.AdditionalSecretMount, + c.OpConfig.AdditionalSecretMountPath); err != nil { return nil, fmt.Errorf("could not generate pod template: %v", err) } @@ -1013,6 +1021,28 @@ func addShmVolume(podSpec *v1.PodSpec) { podSpec.Volumes = volumes } +func addSecretVolume(podSpec *v1.PodSpec, additionalSecretMount string, additionalSecretMountPath string) { + volumes := append(podSpec.Volumes, v1.Volume{ + Name: additionalSecretMount, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: additionalSecretMount, + }, + }, + }) + + for i := range podSpec.Containers { + mounts := append(podSpec.Containers[i].VolumeMounts, + v1.VolumeMount{ + Name: additionalSecretMount, + MountPath: additionalSecretMountPath, + }) + podSpec.Containers[i].VolumeMounts = mounts + } + + podSpec.Volumes = volumes +} + func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string) (*v1.PersistentVolumeClaim, error) { var storageClassName *string @@ -1395,6 +1425,8 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { "", false, false, + "", + "", ""); err != nil { return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err) } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 0ddb3c55f..3980bb4c4 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -389,3 +389,74 @@ func TestCloneEnv(t *testing.T) { } } } + +func TestSecretVolume(t *testing.T) { + testName := "TestSecretVolume" + tests := []struct { + subTest string + podSpec *v1.PodSpec + secretPos int + }{ + { + subTest: "empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{}, + }, + }, + }, + secretPos: 0, + }, + { + subTest: "non empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{{}}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + }, + }, + }, + }, + }, + secretPos: 1, + }, + } + for _, tt := range tests { + additionalSecretMount := "aws-iam-s3-role" + additionalSecretMountPath := "/meta/credentials" + + numMounts := len(tt.podSpec.Containers[0].VolumeMounts) + + addSecretVolume(tt.podSpec, additionalSecretMount, additionalSecretMountPath) + + volumeName := tt.podSpec.Volumes[tt.secretPos].Name + + if volumeName != additionalSecretMount { + t.Errorf("%s %s: Expected volume %s was not created, have %s instead", + testName, tt.subTest, additionalSecretMount, volumeName) + } + + for i := range tt.podSpec.Containers { + volumeMountName := tt.podSpec.Containers[i].VolumeMounts[tt.secretPos].Name + + if volumeMountName != additionalSecretMount { + t.Errorf("%s %s: Expected mount %s was not created, have %s instead", + testName, tt.subTest, additionalSecretMount, volumeMountName) + } + } + + numMountsCheck := len(tt.podSpec.Containers[0].VolumeMounts) + + if numMountsCheck != numMounts+1 { + t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v", + numMountsCheck, numMounts+1) + } + } +} diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index a6d2173f1..c74de8960 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -86,6 +86,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.AWSRegion = fromCRD.AWSGCP.AWSRegion result.LogS3Bucket = fromCRD.AWSGCP.LogS3Bucket result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole + result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount + result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath result.DebugLogging = fromCRD.OperatorDebug.DebugLogging result.EnableDBAccess = fromCRD.OperatorDebug.EnableDBAccess diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index b8188e44e..661574368 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -98,6 +98,8 @@ type Config struct { WALES3Bucket string `name:"wal_s3_bucket"` LogS3Bucket string `name:"log_s3_bucket"` KubeIAMRole string `name:"kube_iam_role"` + AdditionalSecretMount string `name:"additional_secret_mount"` + AdditionalSecretMountPath string `name:"additional_secret_mount_path" default:"/meta/credentials"` DebugLogging bool `name:"debug_logging" default:"true"` EnableDBAccess bool `name:"enable_database_access" default:"true"` EnableTeamsAPI bool `name:"enable_teams_api" default:"true"`