Add secret mount to operator (#535)

* add secret mount to operator

charts/postgres-operator/values.yaml

@@ -69,6 +69,8 @@ configAwsOrGcp:
   # kube_iam_role: ""
   # log_s3_bucket: ""
   # wal_s3_bucket: ""
+  # additional_secret_mount: "some-secret-name"
+  # additional_secret_mount_path: "/some/dir"
 
 configLogicalBackup:
   logical_backup_schedule: "30 00 * * *"

docs/administrator.md

@@ -333,3 +333,19 @@ The operator can manage k8s cron jobs to run logical backups of Postgres cluster
 4. You may use your own image by overwriting the relevant field in the operator configuration. Any such image must ensure the logical backup is able to finish [in presence of pod restarts](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#handling-pod-and-container-failures) and [simultaneous invocations](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) of the backup cron job.
 
 5. For that feature to work, your RBAC policy must enable operations on the `cronjobs` resource from the `batch` API group for the operator service account. See [example RBAC](../manifests/operator-service-account-rbac.yaml)
+
+## Access to cloud resources from clusters in non cloud environment
+
+To access cloud resources like S3 from a cluster in a bare metal setup you can use
+`additional_secret_mount` and `additional_secret_mount_path` config parameters.
+With this you can provision cloud credentials to the containers in the pods of the StatefulSet.
+This works this way that it mounts a volume from the given secret in the pod and this can
+then accessed in the container over the configured mount path. Via [Custum Pod Environment Variables](#custom-pod-environment-variables)
+you can then point the different cloud sdk's (aws, google etc.) to this mounted secret.
+With this credentials the cloud sdk can then access cloud resources to upload logs etc.
+
+A secret can be pre provisioned in different ways:
+
+* Generic secret created via `kubectl create secret generic some-cloud-creds --from-file=some-cloud-credentials-file.json`
+
+* Automaticly provisioned via a Controller like [kube-aws-iam-controller](https://github.com/mikkeloscar/kube-aws-iam-controller). This controller would then also rotate the credentials. Please visit the documention for more information.

docs/reference/operator_parameters.md

@@ -407,6 +407,12 @@ yet officially supported.
 * **aws_region**
   AWS region used to store ESB volumes. The default is `eu-central-1`.
 
+* **additional_secret_mount**
+  Additional Secret (aws or gcp credentials) to mount in the pod. The default is empty.
+
+* **additional_secret_mount_path**
+  Path to mount the above Secret in the filesystem of the container(s). The default is empty.
+
 ## Debugging the operator
 
 Options to aid debugging of the operator itself. Grouped under the `debug` key.

manifests/configmap.yaml

@@ -33,6 +33,8 @@ data:
   #  https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees
   # inherited_labels: ""
   aws_region: eu-central-1
+  # additional_secret_mount: "some-secret-name"
+  # additional_secret_mount_path: "/some/dir"
   db_hosted_zone: db.example.com
   master_dns_name_format: '{cluster}.{team}.staging.{hostedzone}'
   replica_dns_name_format: '{cluster}-repl.{team}.staging.{hostedzone}'

manifests/postgresql-operator-default-configuration.yaml

@@ -67,6 +67,8 @@ configuration:
     # log_s3_bucket: ""
     # kube_iam_role: ""
     aws_region: eu-central-1
+    # additional_secret_mount: "some-secret-name"
+    # additional_secret_mount_path: "/some/dir"
   debug:
     debug_logging: true
     enable_database_access: true

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -105,6 +105,8 @@ type AWSGCPConfiguration struct {
 	AWSRegion                 string `json:"aws_region,omitempty"`
 	LogS3Bucket               string `json:"log_s3_bucket,omitempty"`
 	KubeIAMRole               string `json:"kube_iam_role,omitempty"`
+	AdditionalSecretMount     string `json:"additional_secret_mount,omitempty"`
+	AdditionalSecretMountPath string `json:"additional_secret_mount_path" default:"/meta/credentials"`
 }
 
 // OperatorDebugConfiguration defines options for the debug mode

pkg/cluster/k8sres.go

@@ -445,6 +445,8 @@ func generatePodTemplate(
 	shmVolume bool,
 	podAntiAffinity bool,
 	podAntiAffinityTopologyKey string,
+	additionalSecretMount string,
+	additionalSecretMountPath string,
 ) (*v1.PodTemplateSpec, error) {
 
 	terminateGracePeriodSeconds := terminateGracePeriod
@@ -479,6 +481,10 @@ func generatePodTemplate(
 		podSpec.PriorityClassName = priorityClassName
 	}
 
+	if additionalSecretMount != "" {
+		addSecretVolume(&podSpec, additionalSecretMount, additionalSecretMountPath)
+	}
+
 	template := v1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Labels:    labels,
@@ -864,7 +870,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State
 		effectivePodPriorityClassName,
 		mountShmVolumeNeeded(c.OpConfig, spec),
 		c.OpConfig.EnablePodAntiAffinity,
-		c.OpConfig.PodAntiAffinityTopologyKey); err != nil {
+		c.OpConfig.PodAntiAffinityTopologyKey,
+		c.OpConfig.AdditionalSecretMount,
+		c.OpConfig.AdditionalSecretMountPath); err != nil {
 		return nil, fmt.Errorf("could not generate pod template: %v", err)
 	}
 
@@ -1013,6 +1021,28 @@ func addShmVolume(podSpec *v1.PodSpec) {
 	podSpec.Volumes = volumes
 }
 
+func addSecretVolume(podSpec *v1.PodSpec, additionalSecretMount string, additionalSecretMountPath string) {
+	volumes := append(podSpec.Volumes, v1.Volume{
+		Name: additionalSecretMount,
+		VolumeSource: v1.VolumeSource{
+			Secret: &v1.SecretVolumeSource{
+				SecretName: additionalSecretMount,
+			},
+		},
+	})
+
+	for i := range podSpec.Containers {
+		mounts := append(podSpec.Containers[i].VolumeMounts,
+			v1.VolumeMount{
+				Name:      additionalSecretMount,
+				MountPath: additionalSecretMountPath,
+			})
+		podSpec.Containers[i].VolumeMounts = mounts
+	}
+
+	podSpec.Volumes = volumes
+}
+
 func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string) (*v1.PersistentVolumeClaim, error) {
 
 	var storageClassName *string
@@ -1395,6 +1425,8 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) {
 		"",
 		false,
 		false,
+		"",
+		"",
 		""); err != nil {
 		return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err)
 	}

pkg/cluster/k8sres_test.go

@@ -389,3 +389,74 @@ func TestCloneEnv(t *testing.T) {
 		}
 	}
 }
+
+func TestSecretVolume(t *testing.T) {
+	testName := "TestSecretVolume"
+	tests := []struct {
+		subTest   string
+		podSpec   *v1.PodSpec
+		secretPos int
+	}{
+		{
+			subTest: "empty PodSpec",
+			podSpec: &v1.PodSpec{
+				Volumes: []v1.Volume{},
+				Containers: []v1.Container{
+					{
+						VolumeMounts: []v1.VolumeMount{},
+					},
+				},
+			},
+			secretPos: 0,
+		},
+		{
+			subTest: "non empty PodSpec",
+			podSpec: &v1.PodSpec{
+				Volumes: []v1.Volume{{}},
+				Containers: []v1.Container{
+					{
+						VolumeMounts: []v1.VolumeMount{
+							{
+								Name:      "data",
+								ReadOnly:  false,
+								MountPath: "/data",
+							},
+						},
+					},
+				},
+			},
+			secretPos: 1,
+		},
+	}
+	for _, tt := range tests {
+		additionalSecretMount := "aws-iam-s3-role"
+		additionalSecretMountPath := "/meta/credentials"
+
+		numMounts := len(tt.podSpec.Containers[0].VolumeMounts)
+
+		addSecretVolume(tt.podSpec, additionalSecretMount, additionalSecretMountPath)
+
+		volumeName := tt.podSpec.Volumes[tt.secretPos].Name
+
+		if volumeName != additionalSecretMount {
+			t.Errorf("%s %s: Expected volume %s was not created, have %s instead",
+				testName, tt.subTest, additionalSecretMount, volumeName)
+		}
+
+		for i := range tt.podSpec.Containers {
+			volumeMountName := tt.podSpec.Containers[i].VolumeMounts[tt.secretPos].Name
+
+			if volumeMountName != additionalSecretMount {
+				t.Errorf("%s %s: Expected mount %s was not created, have %s instead",
+					testName, tt.subTest, additionalSecretMount, volumeMountName)
+			}
+		}
+
+		numMountsCheck := len(tt.podSpec.Containers[0].VolumeMounts)
+
+		if numMountsCheck != numMounts+1 {
+			t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v",
+				numMountsCheck, numMounts+1)
+		}
+	}
+}

pkg/controller/operator_config.go

@@ -86,6 +86,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.AWSRegion = fromCRD.AWSGCP.AWSRegion
 	result.LogS3Bucket = fromCRD.AWSGCP.LogS3Bucket
 	result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole
+	result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount
+	result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath
 
 	result.DebugLogging = fromCRD.OperatorDebug.DebugLogging
 	result.EnableDBAccess = fromCRD.OperatorDebug.EnableDBAccess

pkg/util/config/config.go

@@ -98,6 +98,8 @@ type Config struct {
 	WALES3Bucket                           string            `name:"wal_s3_bucket"`
 	LogS3Bucket                            string            `name:"log_s3_bucket"`
 	KubeIAMRole                            string            `name:"kube_iam_role"`
+	AdditionalSecretMount                  string            `name:"additional_secret_mount"`
+	AdditionalSecretMountPath              string            `name:"additional_secret_mount_path" default:"/meta/credentials"`
 	DebugLogging                           bool              `name:"debug_logging" default:"true"`
 	EnableDBAccess                         bool              `name:"enable_database_access" default:"true"`
 	EnableTeamsAPI                         bool              `name:"enable_teams_api" default:"true"`