diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 4f57dd642..4be39325f 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -202,12 +202,18 @@ configAwsOrGcp: # AWS region used to store ESB volumes aws_region: eu-central-1 + # GCP credentials that will be used by the operator / pods + # gcp_credentials: "" + # AWS IAM role to supply in the iam.amazonaws.com/role annotation of Postgres pods # kube_iam_role: "" # S3 bucket to use for shipping postgres daily logs # log_s3_bucket: "" + # GCS bucket to use for shipping WAL segments with WAL-E + # wal_gs_bucket: "" + # S3 bucket to use for shipping WAL segments with WAL-E # wal_s3_bucket: "" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 2a6a181f5..54461d141 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -200,6 +200,12 @@ configAwsOrGcp: # S3 bucket to use for shipping WAL segments with WAL-E # wal_s3_bucket: "" + # GCS bucket to use for shipping WAL segments with WAL-E + # wal_gs_bucket: "" + + # GCP credentials for setting the GOOGLE_APPLICATION_CREDNETIALS environment variable + # gcp_credentials: "" + # configure K8s cron job managed by the operator configLogicalBackup: # image for pods of the logical backup job (example runs pg_dumpall) diff --git a/docs/administrator.md b/docs/administrator.md index 45a328d38..e2c2e01eb 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -518,6 +518,57 @@ A secret can be pre-provisioned in different ways: * Automatically provisioned via a custom K8s controller like [kube-aws-iam-controller](https://github.com/mikkeloscar/kube-aws-iam-controller) +## Google Cloud Platform setup + +To configure the operator on GCP there are some prerequisites that are needed: + +* A service account with the proper IAM setup to access the GCS bucket for the WAL-E logs +* The credentials file for the service account. + +The configuration paramaters that we will be using are: + +* `additional_secret_mount` +* `additional_secret_mount_path` +* `gcp_credentials` +* `wal_gs_bucket` + +### Generate a K8 secret resource + +Generate the K8 secret resource that will contain your service account's +credentials. It's highly recommended to use a service account and limit its +scope to just the WAL-E bucket. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: psql-wale-creds + namespace: default +type: Opaque +stringData: + key.json: |- + +``` + +### Setup your operator configuration values + +With the `psql-wale-creds` resource applied to your cluster, ensure that +the operator's configuration is set up like the following: + +```yml +... +aws_or_gcp: + additional_secret_mount: "pgsql-wale-creds" + additional_secret_mount_path: "/var/secrets/google" # or where ever you want to mount the file + # aws_region: eu-central-1 + # kube_iam_role: "" + # log_s3_bucket: "" + # wal_s3_bucket: "" + wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs + gcp_credentials: "/var/secrets/google/key.json" # combination of the mount path & key in the K8 resource. (i.e. key.json) +... +``` + ## Sidecars for Postgres clusters A list of sidecars is added to each cluster created by the operator. The default diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index a81cabfc4..691e2f262 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -451,6 +451,20 @@ yet officially supported. present and accessible by Postgres pods. At the moment, supported services by Spilo are S3 and GCS. The default is empty. +* **wal_gs_bucket** + GCS bucket to use for shipping WAL segments with WAL-E. A bucket has to be + present and accessible by Postgres pods. Note, only the name of the bucket is + required. At the moment, supported services by Spilo are S3 and GCS. + The default is empty. + +* **gcp_credentials** + Used to set the GOOGLE_APPLICATION_CREDENTIALS environment variable for the pods. + This is used in with conjunction with the `additional_secret_mount` and + `additional_secret_mount_path` to properly set the credentials for the spilo + containers. This will allow users to use specific + [service accounts](https://cloud.google.com/kubernetes-engine/docs/tutorials/authenticating-to-cloud-platform). + The default is empty + * **log_s3_bucket** S3 bucket to use for shipping Postgres daily logs. Works only with S3 on AWS. The bucket has to be present and accessible by Postgres pods. The default is diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 4314b41d3..8ec850bae 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -45,6 +45,7 @@ data: # enable_team_superuser: "false" enable_teams_api: "false" # etcd_host: "" + # gcp_credentials: "" # kubernetes_use_configmaps: "false" # infrastructure_roles_secret_name: postgresql-infrastructure-roles # inherited_labels: application,environment @@ -100,6 +101,7 @@ data: # team_api_role_configuration: "log_statement:all" # teams_api_url: http://fake-teams-api.default.svc.cluster.local # toleration: "" + # wal_gs_bucket: "" # wal_s3_bucket: "" watched_namespace: "*" # listen to all namespaces workers: "4" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 23b5ff0fc..d02ff1682 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -216,10 +216,14 @@ spec: type: string aws_region: type: string + gcp_credentials: + type: string kube_iam_role: type: string log_s3_bucket: type: string + wal_gs_bucket: + type: string wal_s3_bucket: type: string logical_backup: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 049e917f6..a7ca8c4ee 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -88,8 +88,10 @@ configuration: # additional_secret_mount: "some-secret-name" # additional_secret_mount_path: "/some/dir" aws_region: eu-central-1 + # gcp_credentials: "" # kube_iam_role: "" # log_s3_bucket: "" + # wal_gs_bucket: "" # wal_s3_bucket: "" logical_backup: logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-58" diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index d3a9f6ec2..2dd0bbb50 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -111,6 +111,8 @@ type LoadBalancerConfiguration struct { type AWSGCPConfiguration struct { WALES3Bucket string `json:"wal_s3_bucket,omitempty"` AWSRegion string `json:"aws_region,omitempty"` + WALGSBucket string `json:"wal_gs_bucket,omitempty"` + GCPCredentials string `json:"gcp_credentials,omitempty"` LogS3Bucket string `json:"log_s3_bucket,omitempty"` KubeIAMRole string `json:"kube_iam_role,omitempty"` AdditionalSecretMount string `json:"additional_secret_mount,omitempty"` diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index e1093b72b..ef20da062 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -714,12 +714,23 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri if spiloConfiguration != "" { envVars = append(envVars, v1.EnvVar{Name: "SPILO_CONFIGURATION", Value: spiloConfiguration}) } + if c.OpConfig.WALES3Bucket != "" { envVars = append(envVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) } + if c.OpConfig.WALGSBucket != "" { + envVars = append(envVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket}) + envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + } + + if c.OpConfig.GCPCredentials != "" { + envVars = append(envVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials}) + } + if c.OpConfig.LogS3Bucket != "" { envVars = append(envVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket}) envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 78f088389..ff830a1f5 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -20,9 +20,17 @@ import ( policyv1beta1 "k8s.io/api/policy/v1beta1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" ) +// For testing purposes +type ExpectedValue struct { + envIndex int + envVarConstant string + envVarValue string +} + func toIntStr(val int) *intstr.IntOrString { b := intstr.FromInt(val) return &b @@ -93,6 +101,119 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { } } +func TestGenerateSpiloPodEnvVars(t *testing.T) { + var cluster = New( + Config{ + OpConfig: config.Config{ + WALGSBucket: "wale-gs-bucket", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + expectedValuesGSBucket := []ExpectedValue{ + ExpectedValue{ + envIndex: 14, + envVarConstant: "WAL_GS_BUCKET", + envVarValue: "wale-gs-bucket", + }, + ExpectedValue{ + envIndex: 15, + envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", + envVarValue: "/SomeUUID", + }, + ExpectedValue{ + envIndex: 16, + envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", + envVarValue: "", + }, + } + + expectedValuesGCPCreds := []ExpectedValue{ + ExpectedValue{ + envIndex: 14, + envVarConstant: "WAL_GS_BUCKET", + envVarValue: "wale-gs-bucket", + }, + ExpectedValue{ + envIndex: 15, + envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", + envVarValue: "/SomeUUID", + }, + ExpectedValue{ + envIndex: 16, + envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", + envVarValue: "", + }, + ExpectedValue{ + envIndex: 17, + envVarConstant: "GOOGLE_APPLICATION_CREDENTIALS", + envVarValue: "some_path_to_credentials", + }, + } + + testName := "TestGenerateSpiloPodEnvVars" + tests := []struct { + subTest string + opConfig config.Config + uid types.UID + spiloConfig string + cloneDescription *acidv1.CloneDescription + standbyDescription *acidv1.StandbyDescription + customEnvList []v1.EnvVar + expectedValues []ExpectedValue + }{ + { + subTest: "Will set WAL_GS_BUCKET env", + opConfig: config.Config{ + WALGSBucket: "wale-gs-bucket", + }, + uid: "SomeUUID", + spiloConfig: "someConfig", + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + customEnvList: []v1.EnvVar{}, + expectedValues: expectedValuesGSBucket, + }, + { + subTest: "Will set GOOGLE_APPLICATION_CREDENTIALS env", + opConfig: config.Config{ + WALGSBucket: "wale-gs-bucket", + GCPCredentials: "some_path_to_credentials", + }, + uid: "SomeUUID", + spiloConfig: "someConfig", + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + customEnvList: []v1.EnvVar{}, + expectedValues: expectedValuesGCPCreds, + }, + } + + for _, tt := range tests { + cluster.OpConfig = tt.opConfig + + actualEnvs := cluster.generateSpiloPodEnvVars(tt.uid, tt.spiloConfig, tt.cloneDescription, tt.standbyDescription, tt.customEnvList) + + for _, ev := range tt.expectedValues { + env := actualEnvs[ev.envIndex] + + if env.Name != ev.envVarConstant { + t.Errorf("%s %s: Expected env name %s, have %s instead", + testName, tt.subTest, ev.envVarConstant, env.Name) + } + + if env.Value != ev.envVarValue { + t.Errorf("%s %s: Expected env value %s, have %s instead", + testName, tt.subTest, ev.envVarValue, env.Value) + } + } + } +} + func TestCreateLoadBalancerLogic(t *testing.T) { var cluster = New( Config{ diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 41d701fe2..2a3a01fd4 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -111,6 +111,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.AWSRegion = fromCRD.AWSGCP.AWSRegion result.LogS3Bucket = fromCRD.AWSGCP.LogS3Bucket result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole + result.WALGSBucket = fromCRD.AWSGCP.WALGSBucket + result.GCPCredentials = fromCRD.AWSGCP.GCPCredentials result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 348452193..d15c7caa7 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -126,6 +126,8 @@ type Config struct { WALES3Bucket string `name:"wal_s3_bucket"` LogS3Bucket string `name:"log_s3_bucket"` KubeIAMRole string `name:"kube_iam_role"` + WALGSBucket string `name:"wal_gs_bucket"` + GCPCredentials string `name:"gcp_credentials"` AdditionalSecretMount string `name:"additional_secret_mount"` AdditionalSecretMountPath string `name:"additional_secret_mount_path" default:"/meta/credentials"` DebugLogging bool `name:"debug_logging" default:"true"`