Support for GCS WAL-E backups (#620)

* Support for WAL_GS_BUCKET and GOOGLE_APPLICATION_CREDENTIALS environtment variables

* Fixed merge issue but also removed all changes to support macos.

* Updated test to new format

* Missed macos specific changes

* Added documentation and addressed comments

* Update docs/administrator.md

* Update docs/administrator.md

* Update e2e/run.sh

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>

charts/postgres-operator/values-crd.yaml

@@ -202,12 +202,18 @@ configAwsOrGcp:
   # AWS region used to store ESB volumes
   aws_region: eu-central-1
 
+  # GCP credentials that will be used by the operator / pods
+  # gcp_credentials: ""
+
   # AWS IAM role to supply in the iam.amazonaws.com/role annotation of Postgres pods
   # kube_iam_role: ""
 
   # S3 bucket to use for shipping postgres daily logs
   # log_s3_bucket: ""
 
+  # GCS bucket to use for shipping WAL segments with WAL-E
+  # wal_gs_bucket: ""
+
   # S3 bucket to use for shipping WAL segments with WAL-E
   # wal_s3_bucket: ""
 

charts/postgres-operator/values.yaml

@@ -200,6 +200,12 @@ configAwsOrGcp:
   # S3 bucket to use for shipping WAL segments with WAL-E
   # wal_s3_bucket: ""
 
+  # GCS bucket to use for shipping WAL segments with WAL-E
+  # wal_gs_bucket: ""
+
+  # GCP credentials for setting the GOOGLE_APPLICATION_CREDNETIALS environment variable
+  # gcp_credentials: ""
+
 # configure K8s cron job managed by the operator
 configLogicalBackup:
   # image for pods of the logical backup job (example runs pg_dumpall)

docs/administrator.md

@@ -518,6 +518,57 @@ A secret can be pre-provisioned in different ways:
 * Automatically provisioned via a custom K8s controller like
   [kube-aws-iam-controller](https://github.com/mikkeloscar/kube-aws-iam-controller)
 
+## Google Cloud Platform setup
+
+To configure the operator on GCP there are some prerequisites that are needed:
+
+* A service account with the proper IAM setup to access the GCS bucket for the WAL-E logs
+* The credentials file for the service account.
+
+The configuration paramaters that we will be using are:
+
+* `additional_secret_mount`
+* `additional_secret_mount_path`
+* `gcp_credentials`
+* `wal_gs_bucket`
+
+### Generate a K8 secret resource
+
+Generate the K8 secret resource that will contain your service account's 
+credentials. It's highly recommended to use a service account and limit its
+scope to just the WAL-E bucket. 
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: psql-wale-creds
+  namespace: default
+type: Opaque
+stringData:
+  key.json: |-
+    <GCP .json credentials>
+```
+
+### Setup your operator configuration values
+
+With the `psql-wale-creds` resource applied to your cluster, ensure that
+the operator's configuration is set up like the following:
+
+```yml
+...
+aws_or_gcp:
+  additional_secret_mount: "pgsql-wale-creds"
+  additional_secret_mount_path: "/var/secrets/google" # or where ever you want to mount the file
+  # aws_region: eu-central-1
+  # kube_iam_role: ""
+  # log_s3_bucket: ""
+  # wal_s3_bucket: ""
+  wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs
+  gcp_credentials: "/var/secrets/google/key.json" # combination of the mount path & key in the K8 resource. (i.e. key.json)
+...
+```
+
 ## Sidecars for Postgres clusters
 
 A list of sidecars is added to each cluster created by the operator. The default

docs/reference/operator_parameters.md

@@ -451,6 +451,20 @@ yet officially supported.
   present and accessible by Postgres pods. At the moment, supported services by
   Spilo are S3 and GCS. The default is empty.
 
+* **wal_gs_bucket**
+  GCS bucket to use for shipping WAL segments with WAL-E. A bucket has to be
+  present and accessible by Postgres pods. Note, only the name of the bucket is
+  required. At the moment, supported services by Spilo are S3 and GCS.
+  The default is empty.
+
+* **gcp_credentials**
+  Used to set the GOOGLE_APPLICATION_CREDENTIALS environment variable for the pods.
+  This is used in with conjunction with the `additional_secret_mount` and
+  `additional_secret_mount_path` to properly set the credentials for the spilo
+  containers. This will allow users to use specific
+  [service accounts](https://cloud.google.com/kubernetes-engine/docs/tutorials/authenticating-to-cloud-platform).
+  The default is empty
+
 * **log_s3_bucket**
   S3 bucket to use for shipping Postgres daily logs. Works only with S3 on AWS.
   The bucket has to be present and accessible by Postgres pods. The default is

manifests/configmap.yaml

@@ -45,6 +45,7 @@ data:
   # enable_team_superuser: "false"
   enable_teams_api: "false"
   # etcd_host: ""
+  # gcp_credentials: ""
   # kubernetes_use_configmaps: "false"
   # infrastructure_roles_secret_name: postgresql-infrastructure-roles
   # inherited_labels: application,environment
@@ -100,6 +101,7 @@ data:
   # team_api_role_configuration: "log_statement:all"
   # teams_api_url: http://fake-teams-api.default.svc.cluster.local
   # toleration: ""
+  # wal_gs_bucket: ""
   # wal_s3_bucket: ""
   watched_namespace: "*"  # listen to all namespaces
   workers: "4"

manifests/operatorconfiguration.crd.yaml

@@ -216,10 +216,14 @@ spec:
                   type: string
                 aws_region:
                   type: string
+                gcp_credentials:
+                  type: string
                 kube_iam_role:
                   type: string
                 log_s3_bucket:
                   type: string
+                wal_gs_bucket:
+                  type: string
                 wal_s3_bucket:
                   type: string
             logical_backup:

manifests/postgresql-operator-default-configuration.yaml

@@ -88,8 +88,10 @@ configuration:
     # additional_secret_mount: "some-secret-name"
     # additional_secret_mount_path: "/some/dir"
     aws_region: eu-central-1
+    # gcp_credentials: ""
     # kube_iam_role: ""
     # log_s3_bucket: ""
+    # wal_gs_bucket: ""
     # wal_s3_bucket: ""
   logical_backup:
     logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-58"

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -111,6 +111,8 @@ type LoadBalancerConfiguration struct {
 type AWSGCPConfiguration struct {
 	WALES3Bucket              string `json:"wal_s3_bucket,omitempty"`
 	AWSRegion                 string `json:"aws_region,omitempty"`
+	WALGSBucket               string `json:"wal_gs_bucket,omitempty"`
+	GCPCredentials            string `json:"gcp_credentials,omitempty"`
 	LogS3Bucket               string `json:"log_s3_bucket,omitempty"`
 	KubeIAMRole               string `json:"kube_iam_role,omitempty"`
 	AdditionalSecretMount     string `json:"additional_secret_mount,omitempty"`

pkg/cluster/k8sres.go

@@ -714,12 +714,23 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri
 	if spiloConfiguration != "" {
 		envVars = append(envVars, v1.EnvVar{Name: "SPILO_CONFIGURATION", Value: spiloConfiguration})
 	}
+
 	if c.OpConfig.WALES3Bucket != "" {
 		envVars = append(envVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket})
 		envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
 		envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
 	}
 
+	if c.OpConfig.WALGSBucket != "" {
+		envVars = append(envVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket})
+		envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})
+		envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""})
+	}
+
+	if c.OpConfig.GCPCredentials != "" {
+		envVars = append(envVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials})
+	}
+
 	if c.OpConfig.LogS3Bucket != "" {
 		envVars = append(envVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket})
 		envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))})

pkg/cluster/k8sres_test.go

@@ -20,9 +20,17 @@ import (
 	policyv1beta1 "k8s.io/api/policy/v1beta1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
+// For testing purposes
+type ExpectedValue struct {
+	envIndex       int
+	envVarConstant string
+	envVarValue    string
+}
+
 func toIntStr(val int) *intstr.IntOrString {
 	b := intstr.FromInt(val)
 	return &b
@@ -93,6 +101,119 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) {
 	}
 }
 
+func TestGenerateSpiloPodEnvVars(t *testing.T) {
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				WALGSBucket:    "wale-gs-bucket",
+				ProtectedRoles: []string{"admin"},
+				Auth: config.Auth{
+					SuperUsername:       superUserName,
+					ReplicationUsername: replicationUserName,
+				},
+			},
+		}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
+
+	expectedValuesGSBucket := []ExpectedValue{
+		ExpectedValue{
+			envIndex:       14,
+			envVarConstant: "WAL_GS_BUCKET",
+			envVarValue:    "wale-gs-bucket",
+		},
+		ExpectedValue{
+			envIndex:       15,
+			envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX",
+			envVarValue:    "/SomeUUID",
+		},
+		ExpectedValue{
+			envIndex:       16,
+			envVarConstant: "WAL_BUCKET_SCOPE_PREFIX",
+			envVarValue:    "",
+		},
+	}
+
+	expectedValuesGCPCreds := []ExpectedValue{
+		ExpectedValue{
+			envIndex:       14,
+			envVarConstant: "WAL_GS_BUCKET",
+			envVarValue:    "wale-gs-bucket",
+		},
+		ExpectedValue{
+			envIndex:       15,
+			envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX",
+			envVarValue:    "/SomeUUID",
+		},
+		ExpectedValue{
+			envIndex:       16,
+			envVarConstant: "WAL_BUCKET_SCOPE_PREFIX",
+			envVarValue:    "",
+		},
+		ExpectedValue{
+			envIndex:       17,
+			envVarConstant: "GOOGLE_APPLICATION_CREDENTIALS",
+			envVarValue:    "some_path_to_credentials",
+		},
+	}
+
+	testName := "TestGenerateSpiloPodEnvVars"
+	tests := []struct {
+		subTest            string
+		opConfig           config.Config
+		uid                types.UID
+		spiloConfig        string
+		cloneDescription   *acidv1.CloneDescription
+		standbyDescription *acidv1.StandbyDescription
+		customEnvList      []v1.EnvVar
+		expectedValues     []ExpectedValue
+	}{
+		{
+			subTest: "Will set WAL_GS_BUCKET env",
+			opConfig: config.Config{
+				WALGSBucket: "wale-gs-bucket",
+			},
+			uid:                "SomeUUID",
+			spiloConfig:        "someConfig",
+			cloneDescription:   &acidv1.CloneDescription{},
+			standbyDescription: &acidv1.StandbyDescription{},
+			customEnvList:      []v1.EnvVar{},
+			expectedValues:     expectedValuesGSBucket,
+		},
+		{
+			subTest: "Will set GOOGLE_APPLICATION_CREDENTIALS env",
+			opConfig: config.Config{
+				WALGSBucket:    "wale-gs-bucket",
+				GCPCredentials: "some_path_to_credentials",
+			},
+			uid:                "SomeUUID",
+			spiloConfig:        "someConfig",
+			cloneDescription:   &acidv1.CloneDescription{},
+			standbyDescription: &acidv1.StandbyDescription{},
+			customEnvList:      []v1.EnvVar{},
+			expectedValues:     expectedValuesGCPCreds,
+		},
+	}
+
+	for _, tt := range tests {
+		cluster.OpConfig = tt.opConfig
+
+		actualEnvs := cluster.generateSpiloPodEnvVars(tt.uid, tt.spiloConfig, tt.cloneDescription, tt.standbyDescription, tt.customEnvList)
+
+		for _, ev := range tt.expectedValues {
+			env := actualEnvs[ev.envIndex]
+
+			if env.Name != ev.envVarConstant {
+				t.Errorf("%s %s: Expected env name %s, have %s instead",
+					testName, tt.subTest, ev.envVarConstant, env.Name)
+			}
+
+			if env.Value != ev.envVarValue {
+				t.Errorf("%s %s: Expected env value %s, have %s instead",
+					testName, tt.subTest, ev.envVarValue, env.Value)
+			}
+		}
+	}
+}
+
 func TestCreateLoadBalancerLogic(t *testing.T) {
 	var cluster = New(
 		Config{

pkg/controller/operator_config.go

@@ -111,6 +111,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.AWSRegion = fromCRD.AWSGCP.AWSRegion
 	result.LogS3Bucket = fromCRD.AWSGCP.LogS3Bucket
 	result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole
+	result.WALGSBucket = fromCRD.AWSGCP.WALGSBucket
+	result.GCPCredentials = fromCRD.AWSGCP.GCPCredentials
 	result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount
 	result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath
 

pkg/util/config/config.go

@@ -126,6 +126,8 @@ type Config struct {
 	WALES3Bucket                           string            `name:"wal_s3_bucket"`
 	LogS3Bucket                            string            `name:"log_s3_bucket"`
 	KubeIAMRole                            string            `name:"kube_iam_role"`
+	WALGSBucket                            string            `name:"wal_gs_bucket"`
+	GCPCredentials                         string            `name:"gcp_credentials"`
 	AdditionalSecretMount                  string            `name:"additional_secret_mount"`
 	AdditionalSecretMountPath              string            `name:"additional_secret_mount_path" default:"/meta/credentials"`
 	DebugLogging                           bool              `name:"debug_logging" default:"true"`