Add region setting for logical backups to non-AWS storage (#813)

* Add region setting for logical backups to non-AWS storage

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -243,6 +243,8 @@ spec:
                   type: string
                 logical_backup_s3_endpoint:
                   type: string
+                logical_backup_s3_region:
+                  type: string
                 logical_backup_s3_secret_access_key:
                   type: string
                 logical_backup_s3_sse:

charts/postgres-operator/values-crd.yaml

@@ -204,6 +204,8 @@ configLogicalBackup:
   logical_backup_s3_access_key_id: ""
   # S3 bucket to store backup results
   logical_backup_s3_bucket: "my-bucket-url"
+  # S3 region of bucket
+  logical_backup_s3_region: ""
   # S3 endpoint url when not using AWS
   logical_backup_s3_endpoint: ""
   # S3 Secret Access Key

charts/postgres-operator/values.yaml

@@ -195,6 +195,8 @@ configLogicalBackup:
   logical_backup_s3_access_key_id: ""
   # S3 bucket to store backup results
   logical_backup_s3_bucket: "my-bucket-url"
+  # S3 region of bucket
+  logical_backup_s3_region: ""
   # S3 endpoint url when not using AWS
   logical_backup_s3_endpoint: ""
   # S3 Secret Access Key

docker/logical-backup/dump.sh

@@ -40,6 +40,7 @@ function aws_upload {
 
     [[ ! -z "$EXPECTED_SIZE" ]] && args+=("--expected-size=$EXPECTED_SIZE")
     [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT")
+    [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION")
     [[ ! -z "$LOGICAL_BACKUP_S3_SSE" ]] && args+=("--sse=$LOGICAL_BACKUP_S3_SSE")
 
     aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}"

docs/reference/operator_parameters.md

@@ -461,8 +461,11 @@ grouped under the `logical_backup` key.
   S3 bucket to store backup results. The bucket has to be present and
   accessible by Postgres pods. Default: empty.
 
+* **logical_backup_s3_region**
+  Specifies the region of the bucket which is required with some non-AWS S3 storage services. The default is empty.
+
 * **logical_backup_s3_endpoint**
-  When using non-AWS S3 storage, endpoint can be set as a ENV variable.
+  When using non-AWS S3 storage, endpoint can be set as a ENV variable. The default is empty.
 
 * **logical_backup_s3_sse**
   Specify server side encription that S3 storage is using. If empty string

manifests/configmap.yaml

@@ -40,6 +40,7 @@ data:
   # logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
   # logical_backup_s3_access_key_id: ""
   # logical_backup_s3_bucket: "my-bucket-url"
+  # logical_backup_s3_region: ""
   # logical_backup_s3_endpoint: ""
   # logical_backup_s3_secret_access_key: ""
   # logical_backup_s3_sse: "AES256"

manifests/operatorconfiguration.crd.yaml

@@ -219,6 +219,8 @@ spec:
                   type: string
                 logical_backup_s3_endpoint:
                   type: string
+                logical_backup_s3_region:
+                  type: string
                 logical_backup_s3_secret_access_key:
                   type: string
                 logical_backup_s3_sse:

manifests/postgresql-operator-default-configuration.yaml

@@ -88,6 +88,7 @@ configuration:
     # logical_backup_s3_access_key_id: ""
     logical_backup_s3_bucket: "my-bucket-url"
     # logical_backup_s3_endpoint: ""
+    # logical_backup_s3_region: ""
     # logical_backup_s3_secret_access_key: ""
     logical_backup_s3_sse: "AES256"
     logical_backup_schedule: "30 00 * * *"

pkg/apis/acid.zalan.do/v1/crds.go

@@ -909,6 +909,9 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation
 							"logical_backup_s3_endpoint": {
 								Type: "string",
 							},
+							"logical_backup_s3_region": {
+								Type: "string",
+							},
 							"logical_backup_s3_secret_access_key": {
 								Type: "string",
 							},

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -157,6 +157,7 @@ type OperatorLogicalBackupConfiguration struct {
 	Schedule          string `json:"logical_backup_schedule,omitempty"`
 	DockerImage       string `json:"logical_backup_docker_image,omitempty"`
 	S3Bucket          string `json:"logical_backup_s3_bucket,omitempty"`
+	S3Region          string `json:"logical_backup_s3_region,omitempty"`
 	S3Endpoint        string `json:"logical_backup_s3_endpoint,omitempty"`
 	S3AccessKeyID     string `json:"logical_backup_s3_access_key_id,omitempty"`
 	S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"`

pkg/cluster/k8sres.go

@@ -1589,6 +1589,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
 			Name:  "LOGICAL_BACKUP_S3_BUCKET",
 			Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket,
 		},
+		{
+			Name:  "LOGICAL_BACKUP_S3_REGION",
+			Value: c.OpConfig.LogicalBackup.LogicalBackupS3Region,
+		},
 		{
 			Name:  "LOGICAL_BACKUP_S3_ENDPOINT",
 			Value: c.OpConfig.LogicalBackup.LogicalBackupS3Endpoint,

pkg/controller/operator_config.go

@@ -106,6 +106,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.LogicalBackupSchedule = fromCRD.LogicalBackup.Schedule
 	result.LogicalBackupDockerImage = fromCRD.LogicalBackup.DockerImage
 	result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket
+	result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region
 	result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint
 	result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID
 	result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey

pkg/util/config/config.go

@@ -76,6 +76,7 @@ type LogicalBackup struct {
 	LogicalBackupSchedule          string `name:"logical_backup_schedule" default:"30 00 * * *"`
 	LogicalBackupDockerImage       string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"`
 	LogicalBackupS3Bucket          string `name:"logical_backup_s3_bucket" default:""`
+	LogicalBackupS3Region          string `name:"logical_backup_s3_region" default:""`
 	LogicalBackupS3Endpoint        string `name:"logical_backup_s3_endpoint" default:""`
 	LogicalBackupS3AccessKeyID     string `name:"logical_backup_s3_access_key_id" default:""`
 	LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""`