diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 06a5b483f..af05ae56c 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -184,12 +184,20 @@ configAwsOrGcp: # configure K8s cron job managed by the operator configLogicalBackup: - # backup schedule in the cron format - logical_backup_schedule: "30 00 * * *" # image for pods of the logical backup job (example runs pg_dumpall) logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + # S3 Access Key ID + logical_backup_s3_access_key_id: "" # S3 bucket to store backup results logical_backup_s3_bucket: "my-bucket-url" + # S3 endpoint url when not using AWS + logical_backup_s3_endpoint: "" + # S3 Secret Access Key + logical_backup_s3_secret_access_key: "" + # S3 server side encription + logical_backup_s3_sse: "AES256" + # backup schedule in the cron format + logical_backup_schedule: "30 00 * * *" # automate creation of human users with teams API service configTeamsApi: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 2c499a036..b572b5844 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -178,12 +178,20 @@ configAwsOrGcp: # configure K8s cron job managed by the operator configLogicalBackup: - # backup schedule in the cron format - logical_backup_schedule: "30 00 * * *" # image for pods of the logical backup job (example runs pg_dumpall) logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + # S3 Access Key ID + logical_backup_s3_access_key_id: "" # S3 bucket to store backup results logical_backup_s3_bucket: "my-bucket-url" + # S3 endpoint url when not using AWS + logical_backup_s3_endpoint: "" + # S3 Secret Access Key + logical_backup_s3_secret_access_key: "" + # S3 server side encription + logical_backup_s3_sse: "AES256" + # backup schedule in the cron format + logical_backup_schedule: "30 00 * * *" # automate creation of human users with teams API service configTeamsApi: diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh index ee64bd811..5c2e478a9 100755 --- a/docker/logical-backup/dump.sh +++ b/docker/logical-backup/dump.sh @@ -38,11 +38,13 @@ function aws_upload { # NB: $LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX already contains the leading "/" when set by the Postgres Operator PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz - if [ -z "$EXPECTED_SIZE" ]; then - aws s3 cp - "$PATH_TO_BACKUP" --debug --sse="AES256" - else - aws s3 cp - "$PATH_TO_BACKUP" --debug --expected-size "$EXPECTED_SIZE" --sse="AES256" - fi; + args=() + + [[ ! -z "$EXPECTED_SIZE" ]] && args+=("--expected-size=$EXPECTED_SIZE") + [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=\"$LOGICAL_BACKUP_S3_ENDPOINT\"") + [[ ! "$LOGICAL_BACKUP_S3_SSE" == "" ]] && args+=("--sse=\"$LOGICAL_BACKUP_S3_SSE\"") + + aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}" --debug } function get_pods { diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 9e43919fd..a024aad1f 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -441,6 +441,19 @@ grouped under the `logical_backup` key. S3 bucket to store backup results. The bucket has to be present and accessible by Postgres pods. Default: empty. +* **logical_backup_s3_endpoint** + When using non-AWS S3 storage, endpoint can be set as a ENV variable. + +* **logical_backup_s3_sse** + Specify server side encription that S3 storage is using. If empty string + is specified, no argument will be passed to `aws s3` command. Default: "AES256". + +* **logical_backup_s3_access_key_id** + When set, value will be in AWS_ACCESS_KEY_ID env variable. The Default is empty. + +* **logical_backup_s3_secret_access_key** + When set, value will be in AWS_SECRET_ACCESS_KEY env variable. The Default is empty. + ## Debugging the operator Options to aid debugging of the operator itself. Grouped under the `debug` key. diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index c64e45cbd..40aca9716 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -35,7 +35,11 @@ data: # kube_iam_role: "" # log_s3_bucket: "" # logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + # logical_backup_s3_access_key_id: "" # logical_backup_s3_bucket: "my-bucket-url" + # logical_backup_s3_endpoint: "" + # logical_backup_s3_secret_access_key: "" + # logical_backup_s3_sse: "AES256" # logical_backup_schedule: "30 00 * * *" master_dns_name_format: "{cluster}.{team}.staging.{hostedzone}" # master_pod_move_timeout: 10m diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index e1ecd1038..94f91f1f0 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -74,9 +74,14 @@ configuration: # log_s3_bucket: "" # wal_s3_bucket: "" logical_backup: - logical_backup_schedule: "30 00 * * *" + log_s3_bucket: "" logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup" + logical_backup_s3_access_key_id: "" logical_backup_s3_bucket: "my-bucket-url" + logical_backup_s3_endpoint: "" + logical_backup_s3_secret_access_key: "" + logical_backup_s3_sse: "AES256" + logical_backup_schedule: "30 00 * * *" debug: debug_logging: true enable_database_access: true diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 6adfb778e..d00d40532 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -184,8 +184,13 @@ type OperatorConfigurationUsers struct { //Duration shortens this frequently used name type Duration time.Duration +// OperatorLogicalBackupConfiguration defines configration for logical backup type OperatorLogicalBackupConfiguration struct { - Schedule string `json:"logical_backup_schedule,omitempty"` - DockerImage string `json:"logical_backup_docker_image,omitempty"` - S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` + Schedule string `json:"logical_backup_schedule,omitempty"` + DockerImage string `json:"logical_backup_docker_image,omitempty"` + S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` + S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"` + S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` + S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"` + S3SSE string `json:"logical_backup_s3_sse,omitempty"` } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index d5cc4ce39..95e83454f 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1575,6 +1575,14 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { Name: "LOGICAL_BACKUP_S3_BUCKET", Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket, }, + { + Name: "LOGICAL_BACKUP_S3_ENDPOINT", + Value: c.OpConfig.LogicalBackup.LogicalBackupS3Endpoint, + }, + { + Name: "LOGICAL_BACKUP_S3_SSE", + Value: c.OpConfig.LogicalBackup.LogicalBackupS3SSE, + }, { Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())), @@ -1613,8 +1621,15 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { }, } - c.logger.Debugf("Generated logical backup env vars %v", envVars) + if c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID != "" { + envVars = append(envVars, v1.EnvVar{Name: "AWS_ACCESS_KEY_ID", Value: c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID}) + } + if c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey != "" { + envVars = append(envVars, v1.EnvVar{Name: "AWS_SECRET_ACCESS_KEY", Value: c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey}) + } + + c.logger.Debugf("Generated logical backup env vars %v", envVars) return envVars } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index b91fd511f..3ea513879 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -101,6 +101,10 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.LogicalBackupSchedule = fromCRD.LogicalBackup.Schedule result.LogicalBackupDockerImage = fromCRD.LogicalBackup.DockerImage result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket + result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint + result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID + result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey + result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE // debug config result.DebugLogging = fromCRD.OperatorDebug.DebugLogging diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index bdbbf0fdc..e8de85e20 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -68,11 +68,15 @@ type Scalyr struct { ScalyrMemoryLimit string `name:"scalyr_memory_limit" default:"1Gi"` } -// LogicalBackup +// LogicalBackup defines configration for logical backup type LogicalBackup struct { - LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` - LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"` - LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` + LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` + LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"` + LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` + LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` + LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` + LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` + LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:"AES256"` } // Config describes operator config