diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index f510e08f5..f67c04fc5 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -450,6 +450,8 @@ spec: type: string logical_backup_s3_sse: type: string + logical_backup_s3_retention_time: + type: string logical_backup_schedule: type: string pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 288efe763..5d38b5233 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -310,6 +310,8 @@ configLogicalBackup: logical_backup_s3_secret_access_key: "" # S3 server side encryption logical_backup_s3_sse: "AES256" + # S3 retention time for stored backups for example "2 week" or "7 days" + logical_backup_s3_retention_time: "" # backup schedule in the cron format logical_backup_schedule: "30 00 * * *" diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh index c931dc962..85e372664 100755 --- a/docker/logical-backup/dump.sh +++ b/docker/logical-backup/dump.sh @@ -15,6 +15,9 @@ TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1 CERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt +LOGICAL_BACKUP_PROVIDER=${LOGICAL_BACKUP_PROVIDER:="s3"} +LOGICAL_BACKUP_S3_RETENTION_TIME=${LOGICAL_BACKUP_S3_RETENTION_TIME:=""} + function estimate_size { "$PG_BIN"/psql -tqAc "${ALL_DB_SIZE_QUERY}" } @@ -28,6 +31,57 @@ function compress { pigz } +function aws_delete_objects { + args=( + "--bucket=$LOGICAL_BACKUP_S3_BUCKET" + ) + + [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT") + [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION") + + aws s3api delete-objects "${args[@]}" --delete Objects=["$(printf {Key=%q}, "$@")"],Quiet=true +} +export -f aws_delete_objects + +function aws_delete_outdated { + if [[ -z "$LOGICAL_BACKUP_S3_RETENTION_TIME" ]] ; then + echo "no retention time configured: skip cleanup of outdated backups" + return 0 + fi + + # define cutoff date for outdated backups (day precision) + cutoff_date=$(date -d "$LOGICAL_BACKUP_S3_RETENTION_TIME ago" +%F) + + # mimic bucket setup from Spilo + prefix="spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/" + + args=( + "--no-paginate" + "--output=text" + "--prefix=$prefix" + "--bucket=$LOGICAL_BACKUP_S3_BUCKET" + ) + + [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT") + [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION") + + # list objects older than the cutoff date + aws s3api list-objects "${args[@]}" --query="Contents[?LastModified<='$cutoff_date'].[Key]" > /tmp/outdated-backups + + # spare the last backup + sed -i '$d' /tmp/outdated-backups + + count=$(wc -l < /tmp/outdated-backups) + if [[ $count == 0 ]] ; then + echo "no outdated backups to delete" + return 0 + fi + echo "deleting $count outdated backups created before $cutoff_date" + + # deleted outdated files in batches with 100 at a time + tr '\n' '\0' < /tmp/outdated-backups | xargs -0 -P1 -n100 bash -c 'aws_delete_objects "$@"' _ +} + function aws_upload { declare -r EXPECTED_SIZE="$1" @@ -59,6 +113,7 @@ function upload { ;; *) aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF)) + aws_delete_outdated ;; esac } diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index f3d9be88f..cbe31ba57 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -676,6 +676,11 @@ grouped under the `logical_backup` key. Specify server side encryption that S3 storage is using. If empty string is specified, no argument will be passed to `aws s3` command. Default: "AES256". +* **logical_backup_s3_retention_time** + Specify a retention time for logical backups stored in S3. Backups older than the specified retention + time will be deleted after a new backup was uploaded. If empty, all backups will be kept. Example values are + "3 days", "2 weeks", or "1 month". The default is empty. + * **logical_backup_schedule** Backup schedule in the cron format. Please take the [reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule) diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index b3aaa3c66..b9605441e 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -77,6 +77,7 @@ data: # logical_backup_s3_endpoint: "" # logical_backup_s3_secret_access_key: "" logical_backup_s3_sse: "AES256" + # logical_backup_s3_retention_time: "" logical_backup_schedule: "30 00 * * *" major_version_upgrade_mode: "manual" # major_version_upgrade_team_allow_list: "" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index d086998cf..e6c2a2d7c 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -448,6 +448,8 @@ spec: type: string logical_backup_s3_sse: type: string + logical_backup_s3_retention_time: + type: string logical_backup_schedule: type: string pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 87b5436d5..0641bd1b1 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -146,6 +146,7 @@ configuration: # logical_backup_s3_region: "" # logical_backup_s3_secret_access_key: "" logical_backup_s3_sse: "AES256" + # logical_backup_s3_retention_time: "" logical_backup_schedule: "30 00 * * *" debug: debug_logging: true diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 9dc3d167e..d8d46e248 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1556,6 +1556,9 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "logical_backup_s3_sse": { Type: "string", }, + "logical_backup_s3_retention_time": { + Type: "string", + }, "logical_backup_schedule": { Type: "string", Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$", diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 1298c6834..00fce42b5 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -213,6 +213,7 @@ type OperatorLogicalBackupConfiguration struct { S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"` S3SSE string `json:"logical_backup_s3_sse,omitempty"` + RetentionTime string `json:"logical_backup_s3_retention_time,omitempty"` GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"` JobPrefix string `json:"logical_backup_job_prefix,omitempty"` } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index e741c6dc4..91dd20765 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2134,6 +2134,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { Name: "LOGICAL_BACKUP_S3_SSE", Value: c.OpConfig.LogicalBackup.LogicalBackupS3SSE, }, + { + Name: "LOGICAL_BACKUP_S3_RETENTION_TIME", + Value: c.OpConfig.LogicalBackup.LogicalBackupS3RetentionTime, + }, { Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())), diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index fbf12bfb9..543254242 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -170,6 +170,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE + result.LogicalBackupS3RetentionTime = fromCRD.LogicalBackup.RetentionTime result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials result.LogicalBackupJobPrefix = util.Coalesce(fromCRD.LogicalBackup.JobPrefix, "logical-backup-") diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 0dc1004a7..f32762e46 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -128,6 +128,7 @@ type LogicalBackup struct { LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""` + LogicalBackupS3RetentionTime string `name:"logical_backup_s3_retention_time" default:""` LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""` LogicalBackupJobPrefix string `name:"logical_backup_job_prefix" default:"logical-backup-"` }