diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index c903a9319..ef9dff337 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -553,6 +553,9 @@ spec: default: "30 00 * * *" logical_backup_cronjob_environment_secret: type: string + logical_backup_filename_date_format: + type: string + default: "+%s" debug: type: object properties: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 4e5d9b7cb..bcef80fe0 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -395,6 +395,8 @@ configLogicalBackup: logical_backup_schedule: "30 00 * * *" # secret to be used as reference for env variables in cronjob logical_backup_cronjob_environment_secret: "" + # backup filename date format + logical_backup_filename_date_format: "" # automate creation of human users with teams API service configTeamsApi: diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 4327dc45f..543d4da4f 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -889,6 +889,9 @@ grouped under the `logical_backup` key. * **logical_backup_cronjob_environment_secret** Reference to a Kubernetes secret, which keys will be added as environment variables to the cronjob. Default: "" +* **logical_backup_filename_date_format** + Date format to use for the logical backup filename. Uses date linux utility. Default: "+%s" + ## Debugging the operator Options to aid debugging of the operator itself. Grouped under the `debug` key. diff --git a/logical-backup/dump.sh b/logical-backup/dump.sh index a250670a6..01038bf16 100755 --- a/logical-backup/dump.sh +++ b/logical-backup/dump.sh @@ -10,6 +10,7 @@ ALL_DB_SIZE_QUERY="select sum(pg_database_size(datname)::numeric) from pg_databa PG_BIN=$PG_DIR/$PG_VERSION/bin DUMP_SIZE_COEFF=5 ERRORCOUNT=0 +TIMESTAMP=$(eval date $LOGICAL_BACKUP_FILENAME_DATE_FORMAT) TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) KUBERNETES_SERVICE_PORT=${KUBERNETES_SERVICE_PORT:-443} @@ -45,7 +46,7 @@ function compress { } function az_upload { - PATH_TO_BACKUP=$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz + PATH_TO_BACKUP=$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$TIMESTAMP.sql.gz az storage blob upload --file "$1" --account-name "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME" --account-key "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY" -c "$LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER" -n "$PATH_TO_BACKUP" } @@ -107,7 +108,7 @@ function aws_upload { # mimic bucket setup from Spilo # to keep logical backups at the same path as WAL # NB: $LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX already contains the leading "/" when set by the Postgres Operator - PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz + PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$TIMESTAMP.sql.gz args=() @@ -120,7 +121,7 @@ function aws_upload { } function gcs_upload { - PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz + PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$TIMESTAMP.sql.gz #Set local LOGICAL_GOOGLE_APPLICATION_CREDENTIALS to nothing or #value of LOGICAL_GOOGLE_APPLICATION_CREDENTIALS env var. Needed diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 6d51053bb..9908078c3 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -102,6 +102,7 @@ data: logical_backup_s3_sse: "AES256" logical_backup_s3_retention_time: "" logical_backup_schedule: "30 00 * * *" + logical_backup_filename_date_format: "+%s" major_version_upgrade_mode: "manual" # major_version_upgrade_team_allow_list: "" master_dns_name_format: "{cluster}.{namespace}.{hostedzone}" diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 1ab85c905..985e5b972 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -748,6 +748,9 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "logical_backup_cronjob_environment_secret": { Type: "string", }, + "logical_backup_filename_date_format": { + Type: "string", + }, }, }, "debug": { diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 80cfbbcd7..514ae4c78 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -244,6 +244,7 @@ type OperatorLogicalBackupConfiguration struct { MemoryRequest string `json:"logical_backup_memory_request,omitempty"` CPULimit string `json:"logical_backup_cpu_limit,omitempty"` MemoryLimit string `json:"logical_backup_memory_limit,omitempty"` + FilenameDateFormat string `json:"logical_backup_filename_date_format,omitempty"` } // PatroniConfiguration defines configuration for Patroni diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index d78d4c92e..5eb6950e1 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -1660,6 +1660,7 @@ func TestCompareLogicalBackupJob(t *testing.T) { LogicalBackupS3SSE: "aws:kms", LogicalBackupS3RetentionTime: "3 months", LogicalBackupCronjobEnvironmentSecret: "", + LogicalBackupFilenameDateFormat: "+%s", }, }, }, client, pg, logger, eventRecorder) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 224ef26f6..47fdc5430 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2508,6 +2508,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())), }, + { + Name: "LOGICAL_BACKUP_FILENAME_DATE_FORMAT", + Value: c.OpConfig.LogicalBackup.LogicalBackupFilenameDateFormat, + }, } switch backupProvider { diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index e39e18cd5..034524e84 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -4071,21 +4071,26 @@ func TestGenerateLogicalBackupPodEnvVars(t *testing.T) { }, { envIndex: 13, + envVarConstant: "LOGICAL_BACKUP_FILENAME_DATE_FORMAT", + envVarValue: "+%s", + }, + { + envIndex: 14, envVarConstant: "LOGICAL_BACKUP_S3_REGION", envVarValue: "eu-central-1", }, { - envIndex: 14, + envIndex: 15, envVarConstant: "LOGICAL_BACKUP_S3_ENDPOINT", envVarValue: "", }, { - envIndex: 15, + envIndex: 16, envVarConstant: "LOGICAL_BACKUP_S3_SSE", envVarValue: "", }, { - envIndex: 16, + envIndex: 17, envVarConstant: "LOGICAL_BACKUP_S3_RETENTION_TIME", envVarValue: "1 month", }, @@ -4098,7 +4103,7 @@ func TestGenerateLogicalBackupPodEnvVars(t *testing.T) { envVarValue: "gcs", }, { - envIndex: 13, + envIndex: 14, envVarConstant: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS", envVarValue: "some-path-to-credentials", }, @@ -4111,17 +4116,17 @@ func TestGenerateLogicalBackupPodEnvVars(t *testing.T) { envVarValue: "az", }, { - envIndex: 13, + envIndex: 14, envVarConstant: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME", envVarValue: "some-azure-storage-account-name", }, { - envIndex: 14, + envIndex: 15, envVarConstant: "LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER", envVarValue: "some-azure-storage-container", }, { - envIndex: 15, + envIndex: 16, envVarConstant: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY", envVarValue: "some-azure-storage-account-key", }, @@ -4129,7 +4134,7 @@ func TestGenerateLogicalBackupPodEnvVars(t *testing.T) { expectedLogicalBackupRetentionTime := []ExpectedValue{ { - envIndex: 16, + envIndex: 17, envVarConstant: "LOGICAL_BACKUP_S3_RETENTION_TIME", envVarValue: "3 months", }, @@ -4145,11 +4150,12 @@ func TestGenerateLogicalBackupPodEnvVars(t *testing.T) { subTest: "logical backup with provider: s3", opConfig: config.Config{ LogicalBackup: config.LogicalBackup{ - LogicalBackupProvider: "s3", - LogicalBackupS3Bucket: dummyBucket, - LogicalBackupS3BucketPrefix: "spilo", - LogicalBackupS3Region: "eu-central-1", - LogicalBackupS3RetentionTime: "1 month", + LogicalBackupProvider: "s3", + LogicalBackupS3Bucket: dummyBucket, + LogicalBackupS3BucketPrefix: "spilo", + LogicalBackupFilenameDateFormat: "+%s", + LogicalBackupS3Region: "eu-central-1", + LogicalBackupS3RetentionTime: "1 month", }, }, expectedValues: expectedLogicalBackupS3Bucket, diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 24d4ffcd3..b46a3da80 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -201,6 +201,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.LogicalBackupMemoryRequest = fromCRD.LogicalBackup.MemoryRequest result.LogicalBackupCPULimit = fromCRD.LogicalBackup.CPULimit result.LogicalBackupMemoryLimit = fromCRD.LogicalBackup.MemoryLimit + result.LogicalBackupFilenameDateFormat = fromCRD.LogicalBackup.FilenameDateFormat // debug config result.DebugLogging = fromCRD.OperatorDebug.DebugLogging diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 858a58b8c..ba79f91cd 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -131,6 +131,7 @@ type LogicalBackup struct { LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"ghcr.io/zalando/postgres-operator/logical-backup:v1.15.1"` LogicalBackupProvider string `name:"logical_backup_provider" default:"s3"` + LogicalBackupFilenameDateFormat string `name:"logical_backup_filename_date_format"` LogicalBackupAzureStorageAccountName string `name:"logical_backup_azure_storage_account_name" default:""` LogicalBackupAzureStorageContainer string `name:"logical_backup_azure_storage_container" default:""` LogicalBackupAzureStorageAccountKey string `name:"logical_backup_azure_storage_account_key" default:""`