From bb2617a53f10fcc0fc38f621ed6853e148f39c3a Mon Sep 17 00:00:00 2001 From: Stef Graces Date: Thu, 5 Jan 2023 12:16:41 +0100 Subject: [PATCH] Add logical backup for azure (#2052) * Add logical backup for azure --- .../crds/operatorconfigurations.yaml | 6 ++++++ docker/logical-backup/Dockerfile | 1 + docker/logical-backup/dump.sh | 21 ++++++++++++++----- docs/reference/operator_parameters.md | 9 ++++++++ manifests/configmap.yaml | 3 +++ manifests/operatorconfiguration.crd.yaml | 6 ++++++ ...gresql-operator-default-configuration.yaml | 3 +++ pkg/apis/acid.zalan.do/v1/crds.go | 9 ++++++++ .../v1/operator_configuration_type.go | 3 +++ pkg/cluster/k8sres.go | 12 +++++++++++ pkg/controller/operator_config.go | 3 +++ pkg/util/config/config.go | 3 +++ 12 files changed, 74 insertions(+), 5 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index aaadce0c3..b36c58b74 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -498,6 +498,12 @@ spec: type: string pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' default: "30 00 * * *" + logical_backup_azure_storage_account_name: + type: string + logical_backup_azure_storage_container: + type: string + logical_backup_azure_storage_account_key: + type: string debug: type: object properties: diff --git a/docker/logical-backup/Dockerfile b/docker/logical-backup/Dockerfile index 184e3be1c..08553fc5f 100644 --- a/docker/logical-backup/Dockerfile +++ b/docker/logical-backup/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update \ gnupg \ gcc \ libffi-dev \ + && curl -sL https://aka.ms/InstallAzureCLIDeb | bash \ && pip3 install --upgrade pip \ && pip3 install --no-cache-dir awscli --upgrade \ && pip3 install --no-cache-dir gsutil --upgrade \ diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh index 2627ac8c9..0aa723ab5 100755 --- a/docker/logical-backup/dump.sh +++ b/docker/logical-backup/dump.sh @@ -40,6 +40,12 @@ function compress { pigz } +function az_upload { + PATH_TO_BACKUP=$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz + + az storage blob upload --file "$1" --account-name "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME" --account-key "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY" -c "$LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER" -n "$PATH_TO_BACKUP" +} + function aws_delete_objects { args=( "--bucket=$LOGICAL_BACKUP_S3_BUCKET" @@ -120,7 +126,7 @@ function upload { "gcs") gcs_upload ;; - *) + "aws") aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF)) aws_delete_outdated ;; @@ -174,8 +180,13 @@ for search in "${search_strategy[@]}"; do done set -x -dump | compress | upload -[[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 )) -set +x +if [ "$LOGICAL_BACKUP_PROVIDER" == "az" ]; then + dump | compress > /tmp/azure-backup.sql.gz + az_upload /tmp/azure-backup.sql.gz +else + dump | compress | upload + [[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 )) + set +x -exit $ERRORCOUNT + exit $ERRORCOUNT +fi diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index ca5d7b055..dd80ce569 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -765,6 +765,15 @@ grouped under the `logical_backup` key. [reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule) into account. Default: "30 00 \* \* \*" +* **logical_backup_azure_storage_account_name** + Storage account name used to upload logical backups to when using Azure. Default: "" + +* **logical_backup_azure_storage_container** + Storage container used to upload logical backups to when using Azure. Default: "" + +* **logical_backup_azure_storage_account_key** + Storage account key used to authenticate with Azure when uploading logical backups. Default: "" + ## Debugging the operator Options to aid debugging of the operator itself. Grouped under the `debug` key. diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 9485331de..15816c693 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -87,6 +87,9 @@ data: logical_backup_s3_sse: "AES256" # logical_backup_s3_retention_time: "" logical_backup_schedule: "30 00 * * *" + # logical_backup_azure_storage_account_name: "" + # logical_backup_azure_storage_container: "" + # logical_backup_azure_storage_account_key: "" major_version_upgrade_mode: "manual" # major_version_upgrade_team_allow_list: "" master_dns_name_format: "{cluster}.{namespace}.{hostedzone}" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 2f10f7be9..df6343852 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -496,6 +496,12 @@ spec: type: string pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' default: "30 00 * * *" + logical_backup_azure_storage_account_name: + type: string + logical_backup_azure_storage_container: + type: string + logical_backup_azure_storage_account_key: + type: string debug: type: object properties: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 6939ec4d7..36d009bbf 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -163,6 +163,9 @@ configuration: logical_backup_s3_sse: "AES256" # logical_backup_s3_retention_time: "" logical_backup_schedule: "30 00 * * *" + # logical_backup_azure_storage_account_name: "" + # logical_backup_azure_storage_container: "" + # logical_backup_azure_storage_account_key: "" debug: debug_logging: true enable_database_access: true diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 40a8b380e..e0cb148eb 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1655,6 +1655,15 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "string", Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$", }, + "logical_backup_azure_storage_account_name": { + Type: "string", + }, + "logical_backup_azure_storage_container": { + Type: "string", + }, + "logical_backup_azure_storage_account_key": { + Type: "string", + }, }, }, "debug": { diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index d21cbe48b..c57dff0cb 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -227,6 +227,9 @@ type OperatorLogicalBackupConfiguration struct { RetentionTime string `json:"logical_backup_s3_retention_time,omitempty"` GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"` JobPrefix string `json:"logical_backup_job_prefix,omitempty"` + AzureStorageAccountName string `json:"logical_backup_azure_storage_account_name,omitempty"` + AzureStorageContainer string `json:"logical_backup_azure_storage_container,omitempty"` + AzureStorageAccountKey string `json:"logical_backup_azure_storage_account_key,omitempty"` } // PatroniConfiguration defines configuration for Patroni diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 451a189fd..963814941 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2262,6 +2262,18 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { Name: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials, }, + { + Name: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME", + Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountName, + }, + { + Name: "LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER", + Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageContainer, + }, + { + Name: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY", + Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountKey, + }, // Postgres env vars { Name: "PG_VERSION", diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 40d639b20..fc4aba73e 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -183,6 +183,9 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.LogicalBackupS3RetentionTime = fromCRD.LogicalBackup.RetentionTime result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials result.LogicalBackupJobPrefix = util.Coalesce(fromCRD.LogicalBackup.JobPrefix, "logical-backup-") + result.LogicalBackupAzureStorageAccountName = fromCRD.LogicalBackup.AzureStorageAccountName + result.LogicalBackupAzureStorageAccountKey = fromCRD.LogicalBackup.AzureStorageAccountKey + result.LogicalBackupAzureStorageContainer = fromCRD.LogicalBackup.AzureStorageContainer // debug config result.DebugLogging = fromCRD.OperatorDebug.DebugLogging diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 4afc30e1a..30e557447 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -137,6 +137,9 @@ type LogicalBackup struct { LogicalBackupS3RetentionTime string `name:"logical_backup_s3_retention_time" default:""` LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""` LogicalBackupJobPrefix string `name:"logical_backup_job_prefix" default:"logical-backup-"` + LogicalBackupAzureStorageAccountName string `name:"logical_backup_azure_storage_account_name" default:""` + LogicalBackupAzureStorageContainer string `name:"logical_backup_azure_storage_container" default:""` + LogicalBackupAzureStorageAccountKey string `name:"logical_backup_azure_storage_account_key" default:""` } // Operator options for connection pooler