public_git
/
postgres-operator
mirror of https://github.com/zalando/postgres-operator.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add logical backup for azure (#2052) 

* Add logical backup for azure
This commit is contained in:
Stef Graces 2023-01-05 12:16:41 +01:00 committed by GitHub
parent c756cb2f8a
commit bb2617a53f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 74 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
charts/postgres-operator/crds/operatorconfigurations.yaml
View File

@ -498,6 +498,12 @@ spec:
type: string
pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$'
default: "30 00 * * *"
logical_backup_azure_storage_account_name:
type: string
logical_backup_azure_storage_container:
type: string
logical_backup_azure_storage_account_key:
type: string
debug:
type: object
properties:

1
docker/logical-backup/Dockerfile
View File

@ -15,6 +15,7 @@ RUN apt-get update \
gnupg \
gcc \
libffi-dev \
&& curl -sL https://aka.ms/InstallAzureCLIDeb | bash \
&& pip3 install --upgrade pip \
&& pip3 install --no-cache-dir awscli --upgrade \
&& pip3 install --no-cache-dir gsutil --upgrade \

21
docker/logical-backup/dump.sh
View File

@ -40,6 +40,12 @@ function compress {
pigz
}
function az_upload {
PATH_TO_BACKUP=$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
az storage blob upload --file "$1" --account-name "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME" --account-key "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY" -c "$LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER" -n "$PATH_TO_BACKUP"
}
function aws_delete_objects {
args=(
"--bucket=$LOGICAL_BACKUP_S3_BUCKET"
@ -120,7 +126,7 @@ function upload {
"gcs")
gcs_upload
;;
*)
"aws")
aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
aws_delete_outdated
;;
@ -174,8 +180,13 @@ for search in "${search_strategy[@]}"; do
done
set -x
dump | compress | upload
[[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 ))
set +x
if [ "$LOGICAL_BACKUP_PROVIDER" == "az" ]; then
dump | compress > /tmp/azure-backup.sql.gz
az_upload /tmp/azure-backup.sql.gz
else
dump | compress | upload
[[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 ))
set +x
exit $ERRORCOUNT
exit $ERRORCOUNT
fi

9
docs/reference/operator_parameters.md
View File

@ -765,6 +765,15 @@ grouped under the `logical_backup` key.
[reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule)
into account. Default: "30 00 \* \* \*"
* **logical_backup_azure_storage_account_name**
Storage account name used to upload logical backups to when using Azure. Default: ""
* **logical_backup_azure_storage_container**
Storage container used to upload logical backups to when using Azure. Default: ""
* **logical_backup_azure_storage_account_key**
Storage account key used to authenticate with Azure when uploading logical backups. Default: ""
## Debugging the operator
Options to aid debugging of the operator itself. Grouped under the `debug` key.

3
manifests/configmap.yaml
View File

@ -87,6 +87,9 @@ data:
logical_backup_s3_sse: "AES256"
# logical_backup_s3_retention_time: ""
logical_backup_schedule: "30 00 * * *"
# logical_backup_azure_storage_account_name: ""
# logical_backup_azure_storage_container: ""
# logical_backup_azure_storage_account_key: ""
major_version_upgrade_mode: "manual"
# major_version_upgrade_team_allow_list: ""
master_dns_name_format: "{cluster}.{namespace}.{hostedzone}"

6
manifests/operatorconfiguration.crd.yaml
View File

@ -496,6 +496,12 @@ spec:
type: string
pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$'
default: "30 00 * * *"
logical_backup_azure_storage_account_name:
type: string
logical_backup_azure_storage_container:
type: string
logical_backup_azure_storage_account_key:
type: string
debug:
type: object
properties:

3
manifests/postgresql-operator-default-configuration.yaml
View File

@ -163,6 +163,9 @@ configuration:
logical_backup_s3_sse: "AES256"
# logical_backup_s3_retention_time: ""
logical_backup_schedule: "30 00 * * *"
# logical_backup_azure_storage_account_name: ""
# logical_backup_azure_storage_container: ""
# logical_backup_azure_storage_account_key: ""
debug:
debug_logging: true
enable_database_access: true

9
pkg/apis/acid.zalan.do/v1/crds.go
View File

@ -1655,6 +1655,15 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
Type: "string",
Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$",
},
"logical_backup_azure_storage_account_name": {
Type: "string",
},
"logical_backup_azure_storage_container": {
Type: "string",
},
"logical_backup_azure_storage_account_key": {
Type: "string",
},
},
},
"debug": {

3
pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
View File

@ -227,6 +227,9 @@ type OperatorLogicalBackupConfiguration struct {
RetentionTime string `json:"logical_backup_s3_retention_time,omitempty"`
GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"`
JobPrefix string `json:"logical_backup_job_prefix,omitempty"`
AzureStorageAccountName string `json:"logical_backup_azure_storage_account_name,omitempty"`
AzureStorageContainer string `json:"logical_backup_azure_storage_container,omitempty"`
AzureStorageAccountKey string `json:"logical_backup_azure_storage_account_key,omitempty"`
}
// PatroniConfiguration defines configuration for Patroni

12
pkg/cluster/k8sres.go
View File

@ -2262,6 +2262,18 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
Name: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS",
Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials,
},
{
Name: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME",
Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountName,
},
{
Name: "LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER",
Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageContainer,
},
{
Name: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY",
Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountKey,
},
// Postgres env vars
{
Name: "PG_VERSION",

3
pkg/controller/operator_config.go
View File

@ -183,6 +183,9 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
result.LogicalBackupS3RetentionTime = fromCRD.LogicalBackup.RetentionTime
result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials
result.LogicalBackupJobPrefix = util.Coalesce(fromCRD.LogicalBackup.JobPrefix, "logical-backup-")
result.LogicalBackupAzureStorageAccountName = fromCRD.LogicalBackup.AzureStorageAccountName
result.LogicalBackupAzureStorageAccountKey = fromCRD.LogicalBackup.AzureStorageAccountKey
result.LogicalBackupAzureStorageContainer = fromCRD.LogicalBackup.AzureStorageContainer
// debug config
result.DebugLogging = fromCRD.OperatorDebug.DebugLogging

3
pkg/util/config/config.go
View File

@ -137,6 +137,9 @@ type LogicalBackup struct {
LogicalBackupS3RetentionTime string `name:"logical_backup_s3_retention_time" default:""`
LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""`
LogicalBackupJobPrefix string `name:"logical_backup_job_prefix" default:"logical-backup-"`
LogicalBackupAzureStorageAccountName string `name:"logical_backup_azure_storage_account_name" default:""`
LogicalBackupAzureStorageContainer string `name:"logical_backup_azure_storage_container" default:""`
LogicalBackupAzureStorageAccountKey string `name:"logical_backup_azure_storage_account_key" default:""`
}
// Operator options for connection pooler