public_git
/
postgres-operator
mirror of https://github.com/zalando/postgres-operator.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add ability to upload logical backup to gcs (#1173) 

Support logical backup provider/storage S3 and GCS equivalent
This commit is contained in:
Pavel Tumik 2020-12-16 01:41:08 -08:00 committed by GitHub
parent 929075814a
commit fbd04896c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 58 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docker/logical-backup/Dockerfile
View File

@ -13,7 +13,10 @@ RUN apt-get update \
curl \
jq \
gnupg \
gcc \
libffi-dev \
&& pip3 install --no-cache-dir awscli --upgrade \
&& pip3 install --no-cache-dir gsutil --upgrade \
&& echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
&& cat /etc/apt/sources.list.d/pgdg.list \
&& curl --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \

19
docker/logical-backup/dump.sh
View File

@ -46,6 +46,23 @@ function aws_upload {
aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}"
}
function gcs_upload {
PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
gsutil -o Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS cp - "$PATH_TO_BACKUP"
}
function upload {
case $LOGICAL_BACKUP_PROVIDER in
"gcs")
gcs_upload
;;
*)
aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
;;
esac
}
function get_pods {
declare -r SELECTOR="$1"
@ -93,7 +110,7 @@ for search in "${search_strategy[@]}"; do
done
set -x
dump | compress | aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
dump | compress | upload
[[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 ))
set +x

7
docs/reference/operator_parameters.md
View File

@ -563,6 +563,10 @@ grouped under the `logical_backup` key.
The default image is the same image built with the Zalando-internal CI
pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup"
* **logical_backup_provider**
Specifies the storage provider to which the backup should be uploaded (`s3` or `gcs`).
Default: "s3"
* **logical_backup_s3_bucket**
S3 bucket to store backup results. The bucket has to be present and
accessible by Postgres pods. Default: empty.
@ -583,6 +587,9 @@ grouped under the `logical_backup` key.
* **logical_backup_s3_secret_access_key**
When set, value will be in AWS_SECRET_ACCESS_KEY env variable. The Default is empty.
* **logical_backup_google_application_credentials**
Specifies the path of the google cloud service account json file. Default is empty.
## Debugging the operator
Options to aid debugging of the operator itself. Grouped under the `debug` key.

18
pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
View File

@ -186,14 +186,16 @@ type ConnectionPoolerConfiguration struct {
// OperatorLogicalBackupConfiguration defines configuration for logical backup
type OperatorLogicalBackupConfiguration struct {
Schedule string `json:"logical_backup_schedule,omitempty"`
DockerImage string `json:"logical_backup_docker_image,omitempty"`
S3Bucket string `json:"logical_backup_s3_bucket,omitempty"`
S3Region string `json:"logical_backup_s3_region,omitempty"`
S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"`
S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"`
S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"`
S3SSE string `json:"logical_backup_s3_sse,omitempty"`
Schedule string `json:"logical_backup_schedule,omitempty"`
DockerImage string `json:"logical_backup_docker_image,omitempty"`
BackupProvider string `json:"logical_backup_provider,omitempty"`
S3Bucket string `json:"logical_backup_s3_bucket,omitempty"`
S3Region string `json:"logical_backup_s3_region,omitempty"`
S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"`
S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"`
S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"`
S3SSE string `json:"logical_backup_s3_sse,omitempty"`
GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"`
}
// OperatorConfigurationData defines the operation config

8
pkg/cluster/k8sres.go
View File

@ -1988,6 +1988,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
},
},
// Bucket env vars
{
Name: "LOGICAL_BACKUP_PROVIDER",
Value: c.OpConfig.LogicalBackup.LogicalBackupProvider,
},
{
Name: "LOGICAL_BACKUP_S3_BUCKET",
Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket,
@ -2008,6 +2012,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX",
Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())),
},
{
Name: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS",
Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials,
},
// Postgres env vars
{
Name: "PG_VERSION",

2
pkg/controller/operator_config.go
View File

@ -146,12 +146,14 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
// logical backup config
result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *")
result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup")
result.LogicalBackupProvider = util.Coalesce(fromCRD.LogicalBackup.BackupProvider, "s3")
result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket
result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region
result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint
result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID
result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey
result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE
result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials
// debug config
result.DebugLogging = fromCRD.OperatorDebug.DebugLogging

18
pkg/util/config/config.go
View File

@ -111,14 +111,16 @@ type Scalyr struct {
// LogicalBackup defines configuration for logical backup
type LogicalBackup struct {
LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"`
LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"`
LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""`
LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""`
LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""`
LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""`
LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""`
LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""`
LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"`
LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"`
LogicalBackupProvider string `name:"logical_backup_provider" default:"s3"`
LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""`
LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""`
LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""`
LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""`
LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""`
LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""`
LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""`
}
// Operator options for connection pooler