public_git
/
postgres-operator
mirror of https://github.com/zalando/postgres-operator.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Passing endpoint, access and secret key to logical-backup container (#628) 

* Added possibility to add custom annotations to LoadBalancer service.

* Added parameters for custom endpoint, access and secret key for logical backup.

* Modified dump.sh so it knows how to handle new features. Configurable S3 SSE
This commit is contained in:
Armin Nesiren 2019-11-26 10:40:49 +01:00 committed by Sergey Dudoladov
parent 2ce602fcd7
commit 5f87384d7f
10 changed files with 86 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
charts/postgres-operator/values-crd.yaml
View File

@ -184,12 +184,20 @@ configAwsOrGcp:
# configure K8s cron job managed by the operator
configLogicalBackup:
# backup schedule in the cron format
logical_backup_schedule: "30 00 * * *"
# image for pods of the logical backup job (example runs pg_dumpall)
logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
# S3 Access Key ID
logical_backup_s3_access_key_id: ""
# S3 bucket to store backup results
logical_backup_s3_bucket: "my-bucket-url"
# S3 endpoint url when not using AWS
logical_backup_s3_endpoint: ""
# S3 Secret Access Key
logical_backup_s3_secret_access_key: ""
# S3 server side encription
logical_backup_s3_sse: "AES256"
# backup schedule in the cron format
logical_backup_schedule: "30 00 * * *"
# automate creation of human users with teams API service
configTeamsApi:

12
charts/postgres-operator/values.yaml
View File

@ -178,12 +178,20 @@ configAwsOrGcp:
# configure K8s cron job managed by the operator
configLogicalBackup:
# backup schedule in the cron format
logical_backup_schedule: "30 00 * * *"
# image for pods of the logical backup job (example runs pg_dumpall)
logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
# S3 Access Key ID
logical_backup_s3_access_key_id: ""
# S3 bucket to store backup results
logical_backup_s3_bucket: "my-bucket-url"
# S3 endpoint url when not using AWS
logical_backup_s3_endpoint: ""
# S3 Secret Access Key
logical_backup_s3_secret_access_key: ""
# S3 server side encription
logical_backup_s3_sse: "AES256"
# backup schedule in the cron format
logical_backup_schedule: "30 00 * * *"
# automate creation of human users with teams API service
configTeamsApi:

12
docker/logical-backup/dump.sh
View File

@ -38,11 +38,13 @@ function aws_upload {
# NB: $LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX already contains the leading "/" when set by the Postgres Operator
PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
if [ -z "$EXPECTED_SIZE" ]; then
aws s3 cp - "$PATH_TO_BACKUP" --debug --sse="AES256"
else
aws s3 cp - "$PATH_TO_BACKUP" --debug --expected-size "$EXPECTED_SIZE" --sse="AES256"
fi;
args=()
[[ ! -z "$EXPECTED_SIZE" ]] && args+=("--expected-size=$EXPECTED_SIZE")
[[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=\"$LOGICAL_BACKUP_S3_ENDPOINT\"")
[[ ! "$LOGICAL_BACKUP_S3_SSE" == "" ]] && args+=("--sse=\"$LOGICAL_BACKUP_S3_SSE\"")
aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}" --debug
}
function get_pods {

13
docs/reference/operator_parameters.md
View File

@ -441,6 +441,19 @@ grouped under the `logical_backup` key.
S3 bucket to store backup results. The bucket has to be present and
accessible by Postgres pods. Default: empty.
* **logical_backup_s3_endpoint**
When using non-AWS S3 storage, endpoint can be set as a ENV variable.
* **logical_backup_s3_sse**
Specify server side encription that S3 storage is using. If empty string
is specified, no argument will be passed to `aws s3` command. Default: "AES256".
* **logical_backup_s3_access_key_id**
When set, value will be in AWS_ACCESS_KEY_ID env variable. The Default is empty.
* **logical_backup_s3_secret_access_key**
When set, value will be in AWS_SECRET_ACCESS_KEY env variable. The Default is empty.
## Debugging the operator
Options to aid debugging of the operator itself. Grouped under the `debug` key.

4
manifests/configmap.yaml
View File

@ -35,7 +35,11 @@ data:
# kube_iam_role: ""
# log_s3_bucket: ""
# logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
# logical_backup_s3_access_key_id: ""
# logical_backup_s3_bucket: "my-bucket-url"
# logical_backup_s3_endpoint: ""
# logical_backup_s3_secret_access_key: ""
# logical_backup_s3_sse: "AES256"
# logical_backup_schedule: "30 00 * * *"
master_dns_name_format: "{cluster}.{team}.staging.{hostedzone}"
# master_pod_move_timeout: 10m

7
manifests/postgresql-operator-default-configuration.yaml
View File

@ -74,9 +74,14 @@ configuration:
# log_s3_bucket: ""
# wal_s3_bucket: ""
logical_backup:
logical_backup_schedule: "30 00 * * *"
log_s3_bucket: ""
logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
logical_backup_s3_access_key_id: ""
logical_backup_s3_bucket: "my-bucket-url"
logical_backup_s3_endpoint: ""
logical_backup_s3_secret_access_key: ""
logical_backup_s3_sse: "AES256"
logical_backup_schedule: "30 00 * * *"
debug:
debug_logging: true
enable_database_access: true

11
pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
View File

@ -184,8 +184,13 @@ type OperatorConfigurationUsers struct {
//Duration shortens this frequently used name
type Duration time.Duration
// OperatorLogicalBackupConfiguration defines configration for logical backup
type OperatorLogicalBackupConfiguration struct {
Schedule string `json:"logical_backup_schedule,omitempty"`
DockerImage string `json:"logical_backup_docker_image,omitempty"`
S3Bucket string `json:"logical_backup_s3_bucket,omitempty"`
Schedule string `json:"logical_backup_schedule,omitempty"`
DockerImage string `json:"logical_backup_docker_image,omitempty"`
S3Bucket string `json:"logical_backup_s3_bucket,omitempty"`
S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"`
S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"`
S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"`
S3SSE string `json:"logical_backup_s3_sse,omitempty"`
}

17
pkg/cluster/k8sres.go
View File

@ -1575,6 +1575,14 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
Name: "LOGICAL_BACKUP_S3_BUCKET",
Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket,
},
{
Name: "LOGICAL_BACKUP_S3_ENDPOINT",
Value: c.OpConfig.LogicalBackup.LogicalBackupS3Endpoint,
},
{
Name: "LOGICAL_BACKUP_S3_SSE",
Value: c.OpConfig.LogicalBackup.LogicalBackupS3SSE,
},
{
Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX",
Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())),
@ -1613,8 +1621,15 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
},
}
c.logger.Debugf("Generated logical backup env vars %v", envVars)
if c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID != "" {
envVars = append(envVars, v1.EnvVar{Name: "AWS_ACCESS_KEY_ID", Value: c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID})
}
if c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey != "" {
envVars = append(envVars, v1.EnvVar{Name: "AWS_SECRET_ACCESS_KEY", Value: c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey})
}
c.logger.Debugf("Generated logical backup env vars %v", envVars)
return envVars
}

4
pkg/controller/operator_config.go
View File

@ -101,6 +101,10 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
result.LogicalBackupSchedule = fromCRD.LogicalBackup.Schedule
result.LogicalBackupDockerImage = fromCRD.LogicalBackup.DockerImage
result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket
result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint
result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID
result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey
result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE
// debug config
result.DebugLogging = fromCRD.OperatorDebug.DebugLogging

12
pkg/util/config/config.go
View File

@ -68,11 +68,15 @@ type Scalyr struct {
ScalyrMemoryLimit string `name:"scalyr_memory_limit" default:"1Gi"`
}
// LogicalBackup
// LogicalBackup defines configration for logical backup
type LogicalBackup struct {
LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"`
LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"`
LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""`
LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"`
LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"`
LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""`
LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""`
LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""`
LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""`
LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:"AES256"`
}
// Config describes operator config