From fbd04896c2416ea1d0ad7010e3b1bc47a75472a9 Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Wed, 16 Dec 2020 01:41:08 -0800 Subject: [PATCH 1/3] Add ability to upload logical backup to gcs (#1173) Support logical backup provider/storage S3 and GCS equivalent --- docker/logical-backup/Dockerfile | 3 +++ docker/logical-backup/dump.sh | 19 ++++++++++++++++++- docs/reference/operator_parameters.md | 7 +++++++ .../v1/operator_configuration_type.go | 18 ++++++++++-------- pkg/cluster/k8sres.go | 8 ++++++++ pkg/controller/operator_config.go | 2 ++ pkg/util/config/config.go | 18 ++++++++++-------- 7 files changed, 58 insertions(+), 17 deletions(-) diff --git a/docker/logical-backup/Dockerfile b/docker/logical-backup/Dockerfile index 94c524381..6cc875c58 100644 --- a/docker/logical-backup/Dockerfile +++ b/docker/logical-backup/Dockerfile @@ -13,7 +13,10 @@ RUN apt-get update \ curl \ jq \ gnupg \ + gcc \ + libffi-dev \ && pip3 install --no-cache-dir awscli --upgrade \ + && pip3 install --no-cache-dir gsutil --upgrade \ && echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ && cat /etc/apt/sources.list.d/pgdg.list \ && curl --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh index 2d9a39e02..50f7e6e4c 100755 --- a/docker/logical-backup/dump.sh +++ b/docker/logical-backup/dump.sh @@ -46,6 +46,23 @@ function aws_upload { aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}" } +function gcs_upload { + PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz + + gsutil -o Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS cp - "$PATH_TO_BACKUP" +} + +function upload { + case $LOGICAL_BACKUP_PROVIDER in + "gcs") + gcs_upload + ;; + *) + aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF)) + ;; + esac +} + function get_pods { declare -r SELECTOR="$1" @@ -93,7 +110,7 @@ for search in "${search_strategy[@]}"; do done set -x -dump | compress | aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF)) +dump | compress | upload [[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 )) set +x diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 09158e536..cd1cf8782 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -563,6 +563,10 @@ grouped under the `logical_backup` key. The default image is the same image built with the Zalando-internal CI pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup" +* **logical_backup_provider** + Specifies the storage provider to which the backup should be uploaded (`s3` or `gcs`). + Default: "s3" + * **logical_backup_s3_bucket** S3 bucket to store backup results. The bucket has to be present and accessible by Postgres pods. Default: empty. @@ -583,6 +587,9 @@ grouped under the `logical_backup` key. * **logical_backup_s3_secret_access_key** When set, value will be in AWS_SECRET_ACCESS_KEY env variable. The Default is empty. +* **logical_backup_google_application_credentials** + Specifies the path of the google cloud service account json file. Default is empty. + ## Debugging the operator Options to aid debugging of the operator itself. Grouped under the `debug` key. diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 9578f8138..9e5d01040 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -186,14 +186,16 @@ type ConnectionPoolerConfiguration struct { // OperatorLogicalBackupConfiguration defines configuration for logical backup type OperatorLogicalBackupConfiguration struct { - Schedule string `json:"logical_backup_schedule,omitempty"` - DockerImage string `json:"logical_backup_docker_image,omitempty"` - S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` - S3Region string `json:"logical_backup_s3_region,omitempty"` - S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"` - S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` - S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"` - S3SSE string `json:"logical_backup_s3_sse,omitempty"` + Schedule string `json:"logical_backup_schedule,omitempty"` + DockerImage string `json:"logical_backup_docker_image,omitempty"` + BackupProvider string `json:"logical_backup_provider,omitempty"` + S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` + S3Region string `json:"logical_backup_s3_region,omitempty"` + S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"` + S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` + S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"` + S3SSE string `json:"logical_backup_s3_sse,omitempty"` + GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"` } // OperatorConfigurationData defines the operation config diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 5360f2668..9e5ed7050 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1988,6 +1988,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { }, }, // Bucket env vars + { + Name: "LOGICAL_BACKUP_PROVIDER", + Value: c.OpConfig.LogicalBackup.LogicalBackupProvider, + }, { Name: "LOGICAL_BACKUP_S3_BUCKET", Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket, @@ -2008,6 +2012,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())), }, + { + Name: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS", + Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials, + }, // Postgres env vars { Name: "PG_VERSION", diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 9b0c6f194..250705656 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -146,12 +146,14 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // logical backup config result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *") result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup") + result.LogicalBackupProvider = util.Coalesce(fromCRD.LogicalBackup.BackupProvider, "s3") result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE + result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials // debug config result.DebugLogging = fromCRD.OperatorDebug.DebugLogging diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 5e292dedd..acc00b2e8 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -111,14 +111,16 @@ type Scalyr struct { // LogicalBackup defines configuration for logical backup type LogicalBackup struct { - LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` - LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"` - LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` - LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""` - LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` - LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` - LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` - LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""` + LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` + LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"` + LogicalBackupProvider string `name:"logical_backup_provider" default:"s3"` + LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` + LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""` + LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` + LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` + LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` + LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""` + LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""` } // Operator options for connection pooler From 4b90809ade33d2cec63a4a1a312cfaef5e376f99 Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Wed, 16 Dec 2020 10:44:25 +0100 Subject: [PATCH 2/3] =?UTF-8?q?helm-chart:=20allow=20configmaps=20instead?= =?UTF-8?q?=20of=20endpoints=20for=20leader=20elections=E2=80=A6=20(#1037)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * helm-chart: allow configmaps instead of endpoints if leader elections uses the configmaps method * helm-chart: allow endpoints get even if config maps are used * helm-chart: allow configmaps instead of endpoints on the operator role too. Co-authored-by: Enno Boland --- .../templates/clusterrole-postgres-pod.yaml | 22 +++++++++++++++++++ .../templates/clusterrole.yaml | 6 ++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml index ef607ae3c..b3f9f08f5 100644 --- a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml +++ b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml @@ -10,6 +10,27 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} rules: # Patroni needs to watch and manage endpoints +{{- if toString .Values.configGeneral.kubernetes_use_configmaps | eq "true" }} +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get +{{- else }} - apiGroups: - "" resources: @@ -23,6 +44,7 @@ rules: - patch - update - watch +{{- end }} # Patroni needs to watch pods - apiGroups: - "" diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 00ee776f5..46113c4f1 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -63,11 +63,15 @@ rules: - patch - update - watch -# to manage endpoints which are also used by Patroni +# to manage endpoints/configmaps which are also used by Patroni - apiGroups: - "" resources: +{{- if toString .Values.configGeneral.kubernetes_use_configmaps | eq "true" }} + - configmaps +{{- else }} - endpoints +{{- end }} verbs: - create - delete From 5076e669cb4d547e565400adf900d364ac066503 Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Wed, 16 Dec 2020 02:17:08 -0800 Subject: [PATCH 3/3] Fix timestamp regex (#1178) --- charts/postgres-operator/crds/postgresqls.yaml | 2 +- manifests/postgresql.crd.yaml | 2 +- pkg/apis/acid.zalan.do/v1/crds.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index fe54fb600..784bb2a76 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -128,7 +128,7 @@ spec: type: string timestamp: type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC # Example: 1996-12-19T16:39:57-08:00 # Note: this field requires a timezone diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 80046a0f2..7836d07e7 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -124,7 +124,7 @@ spec: type: string timestamp: type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC # Example: 1996-12-19T16:39:57-08:00 # Note: this field requires a timezone diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index cc79a3efe..737346f5e 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -202,7 +202,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "timestamp": { Type: "string", Description: "Date-time format that specifies a timezone as an offset relative to UTC e.g. 1996-12-19T16:39:57-08:00", - Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$", + Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$", }, "uid": { Type: "string",