From 54c4a044e689ec9acc1f85063faba1ec784158aa Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Tue, 16 Apr 2019 16:49:43 +0200 Subject: [PATCH] doc and conf updates --- charts/postgres-operator/values.yaml | 4 +++ docs/administrator.md | 8 +++-- docs/reference/operator_parameters.md | 9 +++--- docs/user.md | 2 +- manifests/complete-postgres-manifest.yaml | 2 +- manifests/configmap.yaml | 5 ++-- manifests/minimal-postgres-manifest.yaml | 3 -- ...gresql-operator-default-configuration.yaml | 6 +++- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 30 +++++++++---------- pkg/cluster/util.go | 3 +- 10 files changed, 38 insertions(+), 34 deletions(-) diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 21813fbe9..9bdae5484 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -62,6 +62,10 @@ config: pod_management_policy: "ordered_ready" enable_pod_antiaffinity: "false" pod_antiaffinity_topology_key: "kubernetes.io/hostname" + enable_logical_backup: "false" + logical_backup_schedule: "30 00 * * *" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-42" + logical_backup_s3_bucket: "your-s3-bucket" rbac: # Specifies whether RBAC resources should be created create: true diff --git a/docs/administrator.md b/docs/administrator.md index 09b36f7a0..4e919ed54 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -344,12 +344,14 @@ The operator logs reasons for a rolling update with the `info` level and a diff ## Logical backups -The operator can manage k8s cron jobs to do periodic logical backups of all PG clusters under its control. The cron job spawns a separate pod with a single container that connects to one of the Postgres replicas for a backup. The operator updates cron jobs during Sync if a schedule or a docker image of a job changes. Notes: +The operator can manage k8s cron jobs to run logical backups of Postgres clusters. The cron job periodically spawns a batch job that runs a single pod. The backup script within this pod's container can connect to a DB for a logical backup. The operator updates cron jobs during Sync if the job schedule changes. Notes: -1. The provided `registry.opensource.zalan.do/acid/logical-backup` image implements the backup via `pg_dumpall` and upload of (compressed) results to an S3 bucket; `pg_dumpall` requires a `superuser` access to a DB. Any such image must ensure the logical backup is able to finish [in presence of pod restarts](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#handling-pod-and-container-failures) and [simultaneous invocations](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) of the backup cron job. +1. The provided `registry.opensource.zalan.do/acid/logical-backup` image implements the backup via `pg_dumpall` and upload of (compressed) results to an S3 bucket; `pg_dumpall` requires a `superuser` access to a DB and runs on the replica when possible. 2. Due to the [limitation of Kubernetes cron jobs](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) it is highly advisable to set up additional monitoring for this feature; such monitoring is outside of the scope of operator responsibilities. 3. The operator does not remove old backups. -For that feature to work, your RBAC policy must enable operations on the `cronjobs` resource from the `batch` API group for the operator service account. \ No newline at end of file +4. You may use your own image by overwriting the relevant field in the operator configuration. Any such image must ensure the logical backup is able to finish [in presence of pod restarts](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#handling-pod-and-container-failures) and [simultaneous invocations](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) of the backup cron job. + +5. For that feature to work, your RBAC policy must enable operations on the `cronjobs` resource from the `batch` API group for the operator service account. See [example RBAC](../manifests/operator-service-account-rbac.yaml) \ No newline at end of file diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index c5f78a3e6..a7c15d094 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -51,6 +51,8 @@ parameters, those parameters have no effect and are replaced by the `CRD_READY_WAIT_INTERVAL` and `CRD_READY_WAIT_TIMEOUT` environment variables. They will be deprecated and removed in the future. +For the configmap operator configuration, the [default parameter values](https://github.com/zalando-incubator/postgres-operator/blob/master/pkg/util/config/config.go#L14) mentioned here are likely to be overwritten in your local operator installation via your local version of the operator configmap. In the case you use the operator CRD, all the CRD defaults are provided in the [operator's default configuration manifest](https://github.com/zalando-incubator/postgres-operator/blob/master/manifests/postgresql-operator-default-configuration.yaml) + Variable names are underscore-separated words. @@ -482,16 +484,13 @@ scalyr sidecar. In the CRD-based configuration they are grouped under the In the CRD-based configuration those parameters are grouped under the `logical_backup` key. * **enable_logical_backup** - Determines if the operator creates a Kubernets Cron job to do a logical backup of all Postgres clusters it controls. Default: false. + Determines if the operator creates a Kubernetes cron job to do logical backups of all Postgres clusters it controls. Default: false. * **logical_backup_schedule** Backup schedule in the cron format. Please take [the reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule) into account. Default: "30 00 \* \* \*" * **logical_backup_docker_image** - Docker image for the pods of the cron job. Must implement backup/uploading logic. Default: empty. + Docker image for the pods of the cron job. Must implement backup logic and correctly handle pod and job restarts. Default: "registry.opensource.zalan.do/acid/logical-backup:master-42" (`pgdumpall` on the replica + upload to S3). * **logical_backup_s3_bucket** S3 bucket to store backup results. The bucket has to be present and accessible by Postgres pods. Default: empty. - - -For the configmap operator configuration, the [default parameter values](https://github.com/zalando-incubator/postgres-operator/blob/master/pkg/util/config/config.go#L14) mentioned here are likely to be overwritten in your local operator installation via your local version of the operator configmap. In the case you use the operator CRD, all the CRD defaults are provided in the [operator's default configuration manifest](https://github.com/zalando-incubator/postgres-operator/blob/master/manifests/postgresql-operator-default-configuration.yaml) diff --git a/docs/user.md b/docs/user.md index 8293a325f..1942bab16 100644 --- a/docs/user.md +++ b/docs/user.md @@ -354,4 +354,4 @@ If you add ``` enableLogicalBackup: true ``` -to the cluster manifest, the operator will create and sync the configuration of a k8s cron job that periodically executes `pg_dumpall` on the target PG cluster and uploads results to an S3 bucket. Note that due to the [limitation of Kubernetes cron jobs](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) it is highly advisable to set up additional monitoring for this feature; such monitoring is outside of the scope of operator responsibilities. See [configuration reference](reference/cluster_manifest.md) for details. +to the cluster manifest, the operator will create and sync a k8s cron job to do periodic logical backups of this particular Postgres cluster. Due to the [limitation of Kubernetes cron jobs](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) it is highly advisable to set up additional monitoring for this feature; such monitoring is outside of the scope of operator responsibilities. See [configuration reference](reference/cluster_manifest.md) and [administrator documentation](administrator.md) for details on how backups are executed. diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index adffd35f2..7f52b2b0a 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -64,7 +64,7 @@ spec: # cluster: "acid-batman" # timestamp: "2017-12-19T12:40:33+01:00" # timezone required (offset relative to UTC, see RFC 3339 section 5.6) - # run periodic backups with k8s cron jobs and pg_dumpall + # run periodic backups with k8s cron jobs # enableLogicalBackup: true # logicalBackupSchedule: "30 00 * * *" maintenanceWindows: diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 237ed203a..d36abd16b 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -55,6 +55,7 @@ data: resource_check_timeout: 10m resync_period: 5m - # enable_logical_backup: "true" + # enable_logical_backup: "false" # logical_backup_schedule: "30 00 * * *" - logical_backup_docker_image: registry.opensource.zalan.do/acid/logical-backup:master-42 + # logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-42" + # logical_backup_s3_bucket: "your-s3-bucket" diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml index c791b3aef..e952df374 100644 --- a/manifests/minimal-postgres-manifest.yaml +++ b/manifests/minimal-postgres-manifest.yaml @@ -17,11 +17,8 @@ spec: # role for application foo foo_user: [] - #databases: name->owner databases: foo: zalando postgresql: version: "10" - - enableLogicalBackup: true diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 5ea5ba87c..5acc1af7c 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -91,4 +91,8 @@ configuration: # scalyr_api_key: "" # scalyr_image: "" # scalyr_server_url: "" - + logical_backup_key: + enable_logical_backup: "false" + logical_backup_schedule: "30 00 * * *" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:master-42" + logical_backup_s3_bucket: "your-s3-bucket" diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 30312fbfa..8f9118d52 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -3,7 +3,7 @@ package v1 import ( "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -43,21 +43,19 @@ type PostgresSpec struct { // load balancers' source ranges are the same for master and replica services AllowedSourceRanges []string `json:"allowedSourceRanges"` - NumberOfInstances int32 `json:"numberOfInstances"` - Users map[string]UserFlags `json:"users"` - MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` - Clone CloneDescription `json:"clone"` - ClusterName string `json:"-"` - Databases map[string]string `json:"databases,omitempty"` - Tolerations []v1.Toleration `json:"tolerations,omitempty"` - Sidecars []Sidecar `json:"sidecars,omitempty"` - InitContainers []v1.Container `json:"init_containers,omitempty"` - PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` - ShmVolume *bool `json:"enableShmVolume,omitempty"` - - // LogicalBackup contains config of a k8s cron job responsible for running pg_dumpall - EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"` - LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` + NumberOfInstances int32 `json:"numberOfInstances"` + Users map[string]UserFlags `json:"users"` + MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` + Clone CloneDescription `json:"clone"` + ClusterName string `json:"-"` + Databases map[string]string `json:"databases,omitempty"` + Tolerations []v1.Toleration `json:"tolerations,omitempty"` + Sidecars []Sidecar `json:"sidecars,omitempty"` + InitContainers []v1.Container `json:"init_containers,omitempty"` + PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` + ShmVolume *bool `json:"enableShmVolume,omitempty"` + EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"` + LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 22825c331..9d6700b22 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -13,7 +13,7 @@ import ( "k8s.io/api/apps/v1beta1" batchv1beta1 "k8s.io/api/batch/v1beta1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" policybeta1 "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -495,7 +495,6 @@ func (c *Cluster) logCronJobChanges(old, new *batchv1beta1.CronJob, isUpdate boo c.logicalBackupJob.Name, ) } - c.logger.Debugf("diff\n%s\n", util.PrettyDiff(old.Spec, new.Spec)) if reason != "" { c.logger.Infof("reason: %s", reason)