From b7ec5d8d973f9f3498dfa6287c23d1617804a150 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 17 Feb 2021 18:40:30 +0100 Subject: [PATCH] improve docs and manifest examples --- README.md | 4 +- docs/administrator.md | 282 ++++++++++++------ docs/user.md | 33 +- manifests/complete-postgres-manifest.yaml | 28 +- ...gresql-operator-default-configuration.yaml | 1 + 5 files changed, 233 insertions(+), 115 deletions(-) diff --git a/README.md b/README.md index 32db8b0ce..e3c9d97e5 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The Postgres Operator delivers an easy to run highly-available [PostgreSQL](https://www.postgresql.org/) -clusters on Kubernetes (K8s) powered by [Patroni](https://github.com/zalando/spilo). +clusters on Kubernetes (K8s) powered by [Patroni](https://github.com/zalando/patroni). It is configured only through Postgres manifests (CRDs) to ease integration into automated CI/CD pipelines with no access to Kubernetes API directly, promoting infrastructure as code vs manual operations. @@ -24,7 +24,7 @@ pipelines with no access to Kubernetes API directly, promoting infrastructure as * Support for custom TLS certificates * UI to create and edit Postgres cluster manifests * Works well on Amazon AWS, Google Cloud, OpenShift and locally on Kind -* Base support for AWS EBS gp3 migration (iops, throughput pending) +* Support for AWS EBS gp3 migration ### PostgreSQL features diff --git a/docs/administrator.md b/docs/administrator.md index 30b612ded..fb9f774fd 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -135,6 +135,26 @@ Every other Postgres cluster which lacks the annotation will be ignored by this operator. Conversely, operators without a defined `CONTROLLER_ID` will ignore clusters with defined ownership of another operator. +## Understanding rolling update of Spilo pods + +The operator logs reasons for a rolling update with the `info` level and a diff +between the old and new StatefulSet specs with the `debug` level. To benefit +from numerous escape characters in the latter log entry, view it in CLI with +`echo -e`. Note that the resultant message will contain some noise because the +`PodTemplate` used by the operator is yet to be updated with the default values +used internally in K8s. + +The operator also support lazy updates of the Spilo image. That means the pod +template of a PG cluster's stateful set is updated immediately with the new +image, but no rolling update follows. This feature saves you a switchover - and +hence downtime - when you know pods are re-started later anyway, for instance +due to the node rotation. To force a rolling update, disable this mode by +setting the `enable_lazy_spilo_upgrade` to `false` in the operator configuration +and restart the operator pod. With the standard eager rolling updates the +operator checks during Sync all pods run images specified in their respective +statefulsets. The operator triggers a rolling upgrade for PG clusters that +violate this condition. + ## Delete protection via annotations To avoid accidental deletes of Postgres clusters the operator can check the @@ -196,7 +216,6 @@ On the next sync event it should change to `Running`. However, as it is in fact a new resource for K8s, the UID will differ which can trigger a rolling update of the pods because the UID is used as part of backup path to S3. - ## Role-based access control for the operator The manifest [`operator-service-account-rbac.yaml`](../manifests/operator-service-account-rbac.yaml) @@ -393,21 +412,24 @@ spec: ## Custom Pod Environment Variables -It is possible to configure a ConfigMap as well as a Secret which are used by the Postgres pods as -an additional provider for environment variables. One use case is to customize -the Spilo image and configure it with environment variables. Another case could be to provide custom -cloud provider or backup settings. -In general the Operator will give preference to the globally configured variables, to not have the custom -ones interfere with core functionality. Variables with the 'WAL_' and 'LOG_' prefix can be overwritten though, to allow -backup and logshipping to be specified differently. +It is possible to configure a ConfigMap as well as a Secret which are used by +the Postgres pods as an additional provider for environment variables. One use +case is a customized Spilo image configured by extra environment variables. +Another case could be to provide custom cloud provider or backup settings. +In general the Operator will give preference to the globally configured +variables, to not have the custom ones interfere with core functionality. +Variables with the 'WAL_' and 'LOG_' prefix can be overwritten though, to +allow backup and log shipping to be specified differently. ### Via ConfigMap -The ConfigMap with the additional settings is referenced in the operator's main configuration. -A namespace can be specified along with the name. If left out, the configured -default namespace of your K8s client will be used and if the ConfigMap is not -found there, the Postgres cluster's namespace is taken when different: + +The ConfigMap with the additional settings is referenced in the operator's +main configuration. A namespace can be specified along with the name. If left +out, the configured default namespace of your K8s client will be used and if +the ConfigMap is not found there, the Postgres cluster's namespace is taken +when different: **postgres-operator ConfigMap** @@ -446,15 +468,15 @@ data: MY_CUSTOM_VAR: value ``` -The key-value pairs of the ConfigMap are then added as environment variables to the -Postgres StatefulSet/pods. - +The key-value pairs of the ConfigMap are then added as environment variables +to the Postgres StatefulSet/pods. ### Via Secret -The Secret with the additional variables is referenced in the operator's main configuration. -To protect the values of the secret from being exposed in the pod spec they are each referenced -as SecretKeyRef. -This does not allow for the secret to be in a different namespace as the pods though + +The Secret with the additional variables is referenced in the operator's main +configuration. To protect the values of the secret from being exposed in the +pod spec they are each referenced as SecretKeyRef. This does not allow for the +secret to be in a different namespace as the pods though **postgres-operator ConfigMap** @@ -493,8 +515,8 @@ data: MY_CUSTOM_VAR: dmFsdWU= ``` -The key-value pairs of the Secret are all accessible as environment variables to the -Postgres StatefulSet/pods. +The key-value pairs of the Secret are all accessible as environment variables +to the Postgres StatefulSet/pods. ## Limiting the number of min and max instances in clusters @@ -503,8 +525,8 @@ instances permitted by each Postgres cluster managed by the operator. If either `min_instances` or `max_instances` is set to a non-zero value, the operator may adjust the number of instances specified in the cluster manifest to match either the min or the max boundary. For instance, of a cluster manifest has 1 -instance and the `min_instances` is set to 3, the cluster will be created with 3 -instances. By default, both parameters are set to `-1`. +instance and the `min_instances` is set to 3, the cluster will be created with +3 instances. By default, both parameters are set to `-1`. ## Load balancers and allowed IP ranges @@ -579,59 +601,6 @@ maintaining and troubleshooting, and (c) additional teams, superuser teams or members associated with the owning team. The latter is managed via the [PostgresTeam CRD](user.md#additional-teams-and-members-per-cluster). - -## Understanding rolling update of Spilo pods - -The operator logs reasons for a rolling update with the `info` level and a diff -between the old and new StatefulSet specs with the `debug` level. To benefit -from numerous escape characters in the latter log entry, view it in CLI with -`echo -e`. Note that the resultant message will contain some noise because the -`PodTemplate` used by the operator is yet to be updated with the default values -used internally in K8s. - -The operator also support lazy updates of the Spilo image. That means the pod -template of a PG cluster's stateful set is updated immediately with the new -image, but no rolling update follows. This feature saves you a switchover - and -hence downtime - when you know pods are re-started later anyway, for instance -due to the node rotation. To force a rolling update, disable this mode by -setting the `enable_lazy_spilo_upgrade` to `false` in the operator configuration -and restart the operator pod. With the standard eager rolling updates the -operator checks during Sync all pods run images specified in their respective -statefulsets. The operator triggers a rolling upgrade for PG clusters that -violate this condition. - -## Logical backups - -The operator can manage K8s cron jobs to run logical backups of Postgres -clusters. The cron job periodically spawns a batch job that runs a single pod. -The backup script within this pod's container can connect to a DB for a logical -backup. The operator updates cron jobs during Sync if the job schedule changes; -the job name acts as the job identifier. These jobs are to be enabled for each -individual Postgres cluster by setting `enableLogicalBackup: true` in its -manifest. Notes: - -1. The [example image](../docker/logical-backup/Dockerfile) implements the -backup via `pg_dumpall` and upload of compressed and encrypted results to an S3 -bucket; the default image ``registry.opensource.zalan.do/acid/logical-backup`` -is the same image built with the Zalando-internal CI pipeline. `pg_dumpall` -requires a `superuser` access to a DB and runs on the replica when possible. - -2. Due to the [limitation of K8s cron jobs](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) -it is highly advisable to set up additional monitoring for this feature; such -monitoring is outside of the scope of operator responsibilities. - -3. The operator does not remove old backups. - -4. You may use your own image by overwriting the relevant field in the operator -configuration. Any such image must ensure the logical backup is able to finish -[in presence of pod restarts](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#handling-pod-and-container-failures) -and [simultaneous invocations](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) -of the backup cron job. - -5. For that feature to work, your RBAC policy must enable operations on the -`cronjobs` resource from the `batch` API group for the operator service account. -See [example RBAC](../manifests/operator-service-account-rbac.yaml) - ## Access to cloud resources from clusters in non-cloud environment To access cloud resources like S3 from a cluster on bare metal you can use @@ -649,26 +618,108 @@ A secret can be pre-provisioned in different ways: * Automatically provisioned via a custom K8s controller like [kube-aws-iam-controller](https://github.com/mikkeloscar/kube-aws-iam-controller) -## Google Cloud Platform setup +## WAL archiving and basebackups -To configure the operator on GCP there are some prerequisites that are needed: +Spilo is shipped with [WAL-E](https://github.com/wal-e/wal-e) and its successor +[WAL-G](https://github.com/wal-g/wal-g) to perform WAL archiving. By default, +WAL-E is used because it is more battle-tested. Additionally to the continuous +backup stream a [basebackup](https://www.postgresql.org/docs/13/app-pgbasebackup.html) +is initialized every night and 1am UTC. + +These are the pre-configured settings in the docker image: + +```bash +archive_command: `envdir "{WALE_ENV_DIR}" {WALE_BINARY} wal-push "%p"` +restore_command: `envdir "{{WALE_ENV_DIR}}" /scripts/restore_command.sh "%f" "%p"` + +AWS_ENDPOINT: 'https://s3.AWS_REGION.amazonaws.com:443' +BACKUP_NUM_TO_RETAIN: 5 +BACKUP_SCHEDULE: '00 01 * * *' +USE_WALG_BACKUP: false (not set) +USE_WALG_RESTORE: false (not set) +WALE_S3_ENDPOINT: 'https+path://s3.AWS_REGION.amazonaws.com:443' +WALE_S3_PREFIX: 's3://bucket-name/very/long/path' +``` + +If the prefix is not specified Spilo will generate it from `WAL_S3_BUCKET`. +When the `AWS_REGION` is set you `AWS_ENDPOINT` and `WALE_S3_ENDPOINT` are +generated automatically. + +The backup path has to be specified in the operator configuration. You have to +make sure that Postgres is allowed to send compressed WAL files to the backup +location, e.g. an S3 bucket. If you want to change some settings you have to +overwrite Spilo's [environment variables](https://github.com/zalando/spilo/blob/master/ENVIRONMENT.rst) +using an [extra configmap or secret](#custom-pod-environment-variables). + +### Using AWS S3 or compliant services + +When using AWS you have to reference the S3 backup path, the IAM role and the +AWS region in the configuration. + +**postgres-operator ConfigMap** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-operator +data: + aws_region: eu-central-1 + kube_iam_role: postgres-pod-role + wal_s3_bucket: your-backup-path +``` + +**OperatorConfiguration** + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration +metadata: + name: postgresql-operator-configuration +configuration: + aws_or_gcp: + aws_region: eu-central-1 + kube_iam_role: postgres-pod-role + wal_s3_bucket: your-backup-path +``` + +The referenced IAM role should contain the following privileges: + +```yaml + PostgresPodRole: + Type: "AWS::IAM::Role" + Properties: + RoleName: "postgres-pod-role" + Path: "/" + Policies: + - PolicyName: "SpiloS3Access" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Action: "s3:*" + Effect: "Allow" + Resource: + - "arn:aws:s3:::your-backup-path" + - "arn:aws:s3:::your-backup-path/*" +``` + +### Google Cloud Platform setup + +To configure the operator on GCP these prerequisites that are needed: * A service account with the proper IAM setup to access the GCS bucket for the WAL-E logs * The credentials file for the service account. -The configuration paramaters that we will be using are: +The configuration parameters that we will be using are: * `additional_secret_mount` * `additional_secret_mount_path` * `gcp_credentials` * `wal_gs_bucket` -### Generate a K8s secret resource - -Generate the K8s secret resource that will contain your service account's +1. Generate the K8s secret resource that will contain your service account's credentials. It's highly recommended to use a service account and limit its scope to just the WAL-E bucket. - ```yaml apiVersion: v1 kind: Secret @@ -681,11 +732,9 @@ stringData: ``` -### Setup your operator configuration values - -With the `psql-wale-creds` resource applied to your cluster, ensure that -the operator's configuration is set up like the following: - +2. Setup your operator configuration values. With the `psql-wale-creds` +resource applied to your cluster, ensure that the operator's configuration +is set up like the following: ```yml ... aws_or_gcp: @@ -700,9 +749,8 @@ aws_or_gcp: ... ``` -### Setup pod environment configmap - -To make postgres-operator work with GCS, use following configmap: +3. Setup pod environment configmap that instructs the operator to use WAL-G, +instead of WAL-E, for backup and restore. ```yml apiVersion: v1 kind: ConfigMap @@ -715,9 +763,8 @@ data: USE_WALG_RESTORE: "true" CLONE_USE_WALG_RESTORE: "true" ``` -This configmap will instruct operator to use WAL-G, instead of WAL-E, for backup and restore. -Then provide this configmap in postgres-operator settings: +4. Then provide this configmap in postgres-operator settings: ```yml ... # namespaced name of the ConfigMap with environment variables to populate on every pod @@ -725,6 +772,52 @@ pod_environment_configmap: "postgres-operator-system/pod-env-overrides" ... ``` +## Logical backups + +The operator can manage K8s cron jobs to run logical backups (SQL dumps) of +Postgres clusters. The cron job periodically spawns a batch job that runs a +single pod. The backup script within this pod's container can connect to a DB +for a logical backup. The operator updates cron jobs during Sync if the job +schedule changes; the job name acts as the job identifier. These jobs are to +be enabled for each individual Postgres cluster by updating the manifest: + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: demo-cluster +spec: + enableLogicalBackup: true +``` + +There a few things to consider when using logical backups: + +1. Logical backups should not seen as a proper alternative to basebackups and WAL +archiving which are described above. At the moment, the operator cannot restore +logical backups automatically and you do not get point-in-time recovery but only +snapshots of your data. In its current state, see logical backups as a way to +quickly create SQL dumps that you can easily restore in an empty test cluster. + +2. The [example image](../docker/logical-backup/Dockerfile) implements the backup +via `pg_dumpall` and upload of compressed and encrypted results to an S3 bucket. +`pg_dumpall` requires a `superuser` access to a DB and runs on the replica when +possible. + +3. Due to the [limitation of K8s cron jobs](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) +it is highly advisable to set up additional monitoring for this feature; such +monitoring is outside of the scope of operator responsibilities. + +4. The operator does not remove old backups. + +5. You may use your own image by overwriting the relevant field in the operator +configuration. Any such image must ensure the logical backup is able to finish +[in presence of pod restarts](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#handling-pod-and-container-failures) +and [simultaneous invocations](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) +of the backup cron job. + +6. For that feature to work, your RBAC policy must enable operations on the +`cronjobs` resource from the `batch` API group for the operator service account. +See [example RBAC](../manifests/operator-service-account-rbac.yaml) ## Sidecars for Postgres clusters @@ -739,6 +832,7 @@ configuration: name: global-sidecar ports: - containerPort: 80 + protocol: TCP volumeMounts: - mountPath: /custom-pgdata-mountpoint name: pgdata @@ -814,7 +908,7 @@ make docker # build in image in minikube docker env eval $(minikube docker-env) -docker build -t registry.opensource.zalan.do/acid/postgres-operator-ui:v1.3.0 . +docker build -t registry.opensource.zalan.do/acid/postgres-operator-ui:v1.6.1 . # apply UI manifests next to a running Postgres Operator kubectl apply -f manifests/ diff --git a/docs/user.md b/docs/user.md index ec5941d9e..6c7f675ad 100644 --- a/docs/user.md +++ b/docs/user.md @@ -30,7 +30,7 @@ spec: databases: foo: zalando postgresql: - version: "12" + version: "13" ``` Once you cloned the Postgres Operator [repository](https://github.com/zalando/postgres-operator) @@ -509,6 +509,25 @@ spec: defaultUsers: true ``` +### Schema `search_path` for default roles + +The schema [`search_path`](https://www.postgresql.org/docs/13/ddl-schemas.html#DDL-SCHEMAS-PATH) +for each role will include the role name and the schemas, this role should have +access to. So `foo_bar_writer` does not have to schema-qualify tables from +schemas `foo_bar_writer, bar`, while `foo_writer` can look up `foo_writer` and +any schema listed under `schemas`. To register the default `public` schema in +the `search_path` (because some extensions are installed there) one has to add +the following (assuming no extra roles are desired only for the public schema): + +```yaml +spec: + preparedDatabases: + foo: + schemas: + public: + defaultRoles: false +``` + ### Database extensions Prepared databases also allow for creating Postgres extensions. They will be @@ -625,6 +644,10 @@ spec: - pci ``` +## In-place major version upgrade + +Starting with Spilo 13, operator supports in-place major version upgrade to a higher major version (e.g. from PG 10 to PG 12). To trigger the upgrade, simply increase the version in the manifest. It is your responsibility to test your applications against the new version before the upgrade; downgrading is not supported. The easiest way to do so is to try the upgrade on the cloned cluster first. For details of how Spilo does the upgrade [see here](https://github.com/zalando/spilo/pull/488), operator implementation is described [in the admin docs](administrator.md#minor-and-major-version-upgrade). + ## How to clone an existing PostgreSQL cluster You can spin up a new cluster as a clone of the existing one, using a `clone` @@ -636,10 +659,6 @@ section in the spec. There are two options here: Note, that cloning can also be used for [major version upgrades](administrator.md#minor-and-major-version-upgrade) of PostgreSQL. -## In-place major version upgrade - -Starting with Spilo 13, operator supports in-place major version upgrade to a higher major version (e.g. from PG 10 to PG 12). To trigger the upgrade, simply increase the version in the manifest. It is your responsibility to test your applications against the new version before the upgrade; downgrading is not supported. The easiest way to do so is to try the upgrade on the cloned cluster first. For details of how Spilo does the upgrade [see here](https://github.com/zalando/spilo/pull/488), operator implementation is described [in the admin docs](administrator.md#minor-and-major-version-upgrade). - ### Clone from S3 Cloning from S3 has the advantage that there is no impact on your production @@ -875,8 +894,8 @@ size of volumes that correspond to the previously running pods is not changed. ## Logical backups -You can enable logical backups from the cluster manifest by adding the following -parameter in the spec section: +You can enable logical backups (SQL dumps) from the cluster manifest by adding +the following parameter in the spec section: ```yaml spec: diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 8140dbb32..835cb0748 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -148,18 +148,22 @@ spec: image: busybox command: [ "/bin/date" ] # sidecars: -# - name: "telegraf-sidecar" -# image: "telegraf:latest" -# resources: -# limits: -# cpu: 500m -# memory: 500Mi -# requests: -# cpu: 100m -# memory: 100Mi -# env: -# - name: "USEFUL_VAR" -# value: "perhaps-true" +# - name: "telegraf-sidecar" +# image: "telegraf:latest" +# ports: +# name: metrics +# containerPort: 8094 +# protocol: TCP +# resources: +# limits: +# cpu: 500m +# memory: 500Mi +# requests: +# cpu: 100m +# memory: 100Mi +# env: +# - name: "USEFUL_VAR" +# value: "perhaps-true" # Custom TLS certificate. Disabled unless tls.secretName has a value. tls: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 656fc94c8..cc0ae609a 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -21,6 +21,7 @@ configuration: # name: global-sidecar-1 # ports: # - containerPort: 80 + # protocol: TCP workers: 8 users: replication_username: standby