From d78ccea3a892009e7c8229800028686fc9407373 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 30 Mar 2022 15:48:56 +0200 Subject: [PATCH] finish PR1544 --- charts/postgres-operator/crds/postgresqls.yaml | 7 +++++++ docs/administrator.md | 14 +++++++++----- docs/reference/cluster_manifest.md | 12 +++++------- manifests/postgresql.crd.yaml | 7 +++++++ manifests/standby-manifest.yaml | 6 +++--- pkg/apis/acid.zalan.do/v1/crds.go | 5 +++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 4 ++-- pkg/cluster/k8sres.go | 13 ++++++------- 8 files changed, 44 insertions(+), 24 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 423fc898c..8aa6f70b3 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -464,6 +464,13 @@ spec: type: string standby_port: type: string + oneOf: + - required: + - s3_wal_path + - required: + - gs_wal_path + - required: + - standby_host streams: type: array nullable: true diff --git a/docs/administrator.md b/docs/administrator.md index dd6bf59b0..050c12a76 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -1088,11 +1088,15 @@ data: ### Standby clusters The setup for [standby clusters](user.md#setting-up-a-standby-cluster) is very -similar to cloning. At the moment, the operator only allows for streaming from -the S3 WAL archive of the master specified in the manifest. Like with cloning, -if you are using [additional environment variables](#custom-pod-environment-variables) -to access your backup location you have to copy those variables and prepend the -`STANDBY_` prefix for Spilo to find the backups and WAL files to stream. +similar to cloning. They can stream from a WAL archive (S3, GCS). Like with +cloning, if you are using [additional environment variables](#custom-pod-environment-variables) +to access your backup location you have to copy those variables and prepend +the `STANDBY_` prefix for Spilo to find the backups and WAL files to stream. + +Alternatively, standby clusters can also stream from a remote primary cluster. +You have to specify the host address. Port is optional and defaults to 5432. +Note, that only one of the options (`s3_wal_path`, `gs_wal_path`, +`standby_host`) can be present under the `standby` top-level key. ## Logical backups diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index c42f3078e..b41550e22 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -395,24 +395,22 @@ under the `clone` top-level key and do not affect the already running cluster. ## Standby cluster On startup, an existing `standby` top-level key creates a standby Postgres -cluster streaming from a remote location. Either from a S3 or GCS WAL -archive or a remote primary. When both of them are set, `standby_host` -takes precedence. +cluster streaming from a remote location - either from a S3 or GCS WAL +archive or a remote primary. Only one of options is allowed and required +if the `standby` key is present. * **s3_wal_path** the url to S3 bucket containing the WAL archive of the remote primary. - Required when the `standby` section is present even when `standby_host` is set. * **gs_wal_path** the url to GS bucket containing the WAL archive of the remote primary. - Optional, but `s3_wal_path` or `gs_wal_path` is required. * **standby_host** hostname or IP address of the primary to stream from. - When set, `s3_wal_path` is ignored. * **standby_port** - TCP port on which the primary is listening for connections. + TCP port on which the primary is listening for connections. Patroni will + use `"5432"` if not set. ## Volume properties diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 242c6a093..e73dc9ad1 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -462,6 +462,13 @@ spec: type: string standby_port: type: string + oneOf: + - required: + - s3_wal_path + - required: + - gs_wal_path + - required: + - standby_host streams: type: array nullable: true diff --git a/manifests/standby-manifest.yaml b/manifests/standby-manifest.yaml index 85c3e8750..77ab52b7c 100644 --- a/manifests/standby-manifest.yaml +++ b/manifests/standby-manifest.yaml @@ -12,6 +12,6 @@ spec: version: "14" # Make this a standby cluster and provide either the s3 bucket path of source cluster or the remote primary host for continuous streaming. standby: - s3_wal_path: "s3://path/to/bucket/containing/wal/of/source/cluster/" - # standby_host: "" - # standby_port: "" \ No newline at end of file + s3_wal_path: " s3://mybucket/spilo/acid-minimal-cluster/abcd1234-2a4b-4b2a-8c9c-c1234defg567/wal/14/" + # standby_host: "acid-minimal-cluster.default" + # standby_port: "5432" diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 897282677..7fd927bb0 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -721,6 +721,11 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "string", }, }, + OneOf: []apiextv1.JSONSchemaProps{ + apiextv1.JSONSchemaProps{Required: []string{"s3_wal_path"}}, + apiextv1.JSONSchemaProps{Required: []string{"gs_wal_path"}}, + apiextv1.JSONSchemaProps{Required: []string{"standby_host"}}, + }, }, "streams": { Type: "array", diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index d4448a277..d0eacf5a4 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -172,8 +172,8 @@ type Patroni struct { // StandbyDescription contains remote primary config or s3 wal path type StandbyDescription struct { - S3WalPath string `json:"s3_wal_path,omitempty"` - GSWalPath string `json:"gs_wal_path,omitempty"` + S3WalPath string `json:"s3_wal_path,omitempty"` + GSWalPath string `json:"gs_wal_path,omitempty"` StandbyHost string `json:"standby_host,omitempty"` StandbyPort string `json:"standby_port,omitempty"` } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index c674904ae..322f836ba 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -762,7 +762,12 @@ func (c *Cluster) generatePodTemplate( } // generatePodEnvVars generates environment variables for the Spilo Pod -func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration string, cloneDescription *acidv1.CloneDescription, standbyDescription *acidv1.StandbyDescription, customPodEnvVarsList []v1.EnvVar) []v1.EnvVar { +func (c *Cluster) generateSpiloPodEnvVars( + uid types.UID, + spiloConfiguration string, + cloneDescription *acidv1.CloneDescription, + standbyDescription *acidv1.StandbyDescription, + customPodEnvVarsList []v1.EnvVar) []v1.EnvVar { envVars := []v1.EnvVar{ { Name: "SCOPE", @@ -1111,12 +1116,6 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef sort.Slice(customPodEnvVarsList, func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name }) - if spec.StandbyCluster != nil { - if spec.StandbyCluster.S3WalPath == "" && spec.StandbyCluster.GSWalPath == "" && spec.StandbyCluster.StandbyHost == "" { - return nil, fmt.Errorf("s3_wal_path, gs_wal_path and standby_host are empty for standby cluster") - } - } - // backward compatible check for InitContainers if spec.InitContainersOld != nil { msg := "manifest parameter init_containers is deprecated."