From bbf28c4df70f9ba2ac0b1ccf6673bea9b975550f Mon Sep 17 00:00:00 2001 From: "teuto.net Netzdienste GmbH" Date: Fri, 14 Jun 2019 12:28:00 +0200 Subject: [PATCH] Add additional S3 settings for cloning (#497) --- docs/reference/cluster_manifest.md | 13 +++++++++++ docs/user.md | 18 +++++++++++++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 12 ++++++---- pkg/apis/acid.zalan.do/v1/util_test.go | 8 +++---- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 7 +++++- pkg/cluster/k8sres.go | 23 +++++++++++++++++++ 6 files changed, 72 insertions(+), 9 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 2768f02aa..d2bbcbe88 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -254,6 +254,19 @@ under the `clone` top-level key and do not affect the already running cluster. timestamp. When this parameter is set the operator will not consider cloning from the live cluster, even if it is running, and instead goes to S3. Optional. +* **s3_endpoint** + the url of the S3-compatible service should be set when cloning from non AWS S3. Optional. + +* **s3_access_key_id** + the access key id, used for authentication on S3 service. Optional. + +* **s3_secret_access_key** + the secret access key, used for authentication on S3 service. Optional. + +* **s3_force_path_style** + to enable path-style addressing(i.e., http://s3.amazonaws.com/BUCKET/KEY) when connecting to an S3-compatible service + that lack of support for sub-domain style bucket URLs (i.e., http://BUCKET.s3.amazonaws.com/KEY). Optional. + ### EBS volume resizing Those parameters are grouped under the `volume` top-level key and define the diff --git a/docs/user.md b/docs/user.md index 0f643f4c5..f33409720 100644 --- a/docs/user.md +++ b/docs/user.md @@ -254,6 +254,24 @@ metadata: Note that timezone is required for `timestamp`. Otherwise, offset is relative to UTC, see [RFC 3339 section 5.6) 3339 section 5.6](https://www.ietf.org/rfc/rfc3339.txt). +For non AWS S3 following settings can be set to support cloning from other S3 implementations: + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-test-cluster +spec: + clone: + uid: "efd12e58-5786-11e8-b5a7-06148230260c" + cluster: "acid-batman" + timestamp: "2017-12-19T12:40:33+01:00" + s3_endpoint: https://s3.acme.org + s3_access_key_id: 0123456789abcdef0123456789abcdef + s3_secret_access_key: 0123456789abcdef0123456789abcdef + s3_force_path_style: true +``` + ## Sidecar Support Each cluster can specify arbitrary sidecars to run. These containers could be used for diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 1a191786e..33c3a159b 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -115,10 +115,14 @@ type Patroni struct { // CloneDescription describes which cluster the new should clone and up to which point in time type CloneDescription struct { - ClusterName string `json:"cluster,omitempty"` - UID string `json:"uid,omitempty"` - EndTimestamp string `json:"timestamp,omitempty"` - S3WalPath string `json:"s3_wal_path,omitempty"` + ClusterName string `json:"cluster,omitempty"` + UID string `json:"uid,omitempty"` + EndTimestamp string `json:"timestamp,omitempty"` + S3WalPath string `json:"s3_wal_path,omitempty"` + S3Endpoint string `json:"s3_endpoint,omitempty"` + S3AccessKeyId string `json:"s3_access_key_id,omitempty"` + S3SecretAccessKey string `json:"s3_secret_access_key,omitempty"` + S3ForcePathStyle *bool `json:"s3_force_path_style,omitempty" defaults:"false"` } // Sidecar defines a container to be run in the same pod as the Postgres container. diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index 537619aaf..02bdcca1c 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -61,12 +61,12 @@ var cloneClusterDescriptions = []struct { in *CloneDescription err error }{ - {&CloneDescription{"foo+bar", "", "NotEmpty", ""}, nil}, - {&CloneDescription{"foo+bar", "", "", ""}, + {&CloneDescription{"foo+bar", "", "NotEmpty", "", "", "", "", nil}, nil}, + {&CloneDescription{"foo+bar", "", "", "", "", "", "", nil}, errors.New(`clone cluster name must confirm to DNS-1035, regex used for validation is "^[a-z]([-a-z0-9]*[a-z0-9])?$"`)}, - {&CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", ""}, + {&CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", "", "", "", "", nil}, errors.New("clone cluster name must be no longer than 63 characters")}, - {&CloneDescription{"foobar", "", "", ""}, nil}, + {&CloneDescription{"foobar", "", "", "", "", "", "", nil}, nil}, } var maintenanceWindows = []struct { diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 282fb311f..65d8ee925 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -50,6 +50,11 @@ func (in *AWSGCPConfiguration) DeepCopy() *AWSGCPConfiguration { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloneDescription) DeepCopyInto(out *CloneDescription) { *out = *in + if in.S3ForcePathStyle != nil { + in, out := &in.S3ForcePathStyle, &out.S3ForcePathStyle + *out = new(bool) + **out = **in + } return } @@ -459,7 +464,7 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - out.Clone = in.Clone + in.Clone.DeepCopyInto(&out.Clone) if in.Databases != nil { in, out := &in.Databases, &out.Databases *out = make(map[string]string, len(*in)) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index a16e810ed..585cf2bed 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1266,6 +1266,29 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) result = append(result, v1.EnvVar{Name: "CLONE_METHOD", Value: "CLONE_WITH_WALE"}) result = append(result, v1.EnvVar{Name: "CLONE_TARGET_TIME", Value: description.EndTimestamp}) result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + + if description.S3Endpoint != "" { + result = append(result, v1.EnvVar{Name: "CLONE_AWS_ENDPOINT", Value: description.S3Endpoint}) + result = append(result, v1.EnvVar{Name: "CLONE_WALE_S3_ENDPOINT", Value: description.S3Endpoint}) + } + + if description.S3AccessKeyId != "" { + result = append(result, v1.EnvVar{Name: "CLONE_AWS_ACCESS_KEY_ID", Value: description.S3AccessKeyId}) + } + + if description.S3SecretAccessKey != "" { + result = append(result, v1.EnvVar{Name: "CLONE_AWS_SECRET_ACCESS_KEY", Value: description.S3SecretAccessKey}) + } + + if description.S3ForcePathStyle != nil { + s3ForcePathStyle := "0" + + if *description.S3ForcePathStyle { + s3ForcePathStyle = "1" + } + + result = append(result, v1.EnvVar{Name: "CLONE_AWS_S3_FORCE_PATH_STYLE", Value: s3ForcePathStyle}) + } } return result