Add additional S3 settings for cloning (#497)

docs/reference/cluster_manifest.md

@@ -254,6 +254,19 @@ under the `clone` top-level key and do not affect the already running cluster.
   timestamp. When this parameter is set the operator will not consider cloning
   from the live cluster, even if it is running, and instead goes to S3. Optional.
 
+* **s3_endpoint**
+  the url of the S3-compatible service should be set when cloning from non AWS S3. Optional.
+
+* **s3_access_key_id**
+  the access key id, used for authentication on S3 service. Optional.
+
+* **s3_secret_access_key**
+  the secret access key, used for authentication on S3 service. Optional.
+
+* **s3_force_path_style**
+  to enable path-style addressing(i.e., http://s3.amazonaws.com/BUCKET/KEY) when connecting to an S3-compatible service
+  that lack of support for sub-domain style bucket URLs (i.e., http://BUCKET.s3.amazonaws.com/KEY). Optional.
+
 ### EBS volume resizing
 
 Those parameters are grouped under the `volume` top-level key and define the

docs/user.md

@@ -254,6 +254,24 @@ metadata:
 Note that timezone is required for `timestamp`. Otherwise, offset is relative
 to UTC, see [RFC 3339 section 5.6) 3339 section 5.6](https://www.ietf.org/rfc/rfc3339.txt).
 
+For non AWS S3 following settings can be set to support cloning from other S3 implementations:
+
+```yaml
+apiVersion: "acid.zalan.do/v1"
+kind: postgresql
+metadata:
+  name: acid-test-cluster
+spec:
+  clone:
+    uid: "efd12e58-5786-11e8-b5a7-06148230260c"
+    cluster: "acid-batman"
+    timestamp: "2017-12-19T12:40:33+01:00"
+    s3_endpoint: https://s3.acme.org
+    s3_access_key_id: 0123456789abcdef0123456789abcdef
+    s3_secret_access_key: 0123456789abcdef0123456789abcdef
+    s3_force_path_style: true
+```
+
 ## Sidecar Support
 
 Each cluster can specify arbitrary sidecars to run. These containers could be used for

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -115,10 +115,14 @@ type Patroni struct {
 
 // CloneDescription describes which cluster the new should clone and up to which point in time
 type CloneDescription struct {
-	ClusterName  string `json:"cluster,omitempty"`
+	ClusterName       string `json:"cluster,omitempty"`
-	UID          string `json:"uid,omitempty"`
+	UID               string `json:"uid,omitempty"`
-	EndTimestamp string `json:"timestamp,omitempty"`
+	EndTimestamp      string `json:"timestamp,omitempty"`
-	S3WalPath    string `json:"s3_wal_path,omitempty"`
+	S3WalPath         string `json:"s3_wal_path,omitempty"`
+	S3Endpoint        string `json:"s3_endpoint,omitempty"`
+	S3AccessKeyId     string `json:"s3_access_key_id,omitempty"`
+	S3SecretAccessKey string `json:"s3_secret_access_key,omitempty"`
+	S3ForcePathStyle  *bool  `json:"s3_force_path_style,omitempty" defaults:"false"`
 }
 
 // Sidecar defines a container to be run in the same pod as the Postgres container.

pkg/apis/acid.zalan.do/v1/util_test.go

@@ -61,12 +61,12 @@ var cloneClusterDescriptions = []struct {
 	in  *CloneDescription
 	err error
 }{
-	{&CloneDescription{"foo+bar", "", "NotEmpty", ""}, nil},
+	{&CloneDescription{"foo+bar", "", "NotEmpty", "", "", "", "", nil}, nil},
-	{&CloneDescription{"foo+bar", "", "", ""},
+	{&CloneDescription{"foo+bar", "", "", "", "", "", "", nil},
 		errors.New(`clone cluster name must confirm to DNS-1035, regex used for validation is "^[a-z]([-a-z0-9]*[a-z0-9])?$"`)},
-	{&CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", ""},
+	{&CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", "", "", "", "", nil},
 		errors.New("clone cluster name must be no longer than 63 characters")},
-	{&CloneDescription{"foobar", "", "", ""}, nil},
+	{&CloneDescription{"foobar", "", "", "", "", "", "", nil}, nil},
 }
 
 var maintenanceWindows = []struct {

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -50,6 +50,11 @@ func (in *AWSGCPConfiguration) DeepCopy() *AWSGCPConfiguration {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CloneDescription) DeepCopyInto(out *CloneDescription) {
 	*out = *in
+	if in.S3ForcePathStyle != nil {
+		in, out := &in.S3ForcePathStyle, &out.S3ForcePathStyle
+		*out = new(bool)
+		**out = **in
+	}
 	return
 }
 
@@ -459,7 +464,7 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
-	out.Clone = in.Clone
+	in.Clone.DeepCopyInto(&out.Clone)
 	if in.Databases != nil {
 		in, out := &in.Databases, &out.Databases
 		*out = make(map[string]string, len(*in))

pkg/cluster/k8sres.go

@@ -1266,6 +1266,29 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
 		result = append(result, v1.EnvVar{Name: "CLONE_METHOD", Value: "CLONE_WITH_WALE"})
 		result = append(result, v1.EnvVar{Name: "CLONE_TARGET_TIME", Value: description.EndTimestamp})
 		result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_PREFIX", Value: ""})
+
+		if description.S3Endpoint != "" {
+		   result = append(result, v1.EnvVar{Name: "CLONE_AWS_ENDPOINT", Value: description.S3Endpoint})
+		   result = append(result, v1.EnvVar{Name: "CLONE_WALE_S3_ENDPOINT", Value: description.S3Endpoint})
+		}
+
+		if description.S3AccessKeyId != "" {
+		   result = append(result, v1.EnvVar{Name: "CLONE_AWS_ACCESS_KEY_ID", Value: description.S3AccessKeyId})
+		}
+
+		if description.S3SecretAccessKey != "" {
+		   result = append(result, v1.EnvVar{Name: "CLONE_AWS_SECRET_ACCESS_KEY", Value: description.S3SecretAccessKey})
+		}
+
+		if description.S3ForcePathStyle != nil {
+		   s3ForcePathStyle := "0"
+
+		   if *description.S3ForcePathStyle {
+			   s3ForcePathStyle = "1"
+		   }
+
+		   result = append(result, v1.EnvVar{Name: "CLONE_AWS_S3_FORCE_PATH_STYLE", Value: s3ForcePathStyle})
+		}
 	}
 
 	return result