From f29bdaf96a1de9396f1b42596198e22b30184d5f Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Fri, 10 May 2019 12:52:42 +0200 Subject: [PATCH] Override clone s3 bucket path (#487) Override clone s3 bucket path Add possibility to use a custom s3 bucket path for cloning a cluster from an arbitrary bucket (e.g. from another k8s cluster). For that a new config options is introduced `s3_wal_path`, that should point to a location that spilo would understand. --- manifests/complete-postgres-manifest.yaml | 1 + pkg/apis/acid.zalan.do/v1/postgresql_type.go | 1 + pkg/apis/acid.zalan.do/v1/util_test.go | 8 +- pkg/cluster/k8sres.go | 31 +++++++- pkg/cluster/k8sres_test.go | 79 ++++++++++++++++++++ 5 files changed, 114 insertions(+), 6 deletions(-) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 4a4c6078a..276e07a1f 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -63,6 +63,7 @@ spec: # uid: "efd12e58-5786-11e8-b5a7-06148230260c" # cluster: "acid-batman" # timestamp: "2017-12-19T12:40:33+01:00" # timezone required (offset relative to UTC, see RFC 3339 section 5.6) + # s3_wal_path: "s3://custom/path/to/bucket" maintenanceWindows: - 01:00-06:00 #UTC - Sat:00:00-04:00 diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 6f7a1f857..4def8ad07 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -114,6 +114,7 @@ type CloneDescription struct { ClusterName string `json:"cluster,omitempty"` UID string `json:"uid,omitempty"` EndTimestamp string `json:"timestamp,omitempty"` + S3WalPath string `json:"s3_wal_path,omitempty"` } // Sidecar defines a container to be run in the same pod as the Postgres container. diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index 6b9f3df99..537619aaf 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -61,12 +61,12 @@ var cloneClusterDescriptions = []struct { in *CloneDescription err error }{ - {&CloneDescription{"foo+bar", "", "NotEmpty"}, nil}, - {&CloneDescription{"foo+bar", "", ""}, + {&CloneDescription{"foo+bar", "", "NotEmpty", ""}, nil}, + {&CloneDescription{"foo+bar", "", "", ""}, errors.New(`clone cluster name must confirm to DNS-1035, regex used for validation is "^[a-z]([-a-z0-9]*[a-z0-9])?$"`)}, - {&CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", ""}, + {&CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", ""}, errors.New("clone cluster name must be no longer than 63 characters")}, - {&CloneDescription{"foobar", "", ""}, nil}, + {&CloneDescription{"foobar", "", "", ""}, nil}, } var maintenanceWindows = []struct { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index a156b89d6..436ae51d1 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1216,10 +1216,37 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) }) } else { // cloning with S3, find out the bucket to clone + msg := "Clone from S3 bucket" + c.logger.Info(msg, description.S3WalPath) + + if description.S3WalPath == "" { + msg := "Figure out which S3 bucket to use from env" + c.logger.Info(msg, description.S3WalPath) + + envs := []v1.EnvVar{ + v1.EnvVar{ + Name: "CLONE_WAL_S3_BUCKET", + Value: c.OpConfig.WALES3Bucket, + }, + v1.EnvVar{ + Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", + Value: getBucketScopeSuffix(description.UID), + }, + } + + result = append(result, envs...) + } else { + msg := "Use custom parsed S3WalPath %s from the manifest" + c.logger.Warningf(msg, description.S3WalPath) + + result = append(result, v1.EnvVar{ + Name: "CLONE_WALE_S3_PREFIX", + Value: description.S3WalPath, + }) + } + result = append(result, v1.EnvVar{Name: "CLONE_METHOD", Value: "CLONE_WITH_WALE"}) - result = append(result, v1.EnvVar{Name: "CLONE_WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) result = append(result, v1.EnvVar{Name: "CLONE_TARGET_TIME", Value: description.EndTimestamp}) - result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(description.UID)}) result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_PREFIX", Value: ""}) } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 6b96f8ec1..dc48c0389 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -129,3 +129,82 @@ func TestShmVolume(t *testing.T) { } } } + +func TestCloneEnv(t *testing.T) { + testName := "TestCloneEnv" + tests := []struct { + subTest string + cloneOpts *acidv1.CloneDescription + env v1.EnvVar + envPos int + }{ + { + subTest: "custom s3 path", + cloneOpts: &acidv1.CloneDescription{ + ClusterName: "test-cluster", + S3WalPath: "s3://some/path/", + EndTimestamp: "somewhen", + }, + env: v1.EnvVar{ + Name: "CLONE_WALE_S3_PREFIX", + Value: "s3://some/path/", + }, + envPos: 1, + }, + { + subTest: "generated s3 path, bucket", + cloneOpts: &acidv1.CloneDescription{ + ClusterName: "test-cluster", + EndTimestamp: "somewhen", + UID: "0000", + }, + env: v1.EnvVar{ + Name: "CLONE_WAL_S3_BUCKET", + Value: "wale-bucket", + }, + envPos: 1, + }, + { + subTest: "generated s3 path, target time", + cloneOpts: &acidv1.CloneDescription{ + ClusterName: "test-cluster", + EndTimestamp: "somewhen", + UID: "0000", + }, + env: v1.EnvVar{ + Name: "CLONE_TARGET_TIME", + Value: "somewhen", + }, + envPos: 4, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + WALES3Bucket: "wale-bucket", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + for _, tt := range tests { + envs := cluster.generateCloneEnvironment(tt.cloneOpts) + + env := envs[tt.envPos] + + if env.Name != tt.env.Name { + t.Errorf("%s %s: Expected env name %s, have %s instead", + testName, tt.subTest, tt.env.Name, env.Name) + } + + if env.Value != tt.env.Value { + t.Errorf("%s %s: Expected env value %s, have %s instead", + testName, tt.subTest, tt.env.Value, env.Value) + } + + } +}