From 602e914028a6ce9840bdcf7fad2832884d168232 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 13 Apr 2022 11:23:22 +0200 Subject: [PATCH] do not overwrite clone and standby variables --- pkg/cluster/k8sres.go | 108 ++++++++++--------------- pkg/cluster/k8sres_test.go | 159 +++++++++++++++++++++++++++---------- 2 files changed, 156 insertions(+), 111 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index d895d9f37..02251fd8a 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -872,6 +872,14 @@ func (c *Cluster) generateSpiloPodEnvVars( envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_USE_CONFIGMAPS", Value: "true"}) } + if cloneDescription != nil && cloneDescription.ClusterName != "" { + envVars = append(envVars, c.generateCloneEnvironment(cloneDescription)...) + } + + if standbyDescription != nil { + envVars = append(envVars, c.generateStandbyEnvironment(standbyDescription)...) + } + // fetch cluster-specific variables that will override all subsequent global variables if len(c.Spec.Env) > 0 { envVars = appendEnvVars(envVars, c.Spec.Env...) @@ -923,14 +931,6 @@ func (c *Cluster) generateSpiloPodEnvVars( opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""}) } - if cloneDescription != nil && cloneDescription.ClusterName != "" { - opConfigEnvVars = append(opConfigEnvVars, c.generateCloneEnvironment(cloneDescription)...) - } - - if c.Spec.StandbyCluster != nil { - opConfigEnvVars = append(opConfigEnvVars, c.generateStandbyEnvironment(standbyDescription)...) - } - envVars = appendEnvVars(envVars, opConfigEnvVars...) //sort.Slice(envVars, @@ -1823,6 +1823,7 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) cluster := description.ClusterName result = append(result, v1.EnvVar{Name: "CLONE_SCOPE", Value: cluster}) if description.EndTimestamp == "" { + c.logger.Infof("cloning with basebackup from %s", cluster) // cloning with basebackup, make a connection string to the cluster to clone from host, port := c.getClusterServiceConnectionParameters(cluster) // TODO: make some/all of those constants @@ -1844,67 +1845,48 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) }, }) } else { - // cloning with S3, find out the bucket to clone - msg := "clone from S3 bucket" - c.logger.Info(msg, description.S3WalPath) - + c.logger.Info("cloning from WAL location") if description.S3WalPath == "" { - msg := "figure out which S3 bucket to use from env" - c.logger.Info(msg, description.S3WalPath) + c.logger.Info("no S3 WAL path defined - taking value from global config", description.S3WalPath) if c.OpConfig.WALES3Bucket != "" { - envs := []v1.EnvVar{ - { - Name: "CLONE_WAL_S3_BUCKET", - Value: c.OpConfig.WALES3Bucket, - }, - } - result = append(result, envs...) + c.logger.Debugf("found WALES3Bucket %s - will set CLONE_WAL_S3_BUCKET", c.OpConfig.WALES3Bucket) + result = append(result, v1.EnvVar{Name: "CLONE_WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) } else if c.OpConfig.WALGSBucket != "" { - envs := []v1.EnvVar{ - { - Name: "CLONE_WAL_GS_BUCKET", - Value: c.OpConfig.WALGSBucket, - }, - { - Name: "CLONE_GOOGLE_APPLICATION_CREDENTIALS", - Value: c.OpConfig.GCPCredentials, - }, + c.logger.Debugf("found WALGSBucket %s - will set CLONE_WAL_GS_BUCKET", c.OpConfig.WALGSBucket) + result = append(result, v1.EnvVar{Name: "CLONE_WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket}) + if c.OpConfig.GCPCredentials != "" { + result = append(result, v1.EnvVar{Name: "CLONE_GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials}) } - result = append(result, envs...) } else if c.OpConfig.WALAZStorageAccount != "" { - envs := []v1.EnvVar{ - { - Name: "CLONE_AZURE_STORAGE_ACCOUNT", - Value: c.OpConfig.WALAZStorageAccount, - }, - } - result = append(result, envs...) + c.logger.Debugf("found WALAZStorageAccount %s - will set CLONE_AZURE_STORAGE_ACCOUNT", c.OpConfig.WALAZStorageAccount) + result = append(result, v1.EnvVar{Name: "CLONE_AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount}) } else { - c.logger.Error("Cannot figure out S3 or GS bucket. Both are empty.") + c.logger.Error("cannot figure out S3 or GS bucket or AZ storage account. All are empty in config.") + return result } + // append suffix because WAL location name is not the whole path + result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(description.UID)}) + } else { + c.logger.Debugf("use S3WalPath %s from the manifest", description.S3WalPath) + envs := []v1.EnvVar{ + { + Name: "CLONE_WALE_S3_PREFIX", + Value: description.S3WalPath, + }, { Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", - Value: getBucketScopeSuffix(description.UID), + Value: "", }, } result = append(result, envs...) - } else { - msg := "use custom parsed S3WalPath %s from the manifest" - c.logger.Warningf(msg, description.S3WalPath) - - result = append(result, v1.EnvVar{ - Name: "CLONE_WALE_S3_PREFIX", - Value: description.S3WalPath, - }) } result = append(result, v1.EnvVar{Name: "CLONE_METHOD", Value: "CLONE_WITH_WALE"}) result = append(result, v1.EnvVar{Name: "CLONE_TARGET_TIME", Value: description.EndTimestamp}) - result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_PREFIX", Value: ""}) if description.S3Endpoint != "" { result = append(result, v1.EnvVar{Name: "CLONE_AWS_ENDPOINT", Value: description.S3Endpoint}) @@ -1937,7 +1919,7 @@ func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescript result := make([]v1.EnvVar, 0) if description.StandbyHost != "" { - // standby from remote primary + c.logger.Info("preparing standby streaming from remote primary") result = append(result, v1.EnvVar{ Name: "STANDBY_HOST", Value: description.StandbyHost, @@ -1949,30 +1931,20 @@ func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescript }) } } else { + c.logger.Info("preparing standby streaming from WAL location") if description.S3WalPath != "" { - // standby with S3, find out the bucket to setup standby - msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s " - c.logger.Infof(msg, description.S3WalPath) - result = append(result, v1.EnvVar{ Name: "STANDBY_WALE_S3_PREFIX", Value: description.S3WalPath, }) } else if description.GSWalPath != "" { - msg := "Standby from GS bucket using custom parsed GSWalPath from the manifest %s " - c.logger.Infof(msg, description.GSWalPath) - - envs := []v1.EnvVar{ - { - Name: "STANDBY_WALE_GS_PREFIX", - Value: description.GSWalPath, - }, - { - Name: "STANDBY_GOOGLE_APPLICATION_CREDENTIALS", - Value: c.OpConfig.GCPCredentials, - }, - } - result = append(result, envs...) + result = append(result, v1.EnvVar{ + Name: "STANDBY_WALE_GS_PREFIX", + Value: description.GSWalPath, + }) + } else { + c.logger.Error("no WAL path specified in standby section") + return result } result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"}) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 3e8b2fa84..9aef0f4fe 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -185,9 +185,9 @@ func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOpt secret := &v1.Secret{} secret.Name = testPodEnvironmentSecretName secret.Data = map[string][]byte{ - "clone_s3_access_key_id": []byte("0123456789abcdef0123456789abcdef"), - "custom_variable": []byte("secret-test"), - "s3_access_key_id": []byte("0123456789abcdef0123456789abcdef"), + "clone_aws_access_key_id": []byte("0123456789abcdef0123456789abcdef"), + "custom_variable": []byte("secret-test"), + "standby_google_application_credentials": []byte("0123456789abcdef0123456789abcdef"), } return secret, nil } @@ -199,14 +199,12 @@ func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.Get configmap := &v1.ConfigMap{} configmap.Name = testPodEnvironmentConfigMapName configmap.Data = map[string]string{ - // hard-coded clone env variable, can be overridden + // hard-coded clone env variable, can set when not specified in manifest "clone_aws_endpoint": "s3.eu-west-1.amazonaws.com", // custom variable, can be overridden by c.Spec.Env "custom_variable": "configmap-test", // hard-coded env variable, can not be overridden "kubernetes_scope_label": "pgaas", - // hard-coded standby env variable, can be overridden - "standby_wal_bucket_scope_prefix": "my-prefix", // hard-coded env variable, can be overridden "wal_s3_bucket": "global-s3-bucket-configmap", } @@ -288,10 +286,6 @@ func TestPodEnvironmentConfigMapVariables(t *testing.T) { Name: "kubernetes_scope_label", Value: "pgaas", }, - { - Name: "standby_wal_bucket_scope_prefix", - Value: "my-prefix", - }, { Name: "wal_s3_bucket", Value: "global-s3-bucket-configmap", @@ -369,13 +363,13 @@ func TestPodEnvironmentSecretVariables(t *testing.T) { }, envVars: []v1.EnvVar{ { - Name: "clone_s3_access_key_id", + Name: "clone_aws_access_key_id", ValueFrom: &v1.EnvVarSource{ SecretKeyRef: &v1.SecretKeySelector{ LocalObjectReference: v1.LocalObjectReference{ Name: testPodEnvironmentSecretName, }, - Key: "clone_s3_access_key_id", + Key: "clone_aws_access_key_id", }, }, }, @@ -391,13 +385,13 @@ func TestPodEnvironmentSecretVariables(t *testing.T) { }, }, { - Name: "s3_access_key_id", + Name: "standby_google_application_credentials", ValueFrom: &v1.EnvVarSource{ SecretKeyRef: &v1.SecretKeySelector{ LocalObjectReference: v1.LocalObjectReference{ Name: testPodEnvironmentSecretName, }, - Key: "s3_access_key_id", + Key: "standby_google_application_credentials", }, }, }, @@ -512,7 +506,7 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) { } expectedS3BucketConfigMap := []ExpectedValue{ { - envIndex: 18, + envIndex: 17, envVarConstant: "WAL_S3_BUCKET", envVarValue: "global-s3-bucket-configmap", }, @@ -552,18 +546,71 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) { envVarValue: "spec-env-test", }, } + expectedCloneEnvSpec := []ExpectedValue{ + { + envIndex: 16, + envVarConstant: "CLONE_WALE_S3_PREFIX", + envVarValue: "s3://another-bucket", + }, + { + envIndex: 17, + envVarConstant: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", + envVarValue: "", + }, + { + envIndex: 20, + envVarConstant: "CLONE_AWS_ENDPOINT", + envVarValue: "s3.eu-central-1.amazonaws.com", + }, + } expectedCloneEnvConfigMap := []ExpectedValue{ { - envIndex: 15, + envIndex: 16, + envVarConstant: "CLONE_WAL_S3_BUCKET", + envVarValue: "global-s3-bucket", + }, + { + envIndex: 17, + envVarConstant: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", + envVarValue: fmt.Sprintf("/%s", dummyUUID), + }, + { + envIndex: 20, envVarConstant: "CLONE_AWS_ENDPOINT", envVarValue: "s3.eu-west-1.amazonaws.com", }, } - expectedStandbyEnvConfigMap := []ExpectedValue{ + expectedCloneEnvSecret := []ExpectedValue{ { - envIndex: 17, - envVarConstant: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", - envVarValue: "my-prefix", + envIndex: 20, + envVarConstant: "CLONE_AWS_ACCESS_KEY_ID", + envVarValueRef: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "clone_aws_access_key_id", + }, + }, + }, + } + expectedStandbyEnvSecret := []ExpectedValue{ + { + envIndex: 15, + envVarConstant: "STANDBY_WALE_GS_PREFIX", + envVarValue: "gs://some/path/", + }, + { + envIndex: 20, + envVarConstant: "STANDBY_GOOGLE_APPLICATION_CREDENTIALS", + envVarValueRef: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "standby_google_application_credentials", + }, + }, }, } @@ -636,7 +683,7 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) { Name: testPodEnvironmentConfigMapName, }, }, - WALES3Bucket: "global-s3-bucket-configmap", + WALES3Bucket: "global-s3-bucket", }, cloneDescription: &acidv1.CloneDescription{}, standbyDescription: &acidv1.StandbyDescription{}, @@ -717,38 +764,76 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) { }, }, { - subTest: "will override global CLONE_AWS_ENDPOINT parameter from pod environment config map", + subTest: "will set CLONE_ parameters from spec and not global config or pod environment config map", opConfig: config.Config{ Resources: config.Resources{ PodEnvironmentConfigMap: spec.NamespacedName{ Name: testPodEnvironmentConfigMapName, }, }, - WALES3Bucket: "global-s3-bucket-configmap", + WALES3Bucket: "global-s3-bucket", }, cloneDescription: &acidv1.CloneDescription{ ClusterName: "test-cluster", EndTimestamp: "somewhen", - UID: "0000", + UID: dummyUUID, + S3WalPath: "s3://another-bucket", + S3Endpoint: "s3.eu-central-1.amazonaws.com", + }, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCloneEnvSpec, + }, + { + subTest: "will set CLONE_AWS_ENDPOINT parameter from pod environment config map", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{ + ClusterName: "test-cluster", + EndTimestamp: "somewhen", + UID: dummyUUID, }, standbyDescription: &acidv1.StandbyDescription{}, expectedValues: expectedCloneEnvConfigMap, }, { - subTest: "will override global STANDBY_WAL_BUCKET_SCOPE_PREFIX parameter from pod environment config map", + subTest: "will set CLONE_AWS_ACCESS_KEY_ID parameter from pod environment secret", opConfig: config.Config{ Resources: config.Resources{ - PodEnvironmentConfigMap: spec.NamespacedName{ - Name: testPodEnvironmentConfigMapName, - }, + PodEnvironmentSecret: testPodEnvironmentSecretName, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), }, - WALES3Bucket: "global-s3-bucket-configmap", + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{ + ClusterName: "test-cluster", + EndTimestamp: "somewhen", + UID: dummyUUID, + }, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCloneEnvSecret, + }, + { + subTest: "will set STANDBY_GOOGLE_APPLICATION_CREDENTIALS parameter from pod environment secret", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentSecretName, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + WALES3Bucket: "global-s3-bucket", }, cloneDescription: &acidv1.CloneDescription{}, standbyDescription: &acidv1.StandbyDescription{ - S3WalPath: "primary-bucket", + GSWalPath: "gs://some/path/", }, - expectedValues: expectedStandbyEnvConfigMap, + expectedValues: expectedStandbyEnvSecret, }, } @@ -881,18 +966,6 @@ func TestStandbyEnv(t *testing.T) { envPos: 0, envLen: 3, }, - { - subTest: "from custom gs path", - standbyOpts: &acidv1.StandbyDescription{ - GSWalPath: "gs://some/path/", - }, - env: v1.EnvVar{ - Name: "STANDBY_GOOGLE_APPLICATION_CREDENTIALS", - Value: "", - }, - envPos: 1, - envLen: 4, - }, { subTest: "ignore gs path if s3 is set", standbyOpts: &acidv1.StandbyDescription{