do not overwrite clone and standby variables

pkg/cluster/k8sres.go

@@ -872,6 +872,14 @@ func (c *Cluster) generateSpiloPodEnvVars(
 		envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_USE_CONFIGMAPS", Value: "true"})
 	}
 
+	if cloneDescription != nil && cloneDescription.ClusterName != "" {
+		envVars = append(envVars, c.generateCloneEnvironment(cloneDescription)...)
+	}
+
+	if standbyDescription != nil {
+		envVars = append(envVars, c.generateStandbyEnvironment(standbyDescription)...)
+	}
+
 	// fetch cluster-specific variables that will override all subsequent global variables
 	if len(c.Spec.Env) > 0 {
 		envVars = appendEnvVars(envVars, c.Spec.Env...)
@@ -923,14 +931,6 @@ func (c *Cluster) generateSpiloPodEnvVars(
 		opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""})
 	}
 
-	if cloneDescription != nil && cloneDescription.ClusterName != "" {
-		opConfigEnvVars = append(opConfigEnvVars, c.generateCloneEnvironment(cloneDescription)...)
-	}
-
-	if c.Spec.StandbyCluster != nil {
-		opConfigEnvVars = append(opConfigEnvVars, c.generateStandbyEnvironment(standbyDescription)...)
-	}
-
 	envVars = appendEnvVars(envVars, opConfigEnvVars...)
 
 	//sort.Slice(envVars,
@@ -1823,6 +1823,7 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
 	cluster := description.ClusterName
 	result = append(result, v1.EnvVar{Name: "CLONE_SCOPE", Value: cluster})
 	if description.EndTimestamp == "" {
+		c.logger.Infof("cloning with basebackup from %s", cluster)
 		// cloning with basebackup, make a connection string to the cluster to clone from
 		host, port := c.getClusterServiceConnectionParameters(cluster)
 		// TODO: make some/all of those constants
@@ -1844,67 +1845,48 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
 				},
 			})
 	} else {
-		// cloning with S3, find out the bucket to clone
+		c.logger.Info("cloning from WAL location")
-		msg := "clone from S3 bucket"
-		c.logger.Info(msg, description.S3WalPath)
-
 		if description.S3WalPath == "" {
-			msg := "figure out which S3 bucket to use from env"
+			c.logger.Info("no S3 WAL path defined - taking value from global config", description.S3WalPath)
-			c.logger.Info(msg, description.S3WalPath)
 
 			if c.OpConfig.WALES3Bucket != "" {
-				envs := []v1.EnvVar{
+				c.logger.Debugf("found WALES3Bucket %s - will set CLONE_WAL_S3_BUCKET", c.OpConfig.WALES3Bucket)
-					{
+				result = append(result, v1.EnvVar{Name: "CLONE_WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket})
-						Name:  "CLONE_WAL_S3_BUCKET",
-						Value: c.OpConfig.WALES3Bucket,
-					},
-				}
-				result = append(result, envs...)
 			} else if c.OpConfig.WALGSBucket != "" {
-				envs := []v1.EnvVar{
+				c.logger.Debugf("found WALGSBucket %s - will set CLONE_WAL_GS_BUCKET", c.OpConfig.WALGSBucket)
-					{
+				result = append(result, v1.EnvVar{Name: "CLONE_WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket})
-						Name:  "CLONE_WAL_GS_BUCKET",
+				if c.OpConfig.GCPCredentials != "" {
-						Value: c.OpConfig.WALGSBucket,
+					result = append(result, v1.EnvVar{Name: "CLONE_GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials})
-					},
-					{
-						Name:  "CLONE_GOOGLE_APPLICATION_CREDENTIALS",
-						Value: c.OpConfig.GCPCredentials,
-					},
 				}
-				result = append(result, envs...)
 			} else if c.OpConfig.WALAZStorageAccount != "" {
-				envs := []v1.EnvVar{
+				c.logger.Debugf("found WALAZStorageAccount %s - will set CLONE_AZURE_STORAGE_ACCOUNT", c.OpConfig.WALAZStorageAccount)
-					{
+				result = append(result, v1.EnvVar{Name: "CLONE_AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount})
-						Name:  "CLONE_AZURE_STORAGE_ACCOUNT",
-						Value: c.OpConfig.WALAZStorageAccount,
-					},
-				}
-				result = append(result, envs...)
 			} else {
-				c.logger.Error("Cannot figure out S3 or GS bucket. Both are empty.")
+				c.logger.Error("cannot figure out S3 or GS bucket or AZ storage account. All are empty in config.")
+				return result
 			}
 
+			// append suffix because WAL location name is not the whole path
+			result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(description.UID)})
+		} else {
+			c.logger.Debugf("use S3WalPath %s from the manifest", description.S3WalPath)
+
 			envs := []v1.EnvVar{
+				{
+					Name:  "CLONE_WALE_S3_PREFIX",
+					Value: description.S3WalPath,
+				},
 				{
 					Name:  "CLONE_WAL_BUCKET_SCOPE_SUFFIX",
-					Value: getBucketScopeSuffix(description.UID),
+					Value: "",
 				},
 			}
 
 			result = append(result, envs...)
-		} else {
-			msg := "use custom parsed S3WalPath %s from the manifest"
-			c.logger.Warningf(msg, description.S3WalPath)
-
-			result = append(result, v1.EnvVar{
-				Name:  "CLONE_WALE_S3_PREFIX",
-				Value: description.S3WalPath,
-			})
 		}
 
 		result = append(result, v1.EnvVar{Name: "CLONE_METHOD", Value: "CLONE_WITH_WALE"})
 		result = append(result, v1.EnvVar{Name: "CLONE_TARGET_TIME", Value: description.EndTimestamp})
-		result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_PREFIX", Value: ""})
 
 		if description.S3Endpoint != "" {
 			result = append(result, v1.EnvVar{Name: "CLONE_AWS_ENDPOINT", Value: description.S3Endpoint})
@@ -1937,7 +1919,7 @@ func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescript
 	result := make([]v1.EnvVar, 0)
 
 	if description.StandbyHost != "" {
-		// standby from remote primary
+		c.logger.Info("preparing standby streaming from remote primary")
 		result = append(result, v1.EnvVar{
 			Name:  "STANDBY_HOST",
 			Value: description.StandbyHost,
@@ -1949,30 +1931,20 @@ func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescript
 			})
 		}
 	} else {
+		c.logger.Info("preparing standby streaming from WAL location")
 		if description.S3WalPath != "" {
-			// standby with S3, find out the bucket to setup standby
-			msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
-			c.logger.Infof(msg, description.S3WalPath)
-
 			result = append(result, v1.EnvVar{
 				Name:  "STANDBY_WALE_S3_PREFIX",
 				Value: description.S3WalPath,
 			})
 		} else if description.GSWalPath != "" {
-			msg := "Standby from GS bucket using custom parsed GSWalPath from the manifest %s "
+			result = append(result, v1.EnvVar{
-			c.logger.Infof(msg, description.GSWalPath)
+				Name:  "STANDBY_WALE_GS_PREFIX",
-
+				Value: description.GSWalPath,
-			envs := []v1.EnvVar{
+			})
-				{
+		} else {
-					Name:  "STANDBY_WALE_GS_PREFIX",
+			c.logger.Error("no WAL path specified in standby section")
-					Value: description.GSWalPath,
+			return result
-				},
-				{
-					Name:  "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
-					Value: c.OpConfig.GCPCredentials,
-				},
-			}
-			result = append(result, envs...)
 		}
 
 		result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"})

pkg/cluster/k8sres_test.go

@@ -185,9 +185,9 @@ func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOpt
 	secret := &v1.Secret{}
 	secret.Name = testPodEnvironmentSecretName
 	secret.Data = map[string][]byte{
-		"clone_s3_access_key_id": []byte("0123456789abcdef0123456789abcdef"),
+		"clone_aws_access_key_id":                []byte("0123456789abcdef0123456789abcdef"),
-		"custom_variable":        []byte("secret-test"),
+		"custom_variable":                        []byte("secret-test"),
-		"s3_access_key_id":       []byte("0123456789abcdef0123456789abcdef"),
+		"standby_google_application_credentials": []byte("0123456789abcdef0123456789abcdef"),
 	}
 	return secret, nil
 }
@@ -199,14 +199,12 @@ func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.Get
 	configmap := &v1.ConfigMap{}
 	configmap.Name = testPodEnvironmentConfigMapName
 	configmap.Data = map[string]string{
-		// hard-coded clone env variable, can be overridden
+		// hard-coded clone env variable, can set when not specified in manifest
 		"clone_aws_endpoint": "s3.eu-west-1.amazonaws.com",
 		// custom variable, can be overridden by c.Spec.Env
 		"custom_variable": "configmap-test",
 		// hard-coded env variable, can not be overridden
 		"kubernetes_scope_label": "pgaas",
-		// hard-coded standby env variable, can be overridden
-		"standby_wal_bucket_scope_prefix": "my-prefix",
 		// hard-coded env variable, can be overridden
 		"wal_s3_bucket": "global-s3-bucket-configmap",
 	}
@@ -288,10 +286,6 @@ func TestPodEnvironmentConfigMapVariables(t *testing.T) {
 					Name:  "kubernetes_scope_label",
 					Value: "pgaas",
 				},
-				{
-					Name:  "standby_wal_bucket_scope_prefix",
-					Value: "my-prefix",
-				},
 				{
 					Name:  "wal_s3_bucket",
 					Value: "global-s3-bucket-configmap",
@@ -369,13 +363,13 @@ func TestPodEnvironmentSecretVariables(t *testing.T) {
 			},
 			envVars: []v1.EnvVar{
 				{
-					Name: "clone_s3_access_key_id",
+					Name: "clone_aws_access_key_id",
 					ValueFrom: &v1.EnvVarSource{
 						SecretKeyRef: &v1.SecretKeySelector{
 							LocalObjectReference: v1.LocalObjectReference{
 								Name: testPodEnvironmentSecretName,
 							},
-							Key: "clone_s3_access_key_id",
+							Key: "clone_aws_access_key_id",
 						},
 					},
 				},
@@ -391,13 +385,13 @@ func TestPodEnvironmentSecretVariables(t *testing.T) {
 					},
 				},
 				{
-					Name: "s3_access_key_id",
+					Name: "standby_google_application_credentials",
 					ValueFrom: &v1.EnvVarSource{
 						SecretKeyRef: &v1.SecretKeySelector{
 							LocalObjectReference: v1.LocalObjectReference{
 								Name: testPodEnvironmentSecretName,
 							},
-							Key: "s3_access_key_id",
+							Key: "standby_google_application_credentials",
 						},
 					},
 				},
@@ -512,7 +506,7 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 	}
 	expectedS3BucketConfigMap := []ExpectedValue{
 		{
-			envIndex:       18,
+			envIndex:       17,
 			envVarConstant: "WAL_S3_BUCKET",
 			envVarValue:    "global-s3-bucket-configmap",
 		},
@@ -552,18 +546,71 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 			envVarValue:    "spec-env-test",
 		},
 	}
+	expectedCloneEnvSpec := []ExpectedValue{
+		{
+			envIndex:       16,
+			envVarConstant: "CLONE_WALE_S3_PREFIX",
+			envVarValue:    "s3://another-bucket",
+		},
+		{
+			envIndex:       17,
+			envVarConstant: "CLONE_WAL_BUCKET_SCOPE_SUFFIX",
+			envVarValue:    "",
+		},
+		{
+			envIndex:       20,
+			envVarConstant: "CLONE_AWS_ENDPOINT",
+			envVarValue:    "s3.eu-central-1.amazonaws.com",
+		},
+	}
 	expectedCloneEnvConfigMap := []ExpectedValue{
 		{
-			envIndex:       15,
+			envIndex:       16,
+			envVarConstant: "CLONE_WAL_S3_BUCKET",
+			envVarValue:    "global-s3-bucket",
+		},
+		{
+			envIndex:       17,
+			envVarConstant: "CLONE_WAL_BUCKET_SCOPE_SUFFIX",
+			envVarValue:    fmt.Sprintf("/%s", dummyUUID),
+		},
+		{
+			envIndex:       20,
 			envVarConstant: "CLONE_AWS_ENDPOINT",
 			envVarValue:    "s3.eu-west-1.amazonaws.com",
 		},
 	}
-	expectedStandbyEnvConfigMap := []ExpectedValue{
+	expectedCloneEnvSecret := []ExpectedValue{
 		{
-			envIndex:       17,
+			envIndex:       20,
-			envVarConstant: "STANDBY_WAL_BUCKET_SCOPE_PREFIX",
+			envVarConstant: "CLONE_AWS_ACCESS_KEY_ID",
-			envVarValue:    "my-prefix",
+			envVarValueRef: &v1.EnvVarSource{
+				SecretKeyRef: &v1.SecretKeySelector{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: testPodEnvironmentSecretName,
+					},
+					Key: "clone_aws_access_key_id",
+				},
+			},
+		},
+	}
+	expectedStandbyEnvSecret := []ExpectedValue{
+		{
+			envIndex:       15,
+			envVarConstant: "STANDBY_WALE_GS_PREFIX",
+			envVarValue:    "gs://some/path/",
+		},
+		{
+			envIndex:       20,
+			envVarConstant: "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
+			envVarValueRef: &v1.EnvVarSource{
+				SecretKeyRef: &v1.SecretKeySelector{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: testPodEnvironmentSecretName,
+					},
+					Key: "standby_google_application_credentials",
+				},
+			},
 		},
 	}
 
@@ -636,7 +683,7 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 						Name: testPodEnvironmentConfigMapName,
 					},
 				},
-				WALES3Bucket: "global-s3-bucket-configmap",
+				WALES3Bucket: "global-s3-bucket",
 			},
 			cloneDescription:   &acidv1.CloneDescription{},
 			standbyDescription: &acidv1.StandbyDescription{},
@@ -717,38 +764,76 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
 			},
 		},
 		{
-			subTest: "will override global CLONE_AWS_ENDPOINT parameter from pod environment config map",
+			subTest: "will set CLONE_ parameters from spec and not global config or pod environment config map",
 			opConfig: config.Config{
 				Resources: config.Resources{
 					PodEnvironmentConfigMap: spec.NamespacedName{
 						Name: testPodEnvironmentConfigMapName,
 					},
 				},
-				WALES3Bucket: "global-s3-bucket-configmap",
+				WALES3Bucket: "global-s3-bucket",
 			},
 			cloneDescription: &acidv1.CloneDescription{
 				ClusterName:  "test-cluster",
 				EndTimestamp: "somewhen",
-				UID:          "0000",
+				UID:          dummyUUID,
+				S3WalPath:    "s3://another-bucket",
+				S3Endpoint:   "s3.eu-central-1.amazonaws.com",
+			},
+			standbyDescription: &acidv1.StandbyDescription{},
+			expectedValues:     expectedCloneEnvSpec,
+		},
+		{
+			subTest: "will set CLONE_AWS_ENDPOINT parameter from pod environment config map",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentConfigMap: spec.NamespacedName{
+						Name: testPodEnvironmentConfigMapName,
+					},
+				},
+				WALES3Bucket: "global-s3-bucket",
+			},
+			cloneDescription: &acidv1.CloneDescription{
+				ClusterName:  "test-cluster",
+				EndTimestamp: "somewhen",
+				UID:          dummyUUID,
 			},
 			standbyDescription: &acidv1.StandbyDescription{},
 			expectedValues:     expectedCloneEnvConfigMap,
 		},
 		{
-			subTest: "will override global STANDBY_WAL_BUCKET_SCOPE_PREFIX parameter from pod environment config map",
+			subTest: "will set CLONE_AWS_ACCESS_KEY_ID parameter from pod environment secret",
 			opConfig: config.Config{
 				Resources: config.Resources{
-					PodEnvironmentConfigMap: spec.NamespacedName{
+					PodEnvironmentSecret:  testPodEnvironmentSecretName,
-						Name: testPodEnvironmentConfigMapName,
+					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
-					},
+					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
 				},
-				WALES3Bucket: "global-s3-bucket-configmap",
+				WALES3Bucket: "global-s3-bucket",
+			},
+			cloneDescription: &acidv1.CloneDescription{
+				ClusterName:  "test-cluster",
+				EndTimestamp: "somewhen",
+				UID:          dummyUUID,
+			},
+			standbyDescription: &acidv1.StandbyDescription{},
+			expectedValues:     expectedCloneEnvSecret,
+		},
+		{
+			subTest: "will set STANDBY_GOOGLE_APPLICATION_CREDENTIALS parameter from pod environment secret",
+			opConfig: config.Config{
+				Resources: config.Resources{
+					PodEnvironmentSecret:  testPodEnvironmentSecretName,
+					ResourceCheckInterval: time.Duration(testResourceCheckInterval),
+					ResourceCheckTimeout:  time.Duration(testResourceCheckTimeout),
+				},
+				WALES3Bucket: "global-s3-bucket",
 			},
 			cloneDescription: &acidv1.CloneDescription{},
 			standbyDescription: &acidv1.StandbyDescription{
-				S3WalPath: "primary-bucket",
+				GSWalPath: "gs://some/path/",
 			},
-			expectedValues: expectedStandbyEnvConfigMap,
+			expectedValues: expectedStandbyEnvSecret,
 		},
 	}
 
@@ -881,18 +966,6 @@ func TestStandbyEnv(t *testing.T) {
 			envPos: 0,
 			envLen: 3,
 		},
-		{
-			subTest: "from custom gs path",
-			standbyOpts: &acidv1.StandbyDescription{
-				GSWalPath: "gs://some/path/",
-			},
-			env: v1.EnvVar{
-				Name:  "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
-				Value: "",
-			},
-			envPos: 1,
-			envLen: 4,
-		},
 		{
 			subTest: "ignore gs path if s3 is set",
 			standbyOpts: &acidv1.StandbyDescription{