Allow cloning clusters from the operator. (#90)

Allow cloning clusters from the operator.

The changes add a new JSON node `clone` with possible values `cluster`
and `timestamp`. `cluster` is mandatory, and setting a non-empty
`timestamp` triggers wal-e point in time recovery. Spilo and Patroni do
the whole heavy-lifting, the operator just defines certain variables and
gathers some data about how to connect to the host to clone or the
target S3 bucket.

As a minor change, set the image pull policy to IfNotPresent instead
of Always to simplify local testing.

Change the default replication username to standby.

pkg/cluster/k8sres.go

@@ -226,7 +226,10 @@ PATRONI_INITDB_PARAMS:
 	return string(result)
 }
 
-func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequirements, pgParameters *spec.PostgresqlParam, patroniParameters *spec.Patroni) *v1.PodTemplateSpec {
+func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequirements,
+	pgParameters *spec.PostgresqlParam,
+	patroniParameters *spec.Patroni,
+	cloneDescription *spec.CloneDescription) *v1.PodTemplateSpec {
 	spiloConfiguration := c.generateSpiloJSONConfiguration(pgParameters, patroniParameters)
 
 	envVars := []v1.EnvVar{
@@ -301,11 +304,17 @@ func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequireme
 	if c.OpConfig.WALES3Bucket != "" {
 		envVars = append(envVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket})
 	}
+	if cloneDescription.ClusterName != "" {
+		cloneVars := c.generateCloneEnvironment(cloneDescription)
+		for _, v := range cloneVars {
+			envVars = append(envVars, v)
+		}
+	}
 	privilegedMode := bool(true)
 	container := v1.Container{
 		Name:            c.containerName(),
 		Image:           c.OpConfig.DockerImage,
-		ImagePullPolicy: v1.PullAlways,
+		ImagePullPolicy: v1.PullIfNotPresent,
 		Resources:       *resourceRequirements,
 		Ports: []v1.ContainerPort{
 			{
@@ -357,13 +366,13 @@ func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequireme
 func (c *Cluster) generateStatefulSet(spec spec.PostgresSpec) (*v1beta1.StatefulSet, error) {
 	resourceRequirements, err := c.resourceRequirements(spec.Resources)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not generate resource requirements: %v", err)
 	}
 
-	podTemplate := c.generatePodTemplate(resourceRequirements, &spec.PostgresqlParam, &spec.Patroni)
+	podTemplate := c.generatePodTemplate(resourceRequirements, &spec.PostgresqlParam, &spec.Patroni, &spec.Clone)
 	volumeClaimTemplate, err := generatePersistentVolumeClaimTemplate(spec.Volume.Size, spec.Volume.StorageClass)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not generate volume claim template: %v", err)
 	}
 
 	statefulSet := &v1beta1.StatefulSet{
@@ -523,3 +532,50 @@ func (c *Cluster) generateMasterEndpoints(subsets []v1.EndpointSubset) *v1.Endpo
 
 	return endpoints
 }
+
+func (c *Cluster) generateCloneEnvironment(description *spec.CloneDescription) []v1.EnvVar {
+	result := make([]v1.EnvVar, 0)
+	if description.ClusterName == "" {
+		return result
+	}
+	cluster := description.ClusterName
+	result = append(result, v1.EnvVar{Name: "CLONE_SCOPE", Value: cluster})
+	if description.EndTimestamp == "" {
+		// cloning with basebackup, make a connection string to the cluster to clone from
+		host, port := c.getClusterServiceConnectionParameters(cluster)
+		// TODO: make some/all of those constants
+		result = append(result, v1.EnvVar{Name: "CLONE_METHOD", Value: "CLONE_WITH_BASEBACKUP"})
+		result = append(result, v1.EnvVar{Name: "CLONE_HOST", Value: host})
+		result = append(result, v1.EnvVar{Name: "CLONE_PORT", Value: port})
+		// TODO: assume replication user name is the same for all clusters, fetch it from secrets otherwise
+		result = append(result, v1.EnvVar{Name: "CLONE_USER", Value: c.OpConfig.ReplicationUsername})
+		result = append(result,
+			v1.EnvVar{Name: "CLONE_PASSWORD",
+				ValueFrom: &v1.EnvVarSource{
+					SecretKeyRef: &v1.SecretKeySelector{
+						LocalObjectReference: v1.LocalObjectReference{
+							Name: c.credentialSecretNameForCluster(c.OpConfig.ReplicationUsername,
+								description.ClusterName),
+						},
+						Key: "password",
+					},
+				},
+			})
+	} else {
+		// cloning with S3, find out the bucket to clone
+		clone_wal_s3_bucket := c.OpConfig.WALES3Bucket
+		result = append(result, v1.EnvVar{Name: "CLONE_METHOD", Value: "CLONE_WITH_WALE"})
+		result = append(result, v1.EnvVar{Name: "CLONE_WAL_S3_BUCKET", Value: clone_wal_s3_bucket})
+		result = append(result, v1.EnvVar{Name: "CLONE_TARGET_TIME", Value: description.EndTimestamp})
+	}
+	return result
+}
+
+// getClusterServiceConnectionParameters fetches cluster host name and port
+// TODO: perhaps we need to query the service (i.e. if non-standard port is used?)
+// TODO: handle clusters in different namespaces
+func (c *Cluster) getClusterServiceConnectionParameters(clusterName string) (host string, port string) {
+	host = clusterName
+	port = "5432"
+	return
+}

pkg/cluster/util.go

@@ -304,11 +304,15 @@ func (c *Cluster) replicaDNSName() string {
 }
 
 func (c *Cluster) credentialSecretName(username string) string {
+	return c.credentialSecretNameForCluster(username, c.Name)
+}
+
+func (c *Cluster) credentialSecretNameForCluster(username string, clusterName string) string {
 	// secret  must consist of lower case alphanumeric characters, '-' or '.',
 	// and must start and end with an alphanumeric character
 	return fmt.Sprintf(constants.UserSecretTemplate,
 		strings.Replace(username, "_", "-", -1),
-		c.Name)
+		clusterName)
 }
 
 func (c *Cluster) podSpiloRole(pod *v1.Pod) string {