refactor pooler tls support and set pooler pod security context (#2255)

* bump pooler image * set pooler pod security context * use hard coded RunAsUser 100 and RunAsGroup 101 for pooler pod * unify generation of TLS secret mounts * extend documentation on tls support * add unit test for testing TLS support for pooler * add e2e test for tls support
2026-09-30 15:54:17 +02:00 · 2023-04-17 11:38:56 +02:00
parent 87b7ac0806
commit 0e7beb5fe5
16 changed files with 369 additions and 120 deletions
@@ -3,7 +3,6 @@ package cluster
 import (
 	"context"
 	"fmt"
-	"path/filepath"
 	"strings"
 	"time"

@@ -25,6 +24,9 @@ import (
 	"github.com/zalando/postgres-operator/pkg/util/retryutil"
 )

+var poolerRunAsUser = int64(100)
+var poolerRunAsGroup = int64(101)
+
 // ConnectionPoolerObjects K8s objects that are belong to connection pooler
 type ConnectionPoolerObjects struct {
 	Deployment  *appsv1.Deployment
@@ -261,6 +263,10 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) (
 		makeDefaultConnectionPoolerResources(&c.OpConfig),
 		connectionPoolerContainer)

+	if err != nil {
+		return nil, fmt.Errorf("could not generate resource requirements: %v", err)
+	}
+
 	effectiveDockerImage := util.Coalesce(
 		connectionPoolerSpec.DockerImage,
 		c.OpConfig.ConnectionPooler.Image)
@@ -269,10 +275,6 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) (
 		connectionPoolerSpec.Schema,
 		c.OpConfig.ConnectionPooler.Schema)

-	if err != nil {
-		return nil, fmt.Errorf("could not generate resource requirements: %v", err)
-	}
-
 	secretSelector := func(key string) *v1.SecretKeySelector {
 		effectiveUser := util.Coalesce(
 			connectionPoolerSpec.User,
@@ -344,62 +346,53 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) (
 	//  2. Reference the secret in a volume
 	//  3. Mount the volume to the container at /tls
 	var poolerVolumes []v1.Volume
+	var volumeMounts []v1.VolumeMount
 	if spec.TLS != nil && spec.TLS.SecretName != "" {
-		// Env vars
-		crtFile := spec.TLS.CertificateFile
-		keyFile := spec.TLS.PrivateKeyFile
-		caFile := spec.TLS.CAFile
-		mountPath := "/tls"
-		mountPathCA := mountPath
+		getPoolerTLSEnv := func(k string) string {
+			keyName := ""
+			switch k {
+			case "tls.crt":
+				keyName = "CONNECTION_POOLER_CLIENT_TLS_CRT"
+			case "tls.key":
+				keyName = "CONNECTION_POOLER_CLIENT_TLS_KEY"
+			case "tls.ca":
+				keyName = "CONNECTION_POOLER_CLIENT_CA_FILE"
+			default:
+				panic(fmt.Sprintf("TLS env key for pooler unknown %s", k))
+			}

-		if crtFile == "" {
-			crtFile = "tls.crt"
+			return keyName
 		}
-		if keyFile == "" {
-			keyFile = "tls.key"
+		tlsEnv, tlsVolumes := generateTlsMounts(spec, getPoolerTLSEnv)
+		envVars = append(envVars, tlsEnv...)
+		for _, vol := range tlsVolumes {
+			poolerVolumes = append(poolerVolumes, v1.Volume{
+				Name:         vol.Name,
+				VolumeSource: vol.VolumeSource,
+			})
+			volumeMounts = append(volumeMounts, v1.VolumeMount{
+				Name:      vol.Name,
+				MountPath: vol.MountPath,
+			})
 		}
-		if caFile == "" {
-			caFile = "ca.crt"
-		}
-		if spec.TLS.CASecretName != "" {
-			mountPathCA = mountPath + "ca"
-		}
-
-		envVars = append(
-			envVars,
-			v1.EnvVar{
-				Name: "CONNECTION_POOLER_CLIENT_TLS_CRT", Value: filepath.Join(mountPath, crtFile),
-			},
-			v1.EnvVar{
-				Name: "CONNECTION_POOLER_CLIENT_TLS_KEY", Value: filepath.Join(mountPath, keyFile),
-			},
-			v1.EnvVar{
-				Name: "CONNECTION_POOLER_CLIENT_CA_FILE", Value: filepath.Join(mountPathCA, caFile),
-			},
-		)
-
-		// Volume
-		mode := int32(0640)
-		volume := v1.Volume{
-			Name: "tls",
-			VolumeSource: v1.VolumeSource{
-				Secret: &v1.SecretVolumeSource{
-					SecretName:  spec.TLS.SecretName,
-					DefaultMode: &mode,
-				},
-			},
-		}
-		poolerVolumes = append(poolerVolumes, volume)
-
-		// Mount
-		poolerContainer.VolumeMounts = []v1.VolumeMount{{
-			Name:      "tls",
-			MountPath: "/tls",
-		}}
 	}

 	poolerContainer.Env = envVars
+	poolerContainer.VolumeMounts = volumeMounts
 	tolerationsSpec := tolerations(&spec.Tolerations, c.OpConfig.PodToleration)
+	securityContext := v1.PodSecurityContext{}
+
+	// determine the User, Group and FSGroup for the pooler pod
+	securityContext.RunAsUser = &poolerRunAsUser
+	securityContext.RunAsGroup = &poolerRunAsGroup
+
+	effectiveFSGroup := c.OpConfig.Resources.SpiloFSGroup
+	if spec.SpiloFSGroup != nil {
+		effectiveFSGroup = spec.SpiloFSGroup
+	}
+	if effectiveFSGroup != nil {
+		securityContext.FSGroup = effectiveFSGroup
+	}

 	podTemplate := &v1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
@@ -412,15 +405,10 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) (
 			Containers:                    []v1.Container{poolerContainer},
 			Tolerations:                   tolerationsSpec,
 			Volumes:                       poolerVolumes,
+			SecurityContext:               &securityContext,
 		},
 	}

-	if spec.TLS != nil && spec.TLS.SecretName != "" && spec.SpiloFSGroup != nil {
-		podTemplate.Spec.SecurityContext = &v1.PodSecurityContext{
-			FSGroup: spec.SpiloFSGroup,
-		}
-	}
-
 	nodeAffinity := c.nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity)
 	if c.OpConfig.EnablePodAntiAffinity {
 		labelsSet := labels.Set(c.connectionPoolerLabels(role, false).MatchLabels)
@@ -1,6 +1,7 @@
 package cluster

 import (
+	"context"
 	"errors"
 	"fmt"
 	"strings"
@@ -11,6 +12,7 @@ import (
 	fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake"
 	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/config"
+	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"

 	appsv1 "k8s.io/api/apps/v1"
@@ -19,6 +21,19 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 )

+func newFakeK8sPoolerTestClient() (k8sutil.KubernetesClient, *fake.Clientset) {
+	acidClientSet := fakeacidv1.NewSimpleClientset()
+	clientSet := fake.NewSimpleClientset()
+
+	return k8sutil.KubernetesClient{
+		PodsGetter:         clientSet.CoreV1(),
+		PostgresqlsGetter:  acidClientSet.AcidV1(),
+		StatefulSetsGetter: clientSet.AppsV1(),
+		DeploymentsGetter:  clientSet.AppsV1(),
+		ServicesGetter:     clientSet.CoreV1(),
+	}, clientSet
+}
+
 func mockInstallLookupFunction(schema string, user string) error {
 	return nil
 }
@@ -919,6 +934,122 @@ func testServiceSelector(cluster *Cluster, service *v1.Service, role PostgresRol
 	return nil
 }

+func TestPoolerTLS(t *testing.T) {
+	client, _ := newFakeK8sPoolerTestClient()
+	clusterName := "acid-test-cluster"
+	namespace := "default"
+	tlsSecretName := "my-secret"
+	spiloFSGroup := int64(103)
+	defaultMode := int32(0640)
+	mountPath := "/tls"
+
+	pg := acidv1.Postgresql{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      clusterName,
+			Namespace: namespace,
+		},
+		Spec: acidv1.PostgresSpec{
+			TeamID: "myapp", NumberOfInstances: 1,
+			EnableConnectionPooler: util.True(),
+			Resources: &acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: "1", Memory: "10"},
+			},
+			Volume: acidv1.Volume{
+				Size: "1G",
+			},
+			TLS: &acidv1.TLSDescription{
+				SecretName: tlsSecretName, CAFile: "ca.crt"},
+			AdditionalVolumes: []acidv1.AdditionalVolume{
+				acidv1.AdditionalVolume{
+					Name:      tlsSecretName,
+					MountPath: mountPath,
+					VolumeSource: v1.VolumeSource{
+						Secret: &v1.SecretVolumeSource{
+							SecretName:  tlsSecretName,
+							DefaultMode: &defaultMode,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				PodManagementPolicy: "ordered_ready",
+				ProtectedRoles:      []string{"admin"},
+				Auth: config.Auth{
+					SuperUsername:       superUserName,
+					ReplicationUsername: replicationUserName,
+				},
+				Resources: config.Resources{
+					ClusterLabels:        map[string]string{"application": "spilo"},
+					ClusterNameLabel:     "cluster-name",
+					DefaultCPURequest:    "300m",
+					DefaultCPULimit:      "300m",
+					DefaultMemoryRequest: "300Mi",
+					DefaultMemoryLimit:   "300Mi",
+					PodRoleLabel:         "spilo-role",
+					SpiloFSGroup:         &spiloFSGroup,
+				},
+				ConnectionPooler: config.ConnectionPooler{
+					ConnectionPoolerDefaultCPURequest:    "100m",
+					ConnectionPoolerDefaultCPULimit:      "100m",
+					ConnectionPoolerDefaultMemoryRequest: "100Mi",
+					ConnectionPoolerDefaultMemoryLimit:   "100Mi",
+				},
+			},
+		}, client, pg, logger, eventRecorder)
+
+	// create a statefulset
+	_, err := cluster.createStatefulSet()
+	assert.NoError(t, err)
+
+	// create pooler resources
+	cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{}
+	cluster.ConnectionPooler[Master] = &ConnectionPoolerObjects{
+		Deployment:     nil,
+		Service:        nil,
+		Name:           cluster.connectionPoolerName(Master),
+		ClusterName:    clusterName,
+		Namespace:      namespace,
+		LookupFunction: false,
+		Role:           Master,
+	}
+
+	_, err = cluster.syncConnectionPoolerWorker(nil, &pg, Master)
+	assert.NoError(t, err)
+
+	deploy, err := client.Deployments(namespace).Get(context.TODO(), cluster.connectionPoolerName(Master), metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	fsGroup := int64(103)
+	assert.Equal(t, &fsGroup, deploy.Spec.Template.Spec.SecurityContext.FSGroup, "has a default FSGroup assigned")
+
+	volume := v1.Volume{
+		Name: "my-secret",
+		VolumeSource: v1.VolumeSource{
+			Secret: &v1.SecretVolumeSource{
+				SecretName:  "my-secret",
+				DefaultMode: &defaultMode,
+			},
+		},
+	}
+	assert.Contains(t, deploy.Spec.Template.Spec.Volumes, volume, "the pod gets a secret volume")
+
+	poolerContainer := deploy.Spec.Template.Spec.Containers[constants.ConnectionPoolerContainer]
+	assert.Contains(t, poolerContainer.VolumeMounts, v1.VolumeMount{
+		MountPath: "/tls",
+		Name:      "my-secret",
+	}, "the volume gets mounted in /tls")
+
+	assert.Contains(t, poolerContainer.Env, v1.EnvVar{Name: "CONNECTION_POOLER_CLIENT_TLS_CRT", Value: "/tls/tls.crt"})
+	assert.Contains(t, poolerContainer.Env, v1.EnvVar{Name: "CONNECTION_POOLER_CLIENT_TLS_KEY", Value: "/tls/tls.key"})
+	assert.Contains(t, poolerContainer.Env, v1.EnvVar{Name: "CONNECTION_POOLER_CLIENT_CA_FILE", Value: "/tls/ca.crt"})
+}
+
 func TestConnectionPoolerServiceSpec(t *testing.T) {
 	testName := "Test connection pooler service spec generation"
 	var cluster = New(
@@ -1288,57 +1288,26 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef

 	// configure TLS with a custom secret volume
 	if spec.TLS != nil && spec.TLS.SecretName != "" {
-		// this is combined with the FSGroup in the section above
-		// to give read access to the postgres user
-		defaultMode := int32(0640)
-		mountPath := "/tls"
-		additionalVolumes = append(additionalVolumes, acidv1.AdditionalVolume{
-			Name:      spec.TLS.SecretName,
-			MountPath: mountPath,
-			VolumeSource: v1.VolumeSource{
-				Secret: &v1.SecretVolumeSource{
-					SecretName:  spec.TLS.SecretName,
-					DefaultMode: &defaultMode,
-				},
-			},
-		})
-
-		// use the same filenames as Secret resources by default
-		certFile := ensurePath(spec.TLS.CertificateFile, mountPath, "tls.crt")
-		privateKeyFile := ensurePath(spec.TLS.PrivateKeyFile, mountPath, "tls.key")
-		spiloEnvVars = appendEnvVars(
-			spiloEnvVars,
-			v1.EnvVar{Name: "SSL_CERTIFICATE_FILE", Value: certFile},
-			v1.EnvVar{Name: "SSL_PRIVATE_KEY_FILE", Value: privateKeyFile},
-		)
-
-		if spec.TLS.CAFile != "" {
-			// support scenario when the ca.crt resides in a different secret, diff path
-			mountPathCA := mountPath
-			if spec.TLS.CASecretName != "" {
-				mountPathCA = mountPath + "ca"
+		getSpiloTLSEnv := func(k string) string {
+			keyName := ""
+			switch k {
+			case "tls.crt":
+				keyName = "SSL_CERTIFICATE_FILE"
+			case "tls.key":
+				keyName = "SSL_PRIVATE_KEY_FILE"
+			case "tls.ca":
+				keyName = "SSL_CA_FILE"
+			default:
+				panic(fmt.Sprintf("TLS env key unknown %s", k))
 			}

-			caFile := ensurePath(spec.TLS.CAFile, mountPathCA, "")
-			spiloEnvVars = appendEnvVars(
-				spiloEnvVars,
-				v1.EnvVar{Name: "SSL_CA_FILE", Value: caFile},
-			)
-
-			// the ca file from CASecretName secret takes priority
-			if spec.TLS.CASecretName != "" {
-				additionalVolumes = append(additionalVolumes, acidv1.AdditionalVolume{
-					Name:      spec.TLS.CASecretName,
-					MountPath: mountPathCA,
-					VolumeSource: v1.VolumeSource{
-						Secret: &v1.SecretVolumeSource{
-							SecretName:  spec.TLS.CASecretName,
-							DefaultMode: &defaultMode,
-						},
-					},
-				})
-			}
+			return keyName
 		}
+		tlsEnv, tlsVolumes := generateTlsMounts(spec, getSpiloTLSEnv)
+		for _, env := range tlsEnv {
+			spiloEnvVars = appendEnvVars(spiloEnvVars, env)
+		}
+		additionalVolumes = append(additionalVolumes, tlsVolumes...)
 	}

 	// generate the spilo container
@@ -1492,6 +1461,59 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 	return statefulSet, nil
 }

+func generateTlsMounts(spec *acidv1.PostgresSpec, tlsEnv func(key string) string) ([]v1.EnvVar, []acidv1.AdditionalVolume) {
+	// this is combined with the FSGroup in the section above
+	// to give read access to the postgres user
+	defaultMode := int32(0640)
+	mountPath := "/tls"
+	env := make([]v1.EnvVar, 0)
+	volumes := make([]acidv1.AdditionalVolume, 0)
+
+	volumes = append(volumes, acidv1.AdditionalVolume{
+		Name:      spec.TLS.SecretName,
+		MountPath: mountPath,
+		VolumeSource: v1.VolumeSource{
+			Secret: &v1.SecretVolumeSource{
+				SecretName:  spec.TLS.SecretName,
+				DefaultMode: &defaultMode,
+			},
+		},
+	})
+
+	// use the same filenames as Secret resources by default
+	certFile := ensurePath(spec.TLS.CertificateFile, mountPath, "tls.crt")
+	privateKeyFile := ensurePath(spec.TLS.PrivateKeyFile, mountPath, "tls.key")
+	env = append(env, v1.EnvVar{Name: tlsEnv("tls.crt"), Value: certFile})
+	env = append(env, v1.EnvVar{Name: tlsEnv("tls.key"), Value: privateKeyFile})
+
+	if spec.TLS.CAFile != "" {
+		// support scenario when the ca.crt resides in a different secret, diff path
+		mountPathCA := mountPath
+		if spec.TLS.CASecretName != "" {
+			mountPathCA = mountPath + "ca"
+		}
+
+		caFile := ensurePath(spec.TLS.CAFile, mountPathCA, "")
+		env = append(env, v1.EnvVar{Name: tlsEnv("tls.ca"), Value: caFile})
+
+		// the ca file from CASecretName secret takes priority
+		if spec.TLS.CASecretName != "" {
+			volumes = append(volumes, acidv1.AdditionalVolume{
+				Name:      spec.TLS.CASecretName,
+				MountPath: mountPathCA,
+				VolumeSource: v1.VolumeSource{
+					Secret: &v1.SecretVolumeSource{
+						SecretName:  spec.TLS.CASecretName,
+						DefaultMode: &defaultMode,
+					},
+				},
+			})
+		}
+	}
+
+	return env, volumes
+}
+
 func (c *Cluster) generatePodAnnotations(spec *acidv1.PostgresSpec) map[string]string {
 	annotations := make(map[string]string)
 	for k, v := range c.OpConfig.CustomPodAnnotations {