Connection pooler (#799)

Connection pooler support

Add support for a connection pooler. The idea is to make it generic enough to
be able to switch between different implementations (e.g. pgbouncer or
odyssey). Operator needs to create a deployment with pooler and a service for
it to access.

For connection pool to work properly, a database needs to be prepared by
operator, namely a separate user have to be created with an access to an
installed lookup function (to fetch credential for other users).

This setups is supposed to be used only by robot/application users. Usually a
connection pool implementation is more CPU bounded, so it makes sense to create
several pods for connection pool with more emphasize on cpu resources. At the
moment there are no special affinity or tolerations assigned to bring those
pods closer to the database. For availability purposes minimal number of
connection pool pods is 2, ideally they have to be distributed between
different nodes/AZ, but it's not enforced in the operator itself. Available
configuration supposed to be ergonomic and in the normal case require minimum
changes to a manifest to enable connection pool. To have more control over the
configuration and functionality on the pool side one can customize the
corresponding docker image.

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>

pkg/apis/acid.zalan.do/v1/crds.go

@@ -105,6 +105,7 @@ var OperatorConfigCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefin
 
 var min0 = 0.0
 var min1 = 1.0
+var min2 = 2.0
 var minDisable = -1.0
 
 // PostgresCRDResourceValidation to check applied manifest parameters
@@ -176,6 +177,76 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{
 							},
 						},
 					},
+					"connectionPool": {
+						Type: "object",
+						Properties: map[string]apiextv1beta1.JSONSchemaProps{
+							"dockerImage": {
+								Type: "string",
+							},
+							"maxDBConnections": {
+								Type: "integer",
+							},
+							"mode": {
+								Type: "string",
+								Enum: []apiextv1beta1.JSON{
+									{
+										Raw: []byte(`"session"`),
+									},
+									{
+										Raw: []byte(`"transaction"`),
+									},
+								},
+							},
+							"numberOfInstances": {
+								Type:    "integer",
+								Minimum: &min2,
+							},
+							"resources": {
+								Type:     "object",
+								Required: []string{"requests", "limits"},
+								Properties: map[string]apiextv1beta1.JSONSchemaProps{
+									"limits": {
+										Type:     "object",
+										Required: []string{"cpu", "memory"},
+										Properties: map[string]apiextv1beta1.JSONSchemaProps{
+											"cpu": {
+												Type:        "string",
+												Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0",
+												Pattern:     "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
+											},
+											"memory": {
+												Type:        "string",
+												Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0",
+												Pattern:     "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+											},
+										},
+									},
+									"requests": {
+										Type:     "object",
+										Required: []string{"cpu", "memory"},
+										Properties: map[string]apiextv1beta1.JSONSchemaProps{
+											"cpu": {
+												Type:        "string",
+												Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0",
+												Pattern:     "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
+											},
+											"memory": {
+												Type:        "string",
+												Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0",
+												Pattern:     "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+											},
+										},
+									},
+								},
+							},
+							"schema": {
+								Type: "string",
+							},
+							"user": {
+								Type: "string",
+							},
+						},
+					},
 					"databases": {
 						Type: "object",
 						AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{
@@ -188,6 +259,9 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{
 					"dockerImage": {
 						Type: "string",
 					},
+					"enableConnectionPool": {
+						Type: "boolean",
+					},
 					"enableLogicalBackup": {
 						Type: "boolean",
 					},
@@ -418,7 +492,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{
 						Type: "string",
 					},
 					"tls": {
-						Type: "object",
+						Type:     "object",
 						Required: []string{"secretName"},
 						Properties: map[string]apiextv1beta1.JSONSchemaProps{
 							"secretName": {
@@ -1055,6 +1129,54 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation
 							},
 						},
 					},
+					"connection_pool": {
+						Type: "object",
+						Properties: map[string]apiextv1beta1.JSONSchemaProps{
+							"connection_pool_default_cpu_limit": {
+								Type:    "string",
+								Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
+							},
+							"connection_pool_default_cpu_request": {
+								Type:    "string",
+								Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
+							},
+							"connection_pool_default_memory_limit": {
+								Type:    "string",
+								Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+							},
+							"connection_pool_default_memory_request": {
+								Type:    "string",
+								Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
+							},
+							"connection_pool_image": {
+								Type: "string",
+							},
+							"connection_pool_max_db_connections": {
+								Type: "integer",
+							},
+							"connection_pool_mode": {
+								Type: "string",
+								Enum: []apiextv1beta1.JSON{
+									{
+										Raw: []byte(`"session"`),
+									},
+									{
+										Raw: []byte(`"transaction"`),
+									},
+								},
+							},
+							"connection_pool_number_of_instances": {
+								Type:    "integer",
+								Minimum: &min2,
+							},
+							"connection_pool_schema": {
+								Type: "string",
+							},
+							"connection_pool_user": {
+								Type: "string",
+							},
+						},
+					},
 				},
 			},
 			"status": {

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -1,5 +1,7 @@
 package v1
 
+// Operator configuration CRD definition, please use snake_case for field names.
+
 import (
 	"github.com/zalando/postgres-operator/pkg/util/config"
 
@@ -65,12 +67,12 @@ type KubernetesMetaConfiguration struct {
 	// TODO: use a proper toleration structure?
 	PodToleration map[string]string `json:"toleration,omitempty"`
 	// TODO: use namespacedname
-	PodEnvironmentConfigMap    string        `json:"pod_environment_configmap,omitempty"`
-	PodPriorityClassName       string        `json:"pod_priority_class_name,omitempty"`
-	MasterPodMoveTimeout       Duration      `json:"master_pod_move_timeout,omitempty"`
-	EnablePodAntiAffinity      bool          `json:"enable_pod_antiaffinity,omitempty"`
-	PodAntiAffinityTopologyKey string        `json:"pod_antiaffinity_topology_key,omitempty"`
-	PodManagementPolicy        string        `json:"pod_management_policy,omitempty"`
+	PodEnvironmentConfigMap    string   `json:"pod_environment_configmap,omitempty"`
+	PodPriorityClassName       string   `json:"pod_priority_class_name,omitempty"`
+	MasterPodMoveTimeout       Duration `json:"master_pod_move_timeout,omitempty"`
+	EnablePodAntiAffinity      bool     `json:"enable_pod_antiaffinity,omitempty"`
+	PodAntiAffinityTopologyKey string   `json:"pod_antiaffinity_topology_key,omitempty"`
+	PodManagementPolicy        string   `json:"pod_management_policy,omitempty"`
 }
 
 // PostgresPodResourcesDefaults defines the spec of default resources
@@ -152,6 +154,20 @@ type ScalyrConfiguration struct {
 	ScalyrMemoryLimit   string `json:"scalyr_memory_limit,omitempty"`
 }
 
+// Defines default configuration for connection pool
+type ConnectionPoolConfiguration struct {
+	NumberOfInstances    *int32 `json:"connection_pool_number_of_instances,omitempty"`
+	Schema               string `json:"connection_pool_schema,omitempty"`
+	User                 string `json:"connection_pool_user,omitempty"`
+	Image                string `json:"connection_pool_image,omitempty"`
+	Mode                 string `json:"connection_pool_mode,omitempty"`
+	MaxDBConnections     *int32 `json:"connection_pool_max_db_connections,omitempty"`
+	DefaultCPURequest    string `json:"connection_pool_default_cpu_request,omitempty"`
+	DefaultMemoryRequest string `json:"connection_pool_default_memory_request,omitempty"`
+	DefaultCPULimit      string `json:"connection_pool_default_cpu_limit,omitempty"`
+	DefaultMemoryLimit   string `json:"connection_pool_default_memory_limit,omitempty"`
+}
+
 // OperatorLogicalBackupConfiguration defines configuration for logical backup
 type OperatorLogicalBackupConfiguration struct {
 	Schedule          string `json:"logical_backup_schedule,omitempty"`
@@ -188,6 +204,7 @@ type OperatorConfigurationData struct {
 	LoggingRESTAPI             LoggingRESTAPIConfiguration        `json:"logging_rest_api"`
 	Scalyr                     ScalyrConfiguration                `json:"scalyr"`
 	LogicalBackup              OperatorLogicalBackupConfiguration `json:"logical_backup"`
+	ConnectionPool             ConnectionPoolConfiguration        `json:"connection_pool"`
 }
 
 //Duration shortens this frequently used name

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -1,5 +1,7 @@
 package v1
 
+// Postgres CRD definition, please use CamelCase for field names.
+
 import (
 	"time"
 
@@ -27,6 +29,9 @@ type PostgresSpec struct {
 	Patroni         `json:"patroni,omitempty"`
 	Resources       `json:"resources,omitempty"`
 
+	EnableConnectionPool *bool           `json:"enableConnectionPool,omitempty"`
+	ConnectionPool       *ConnectionPool `json:"connectionPool,omitempty"`
+
 	TeamID      string `json:"teamId"`
 	DockerImage string `json:"dockerImage,omitempty"`
 
@@ -162,3 +167,24 @@ type UserFlags []string
 type PostgresStatus struct {
 	PostgresClusterStatus string `json:"PostgresClusterStatus"`
 }
+
+// Options for connection pooler
+//
+// TODO: prepared snippets of configuration, one can choose via type, e.g.
+// pgbouncer-large (with higher resources) or odyssey-small (with smaller
+// resources)
+// Type              string `json:"type,omitempty"`
+//
+// TODO: figure out what other important parameters of the connection pool it
+// makes sense to expose. E.g. pool size (min/max boundaries), max client
+// connections etc.
+type ConnectionPool struct {
+	NumberOfInstances *int32 `json:"numberOfInstances,omitempty"`
+	Schema            string `json:"schema,omitempty"`
+	User              string `json:"user,omitempty"`
+	Mode              string `json:"mode,omitempty"`
+	DockerImage       string `json:"dockerImage,omitempty"`
+	MaxDBConnections  *int32 `json:"maxDBConnections,omitempty"`
+
+	Resources `json:"resources,omitempty"`
+}

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -68,6 +68,59 @@ func (in *CloneDescription) DeepCopy() *CloneDescription {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ConnectionPool) DeepCopyInto(out *ConnectionPool) {
+	*out = *in
+	if in.NumberOfInstances != nil {
+		in, out := &in.NumberOfInstances, &out.NumberOfInstances
+		*out = new(int32)
+		**out = **in
+	}
+	if in.MaxDBConnections != nil {
+		in, out := &in.MaxDBConnections, &out.MaxDBConnections
+		*out = new(int32)
+		**out = **in
+	}
+	out.Resources = in.Resources
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPool.
+func (in *ConnectionPool) DeepCopy() *ConnectionPool {
+	if in == nil {
+		return nil
+	}
+	out := new(ConnectionPool)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ConnectionPoolConfiguration) DeepCopyInto(out *ConnectionPoolConfiguration) {
+	*out = *in
+	if in.NumberOfInstances != nil {
+		in, out := &in.NumberOfInstances, &out.NumberOfInstances
+		*out = new(int32)
+		**out = **in
+	}
+	if in.MaxDBConnections != nil {
+		in, out := &in.MaxDBConnections, &out.MaxDBConnections
+		*out = new(int32)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPoolConfiguration.
+func (in *ConnectionPoolConfiguration) DeepCopy() *ConnectionPoolConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(ConnectionPoolConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfiguration) {
 	*out = *in
@@ -254,6 +307,7 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData
 	out.LoggingRESTAPI = in.LoggingRESTAPI
 	out.Scalyr = in.Scalyr
 	out.LogicalBackup = in.LogicalBackup
+	in.ConnectionPool.DeepCopyInto(&out.ConnectionPool)
 	return
 }
 
@@ -416,6 +470,16 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 	out.Volume = in.Volume
 	in.Patroni.DeepCopyInto(&out.Patroni)
 	out.Resources = in.Resources
+	if in.EnableConnectionPool != nil {
+		in, out := &in.EnableConnectionPool, &out.EnableConnectionPool
+		*out = new(bool)
+		**out = **in
+	}
+	if in.ConnectionPool != nil {
+		in, out := &in.ConnectionPool, &out.ConnectionPool
+		*out = new(ConnectionPool)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.SpiloFSGroup != nil {
 		in, out := &in.SpiloFSGroup, &out.SpiloFSGroup
 		*out = new(int64)