Cleanup configuration

Add pool configuration into CRD & charts. Add preliminary documentation.
Rename NumberOfInstances to Replicas like in Deployment. Mention couple
of potential improvement points for connection pool specification.

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -321,6 +321,24 @@ spec:
             connection_pool:
               type: object
               properties:
+                connection_pool_schema:
+                  type: string
+                  #default: "pooler"
+                connection_pool_user:
+                  type: string
+                  #default: "pooler"
+                connection_pool_instances_number:
+                  type: integer
+                  #default: 1
+                connection_pool_image:
+                  type: string
+                  #default: "registry.opensource.zalan.do/acid/pgbouncer:1.0.0"
+                connection_pool_mode:
+                  type: string
+                  enum:
+                    - "session"
+                    - "transaction"
+                  #default: "transaction"
                 connection_pool_default_cpu_limit:
                   type: string
                   pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
@@ -336,25 +354,7 @@ spec:
                 connection_pool_default_memory_request:
                   type: string
                   pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
-                  #default: "100Mi"
-                connection_pool_image:
-                  type: string
-                  #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1"
-                connection_pool_instances_number:
-                  type: integer
-                  #default: 1
-                connection_pool_mode:
-                  type: string
-                  enum:
-                    - "session"
-                    - "transaction"
-                  #default: "transaction"
-                connection_pool_schema:
-                  type: string
-                  #default: "pooler"
-                connection_pool_user:
-                  type: string
-                  #default: "pooler"
+                  #default: "100m"
         status:
           type: object
           additionalProperties:

charts/postgres-operator/crds/postgresqls.yaml

@@ -242,6 +242,22 @@ spec:
                     type: string
             replicaLoadBalancer:  # deprecated
               type: boolean
+            connectionPool:
+              type: object
+              properties:
+                schema:
+                  type: string
+                user:
+                  type: string
+                replicas:
+                  type: integer
+                dockerImage:
+                  type: string
+                mode:
+                  type: string
+                  enum:
+                    - "session"
+                    - "transaction"
             resources:
               type: object
               required:

charts/postgres-operator/values-crd.yaml

@@ -262,15 +262,21 @@ configScalyr:
   scalyr_memory_request: 50Mi
 
 configConnectionPool:
-    connection_pool_default_cpu_limit: "1"
-    connection_pool_default_cpu_request: "1"
-    connection_pool_default_memory_limit: 100m
-    connection_pool_default_memory_request: "100Mi"
-    # connection_pool_image: ""
-    connection_pool_instances_number: 1
-    connection_pool_mode: "transaction"
-    # connection_pool_schema: "pooler"
-    # connection_pool_user: "pooler"
+  # number of pooler instances
+  connection_pool_replicas: 1
+  # db schema to install lookup function into
+  connection_pool_schema: "pooler"
+  # db user for pooler to use
+  connection_pool_user: "pooler"
+  # docker image
+  connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer"
+  # default pooling mode
+  connection_pool_mode: "transaction"
+  # default resources
+  connection_pool_default_cpu_request: "100m"
+  connection_pool_default_memory_request: "100M"
+  connection_pool_default_cpu_limit: "100m"
+  connection_pool_default_memory_limit: "100M"
 
 rbac:
   # Specifies whether RBAC resources should be created

charts/postgres-operator/values.yaml

@@ -239,15 +239,21 @@ configTeamsApi:
 
 # configure connection pooler deployment created by the operator
 configConnectionPool:
-  connection_pool_default_cpu_limit: "1"
-  connection_pool_default_cpu_request: "1"
-  connection_pool_default_memory_limit: 100m
-  connection_pool_default_memory_request: "100Mi"
-  # connection_pool_image: ""
-  connection_pool_instances_number: 1
+  # number of pooler instances
+  connection_pool_replicas: 1
+  # db schema to install lookup function into
+  connection_pool_schema: "pooler"
+  # db user for pooler to use
+  connection_pool_user: "pooler"
+  # docker image
+  connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer"
+  # default pooling mode
   connection_pool_mode: "transaction"
-  # connection_pool_schema: "pooler"
-  # connection_pool_user: "pooler"
+  # default resources
+  connection_pool_default_cpu_request: "100m"
+  connection_pool_default_memory_request: "100M"
+  connection_pool_default_cpu_limit: "100m"
+  connection_pool_default_memory_limit: "100M"
 
 rbac:
   # Specifies whether RBAC resources should be created

docs/reference/cluster_manifest.md

@@ -149,6 +149,11 @@ These parameters are grouped directly under  the `spec` key in the manifest.
   [the reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule)
   into account. Optional. Default is: "30 00 \* \* \*"
 
+* enableConnectionPool
+  Tells the operator to create a connection pool with a database. If this
+  field is true, a connection pool deployment will be created even if
+  `connectionPool` section is empty.
+
 ## Postgres parameters
 
 Those parameters are grouped under the `postgresql` top-level key, which is
@@ -359,3 +364,25 @@ CPU and memory limits for the sidecar container.
 * **memory**
   memory limits for the sidecar container. Optional, overrides the
   `default_memory_limits` operator configuration parameter. Optional.
+
+## Connection pool
+
+Parameters are grouped under the `connectionPool` top-level key and specify
+configuration for connection pool. If this section is not empty, a connection
+pool will be created for a database even if `enableConnectionPool` is not
+present.
+
+* **replicas**
+  How many instances of connection pool to create.
+
+* **mode**
+  In which mode to run connection pool, transaction or section.
+
+* **schema**
+  Schema to create for credentials lookup function.
+
+* **user**
+  User to create for connection pool to be able to connect to a database.
+
+* **resources**
+  Resource configuration for connection pool deployment.

docs/reference/operator_parameters.md

@@ -592,3 +592,31 @@ scalyr sidecar. In the CRD-based configuration they are grouped under the
 
 * **scalyr_memory_limit**
   Memory limit value for the Scalyr sidecar. The default is `500Mi`.
+
+## Connection pool configuration
+
+Parameters are grouped under the `connection_pool` top-level key and specify
+default configuration for connection pool, if a postgres manifest requests it
+but do not specify some of the parameters. All of them are optional with the
+operator being able to provide some reasonable defaults.
+
+* **connection_pool_replicas**
+  How many instances of connection pool to create.
+
+* **connection_pool_schema**
+  Schema to create for credentials lookup function.
+
+* **connection_pool_user**
+  User to create for connection pool to be able to connect to a database.
+
+* **connection_pool_image**
+  Docker image to use for connection pool deployment.
+
+* **connection_pool_mode**
+  Default pool mode, sesssion or transaction.
+
+* **connection_pool_default_cpu_request**
+  **connection_pool_default_memory_reques**
+  **connection_pool_default_cpu_limit**
+  **connection_pool_default_memory_limit**
+  Default resource configuration for connection pool deployment.

docs/user.md

@@ -454,3 +454,56 @@ monitoring is outside the scope of operator responsibilities. See
 [configuration reference](reference/cluster_manifest.md) and
 [administrator documentation](administrator.md) for details on how backups are
 executed.
+
+## Connection pool
+
+The operator can create a database side connection pool for those applications,
+where an application side pool is not feasible, but a number of connections is
+high. To create a connection pool together with a database, modify the
+manifest:
+
+```yaml
+spec:
+  enableConnectionPool: true
+```
+
+This will tell the operator to create a connection pool with default
+configuration, though which one can access the master via a separate service
+`{cluster-name}-pooler`. In most of the cases provided default configuration
+should be good enough.
+
+To configure a new connection pool, specify:
+
+```
+spec:
+  connectionPool:
+    # how many instances of connection pool to create
+    replicas: 1
+
+    # in which mode to run, session or transaction
+    mode: "transaction"
+
+    # schema, which operator will create to install credentials lookup
+    # function
+    schema: "pooler"
+
+    # user, which operator will create for connection pool
+    user: "pooler"
+
+    # resources for each instance
+    resources:
+      requests:
+        cpu: "100m"
+        memory: "100M"
+      limits:
+        cpu: "100m"
+        memory: "100M"
+```
+
+By default `pgbouncer` is used to create a connection pool. To find out about
+pool modes see [docs](https://www.pgbouncer.org/config.html#pool_mode) (but it
+should be general approach between different implementation).
+
+Note, that using `pgbouncer` means meaningful resource CPU limit should be less
+than 1 core (there is a way to utilize more than one, but in K8S it's easier
+just to spin up more instances).

manifests/operatorconfiguration.crd.yaml

@@ -297,6 +297,24 @@ spec:
             connection_pool:
               type: object
               properties:
+                connection_pool_schema:
+                  type: string
+                  #default: "pooler"
+                connection_pool_user:
+                  type: string
+                  #default: "pooler"
+                connection_pool_instances_number:
+                  type: integer
+                  #default: 1
+                connection_pool_image:
+                  type: string
+                  #default: "registry.opensource.zalan.do/acid/pgbouncer:1.0.0"
+                connection_pool_mode:
+                  type: string
+                  enum:
+                    - "session"
+                    - "transaction"
+                  #default: "transaction"
                 connection_pool_default_cpu_limit:
                   type: string
                   pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
@@ -313,24 +331,6 @@ spec:
                   type: string
                   pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
                   #default: "100Mi"
-                connection_pool_image:
-                  type: string
-                  #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1"
-                connection_pool_instances_number:
-                  type: integer
-                  #default: 1
-                connection_pool_mode:
-                  type: string
-                  enum:
-                    - "session"
-                    - "transaction"
-                  #default: "transaction"
-                connection_pool_schema:
-                  type: string
-                  #default: "pooler"
-                connection_pool_user:
-                  type: string
-                  #default: "pooler"
         status:
           type: object
           additionalProperties:

manifests/postgresql.crd.yaml

@@ -206,6 +206,22 @@ spec:
                     type: string
             replicaLoadBalancer:  # deprecated
               type: boolean
+            connectionPool:
+              type: object
+              properties:
+                schema:
+                  type: string
+                user:
+                  type: string
+                replicas:
+                  type: integer
+                dockerImage:
+                  type: string
+                mode:
+                  type: string
+                  enum:
+                    - "session"
+                    - "transaction"
             resources:
               type: object
               required:

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -65,12 +65,12 @@ type KubernetesMetaConfiguration struct {
 	// TODO: use a proper toleration structure?
 	PodToleration map[string]string `json:"toleration,omitempty"`
 	// TODO: use namespacedname
-	PodEnvironmentConfigMap    string        `json:"pod_environment_configmap,omitempty"`
-	PodPriorityClassName       string        `json:"pod_priority_class_name,omitempty"`
-	MasterPodMoveTimeout       Duration      `json:"master_pod_move_timeout,omitempty"`
-	EnablePodAntiAffinity      bool          `json:"enable_pod_antiaffinity,omitempty"`
-	PodAntiAffinityTopologyKey string        `json:"pod_antiaffinity_topology_key,omitempty"`
-	PodManagementPolicy        string        `json:"pod_management_policy,omitempty"`
+	PodEnvironmentConfigMap    string   `json:"pod_environment_configmap,omitempty"`
+	PodPriorityClassName       string   `json:"pod_priority_class_name,omitempty"`
+	MasterPodMoveTimeout       Duration `json:"master_pod_move_timeout,omitempty"`
+	EnablePodAntiAffinity      bool     `json:"enable_pod_antiaffinity,omitempty"`
+	PodAntiAffinityTopologyKey string   `json:"pod_antiaffinity_topology_key,omitempty"`
+	PodManagementPolicy        string   `json:"pod_management_policy,omitempty"`
 }
 
 // PostgresPodResourcesDefaults defines the spec of default resources
@@ -154,10 +154,9 @@ type ScalyrConfiguration struct {
 
 // Defines default configuration for connection pool
 type ConnectionPoolConfiguration struct {
-	NumberOfInstances    *int32 `json:"connection_pool_instances_number,omitempty"`
+	Replicas             *int32 `json:"connection_pool_replicas,omitempty"`
 	Schema               string `json:"connection_pool_schema,omitempty"`
 	User                 string `json:"connection_pool_user,omitempty"`
-	Type                 string `json:"connection_pool_type,omitempty"`
 	Image                string `json:"connection_pool_image,omitempty"`
 	Mode                 string `json:"connection_pool_mode,omitempty"`
 	DefaultCPURequest    string `name:"connection_pool_default_cpu_request,omitempty"`

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -27,7 +27,7 @@ type PostgresSpec struct {
 	Patroni         `json:"patroni,omitempty"`
 	Resources       `json:"resources,omitempty"`
 
-	EnableConnectionPool bool            `json:"enable_connection_pool,omitempty"`
+	EnableConnectionPool bool            `json:"enableConnectionPool,omitempty"`
 	ConnectionPool       *ConnectionPool `json:"connectionPool,omitempty"`
 
 	TeamID      string `json:"teamId"`
@@ -159,16 +159,21 @@ type PostgresStatus struct {
 }
 
 // Options for connection pooler
+//
+// TODO: prepared snippets of configuration, one can choose via type, e.g.
+// pgbouncer-large (with higher resources) or odyssey-small (with smaller
+// resources)
+// Type              string `json:"type,omitempty"`
+//
+// TODO: figure out what other important parameters of the connection pool it
+// makes sense to expose. E.g. pool size (min/max boundaries), max client
+// connections etc.
 type ConnectionPool struct {
-	NumberOfInstances *int32 `json:"instancesNumber,omitempty"`
-	Schema            string `json:"schema,omitempty"`
-	User              string `json:"user,omitempty"`
-	Mode              string `json:"mode,omitempty"`
-	DockerImage       string `json:"dockerImage,omitempty"`
-	// TODO: prepared snippets of configuration, one can choose via type, e.g.
-	// pgbouncer-large (with higher resources) or odyssey-small (with smaller
-	// resources)
-	// Type              string `json:"type,omitempty"`
+	Replicas    *int32 `json:"replicas,omitempty"`
+	Schema      string `json:"schema,omitempty"`
+	User        string `json:"user,omitempty"`
+	Mode        string `json:"mode,omitempty"`
+	DockerImage string `json:"dockerImage,omitempty"`
 
 	Resources `json:"resources,omitempty"`
 }

pkg/cluster/k8sres.go

@@ -1861,9 +1861,9 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) (
 	*appsv1.Deployment, error) {
 
 	podTemplate, err := c.generateConnPoolPodTemplate(spec)
-	numberOfInstances := spec.ConnectionPool.NumberOfInstances
+	numberOfInstances := spec.ConnectionPool.Replicas
 	if numberOfInstances == nil {
-		numberOfInstances = c.OpConfig.ConnectionPool.NumberOfInstances
+		numberOfInstances = c.OpConfig.ConnectionPool.Replicas
 	}
 
 	if err != nil {

pkg/cluster/sync_test.go

@@ -99,14 +99,14 @@ func TestConnPoolSynchronization(t *testing.T) {
 			oldSpec: &acidv1.Postgresql{
 				Spec: acidv1.PostgresSpec{
 					ConnectionPool: &acidv1.ConnectionPool{
-						NumberOfInstances: int32ToPointer(1),
+						Replicas: int32ToPointer(1),
 					},
 				},
 			},
 			newSpec: &acidv1.Postgresql{
 				Spec: acidv1.PostgresSpec{
 					ConnectionPool: &acidv1.ConnectionPool{
-						NumberOfInstances: int32ToPointer(2),
+						Replicas: int32ToPointer(2),
 					},
 				},
 			},

pkg/controller/operator_config.go

@@ -146,13 +146,13 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 
 	// Connection pool. Looks like we can't use defaulting in CRD before 1.17,
 	// so ensure default values here.
-	result.ConnectionPool.NumberOfInstances = fromCRD.ConnectionPool.NumberOfInstances
-	if result.ConnectionPool.NumberOfInstances == nil ||
-		*result.ConnectionPool.NumberOfInstances < 1 {
+	result.ConnectionPool.Replicas = fromCRD.ConnectionPool.Replicas
+	if result.ConnectionPool.Replicas == nil ||
+		*result.ConnectionPool.Replicas < 1 {
 		var value int32
 
 		value = 1
-		result.ConnectionPool.NumberOfInstances = &value
+		result.ConnectionPool.Replicas = &value
 	}
 
 	result.ConnectionPool.Schema = util.Coalesce(
@@ -163,10 +163,6 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 		fromCRD.ConnectionPool.User,
 		constants.ConnectionPoolUserName)
 
-	result.ConnectionPool.Type = util.Coalesce(
-		fromCRD.ConnectionPool.Type,
-		constants.ConnectionPoolDefaultType)
-
 	result.ConnectionPool.Image = util.Coalesce(
 		fromCRD.ConnectionPool.Image,
 		"pgbouncer:0.0.1")

pkg/util/config/config.go

@@ -85,10 +85,9 @@ type LogicalBackup struct {
 
 // Operator options for connection pooler
 type ConnectionPool struct {
-	NumberOfInstances            *int32 `name:"connection_pool_instances_number" default:"1"`
+	Replicas                     *int32 `name:"connection_pool_replicas" default:"1"`
 	Schema                       string `name:"connection_pool_schema" default:"pooler"`
 	User                         string `name:"connection_pool_user" default:"pooler"`
-	Type                         string `name:"connection_pool_type" default:"pgbouncer"`
 	Image                        string `name:"connection_pool_image"  default:"pgbouncer:1.0"`
 	Mode                         string `name:"connection_pool_mode"  default:"session"`
 	ConnPoolDefaultCPURequest    string `name:"connection_pool_default_cpu_request" default:"100m"`