[UI] add toggle for connection pooler (#953)

* [UI] add toggle for connection pooler

* remove team service logger

* fix new.tag.pug and change port in Makefile

ui/Dockerfile

@@ -1,7 +1,7 @@
 FROM alpine:3.6
 MAINTAINER team-acid@zalando.de
 
-EXPOSE 8080
+EXPOSE 8081
 
 RUN \
   apk add --no-cache \
@@ -29,6 +29,7 @@ RUN \
     /var/cache/apk/*
 
 COPY requirements.txt /
+COPY start_server.sh /
 RUN pip3 install -r /requirements.txt
 
 COPY operator_ui /operator_ui
@@ -37,4 +38,4 @@ ARG VERSION=dev
 RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" /operator_ui/__init__.py
 
 WORKDIR /
-ENTRYPOINT ["/usr/bin/python3", "-m", "operator_ui"]
+CMD ["/usr/bin/python3", "-m", "operator_ui"]

ui/Makefile

@@ -36,4 +36,4 @@ push:
 	docker push "$(IMAGE):$(TAG)$(CDP_TAG)"
 
 mock:
-	docker run -it -p 8080:8080 "$(IMAGE):$(TAG)" --mock
+	docker run -it -p 8081:8081 "$(IMAGE):$(TAG)" --mock

ui/app/src/edit.tag.pug

@@ -137,6 +137,7 @@ edit
         o.spec.numberOfInstances = i.spec.numberOfInstances
         o.spec.enableMasterLoadBalancer = i.spec.enableMasterLoadBalancer || false
         o.spec.enableReplicaLoadBalancer = i.spec.enableReplicaLoadBalancer || false
+        o.spec.enableConnectionPooler = i.spec.enableConnectionPooler || false
         o.spec.volume = { size: i.spec.volume.size }
 
         if ('users' in i.spec && typeof i.spec.users === 'object') {

ui/app/src/new.tag.pug

@@ -239,6 +239,18 @@ new
                   |
                   | Enable replica ELB
 
+            tr
+              td Enable Connection Pool
+              td
+                label
+                  input(
+                    type='checkbox'
+                    value='{ enableConnectionPooler }'
+                    onchange='{ toggleEnableConnectionPooler }'
+                  )
+                  |
+                  | Enable Connection Pool (using PGBouncer)
+
             tr
               td Volume size
               td
@@ -493,6 +505,9 @@ new
         {{#if enableReplicaLoadBalancer}}
         enableReplicaLoadBalancer: true
         {{/if}}
+        {{#if enableConnectionPooler}}
+        enableConnectionPooler: true
+        {{/if}}
         volume:
           size: "{{ volumeSize }}Gi"
         {{#if users}}
@@ -516,13 +531,14 @@ new
           - {{ odd }}/32
         {{/if}}
 
+        {{#if resourcesVisible}}
         resources:
           requests:
             cpu: {{ cpu.state.request.state }}m
             memory: {{ memory.state.request.state }}Mi
           limits:
             cpu: {{ cpu.state.limit.state }}m
-            memory: {{ memory.state.limit.state }}Mi{{#if restoring}}
+            memory: {{ memory.state.limit.state }}Mi{{/if}}{{#if restoring}}
 
         clone:
           cluster: "{{ backup.state.name.state }}"
@@ -542,6 +558,7 @@ new
         instanceCount: this.instanceCount,
         enableMasterLoadBalancer: this.enableMasterLoadBalancer,
         enableReplicaLoadBalancer: this.enableReplicaLoadBalancer,
+        enableConnectionPooler: this.enableConnectionPooler,
         volumeSize: this.volumeSize,
         users: this.users.valids,
         databases: this.databases.valids,
@@ -552,6 +569,7 @@ new
         memory: this.memory,
         backup: this.backup,
         namespace: this.namespace,
+        resourcesVisible: this.config.resources_visible,
         restoring: this.backup.state.type.state !== 'empty',
         pitr: this.backup.state.type.state === 'pitr',
       }
@@ -598,6 +616,10 @@ new
       this.enableReplicaLoadBalancer = !this.enableReplicaLoadBalancer
     }
 
+    this.toggleEnableConnectionPooler = e => {
+      this.enableConnectionPooler = !this.enableConnectionPooler
+    }
+
     this.volumeChange = e => {
       this.volumeSize = +e.target.value
     }
@@ -892,6 +914,7 @@ new
       this.odd = ''
       this.enableMasterLoadBalancer = false
       this.enableReplicaLoadBalancer = false
+      this.enableConnectionPooler = false
 
       this.postgresqlVersion = this.postgresqlVersion = (
         this.config.postgresql_versions[0]

ui/app/src/postgresql.tag.pug

@@ -92,6 +92,8 @@ postgresql
         .alert.alert-success(if='{ progress.masterLabel }') PostgreSQL master available, label is attached
         .alert.alert-success(if='{ progress.masterLabel && progress.dnsName }') PostgreSQL ready: <strong>{ progress.dnsName }</strong>
 
+        .alert.alert-success(if='{ progress.pooler }') Connection pooler deployment created
+
       .col-lg-3
         help-general(config='{ opts.config }')
 
@@ -122,9 +124,11 @@ postgresql
       jQuery.get(
         '/postgresqls/' + this.cluster_path,
       ).done(data => {
+        this.progress.pooler = false
         this.progress.postgresql = true
         this.progress.postgresqlManifest = data
         this.progress.createdTimestamp = data.metadata.creationTimestamp
+        this.progress.poolerEnabled = data.spec.enableConnectionPooler
         this.uid = this.progress.postgresqlManifest.metadata.uid
         this.update()
 
@@ -160,6 +164,11 @@ postgresql
                 this.progress.dnsName = data.metadata.name + '.' + data.metadata.namespace
               }
 
+              jQuery.get('/pooler/' + this.cluster_path).done(data => {
+                this.progress.pooler = {"url": ""}
+                this.update()
+              })
+
               this.update()
             })
           })

ui/manifests/deployment.yaml

@@ -44,6 +44,8 @@ spec:
               value: "http://postgres-operator:8080"
             - name: "OPERATOR_CLUSTER_NAME_LABEL"
               value: "cluster-name"
+            - name: "RESOURCES_VISIBLE"
+              value: "False"
             - name: "TARGET_NAMESPACE"
               value: "default"
             - name: "TEAMS"

ui/manifests/ui-service-account-rbac.yaml

@@ -39,6 +39,7 @@ rules:
 - apiGroups:
   - apps
   resources:
+  - deployments
   - statefulsets
   verbs:
   - get

ui/operator_ui/main.py

@@ -25,7 +25,7 @@ from flask import (
 from flask_oauthlib.client import OAuth
 from functools import wraps
 from gevent import sleep, spawn
-from gevent.wsgi import WSGIServer
+from gevent.pywsgi import WSGIServer
 from jq import jq
 from json import dumps, loads
 from logging import DEBUG, ERROR, INFO, basicConfig, exception, getLogger
@@ -44,6 +44,7 @@ from .spiloutils import (
     create_postgresql,
     read_basebackups,
     read_namespaces,
+    read_pooler,
     read_pods,
     read_postgresql,
     read_postgresqls,
@@ -80,6 +81,7 @@ OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-nam
 OPERATOR_UI_CONFIG = getenv('OPERATOR_UI_CONFIG', '{}')
 OPERATOR_UI_MAINTENANCE_CHECK = getenv('OPERATOR_UI_MAINTENANCE_CHECK', '{}')
 READ_ONLY_MODE = getenv('READ_ONLY_MODE', False) in [True, 'true']
+RESOURCES_VISIBLE = getenv('RESOURCES_VISIBLE', True)
 SPILO_S3_BACKUP_PREFIX = getenv('SPILO_S3_BACKUP_PREFIX', 'spilo/')
 SUPERUSER_TEAM = getenv('SUPERUSER_TEAM', 'acid')
 TARGET_NAMESPACE = getenv('TARGET_NAMESPACE')
@@ -312,6 +314,7 @@ DEFAULT_UI_CONFIG = {
 def get_config():
     config = loads(OPERATOR_UI_CONFIG) or DEFAULT_UI_CONFIG
     config['read_only_mode'] = READ_ONLY_MODE
+    config['resources_visible'] = RESOURCES_VISIBLE
     config['superuser_team'] = SUPERUSER_TEAM
     config['target_namespace'] = TARGET_NAMESPACE
 
@@ -397,6 +400,22 @@ def get_service(namespace: str, cluster: str):
     )
 
 
+@app.route('/pooler/<namespace>/<cluster>')
+@authorize
+def get_list_poolers(namespace: str, cluster: str):
+
+    if TARGET_NAMESPACE not in ['', '*', namespace]:
+        return wrong_namespace()
+
+    return respond(
+        read_pooler(
+            get_cluster(),
+            namespace,
+            "{}-pooler".format(cluster),
+        ),
+    )
+
+
 @app.route('/statefulsets/<namespace>/<cluster>')
 @authorize
 def get_list_clusters(namespace: str, cluster: str):
@@ -587,6 +606,17 @@ def update_postgresql(namespace: str, cluster: str):
 
         spec['volume'] = {'size': size}
 
+    if 'enableConnectionPooler' in postgresql['spec']:
+        cp = postgresql['spec']['enableConnectionPooler']
+        if not cp:
+            if 'enableConnectionPooler' in o['spec']:
+                del o['spec']['enableConnectionPooler']
+        else:
+            spec['enableConnectionPooler'] = True
+    else:
+        if 'enableConnectionPooler' in o['spec']:
+            del o['spec']['enableConnectionPooler']
+
     if 'enableReplicaLoadBalancer' in postgresql['spec']:
         rlb = postgresql['spec']['enableReplicaLoadBalancer']
         if not rlb:
@@ -1006,7 +1036,7 @@ def init_cluster():
 def main(port, secret_key, debug, clusters: list):
     global TARGET_NAMESPACE
 
-    basicConfig(level=DEBUG if debug else INFO)
+    basicConfig(stream=sys.stdout, level=(DEBUG if debug else INFO), format='%(asctime)s %(levelname)s: %(message)s',)
 
     init_cluster()
 

ui/operator_ui/spiloutils.py

@@ -1,7 +1,7 @@
 from boto3 import client
 from datetime import datetime, timezone
 from furl import furl
-from json import dumps
+from json import dumps, loads
 from logging import getLogger
 from os import environ, getenv
 from requests import Session
@@ -18,6 +18,15 @@ session = Session()
 
 OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-name')
 
+COMMON_CLUSTER_LABEL = getenv('COMMON_CLUSTER_LABEL', '{"application":"spilo"}')
+COMMON_POOLER_LABEL = getenv('COMMONG_POOLER_LABEL', '{"application":"db-connection-pooler"}')
+
+logger.info("Common Cluster Label: {}".format(COMMON_CLUSTER_LABEL))
+logger.info("Common Pooler Label: {}".format(COMMON_POOLER_LABEL))
+
+COMMON_CLUSTER_LABEL = loads(COMMON_CLUSTER_LABEL)
+COMMON_POOLER_LABEL = loads(COMMON_POOLER_LABEL)
+
 
 def request(cluster, path, **kwargs):
     if 'timeout' not in kwargs:
@@ -85,6 +94,7 @@ def resource_api_version(resource_type):
     return {
         'postgresqls': 'apis/acid.zalan.do/v1',
         'statefulsets': 'apis/apps/v1',
+        'deployments': 'apis/apps/v1',
     }.get(resource_type, 'api/v1')
 
 
@@ -149,7 +159,7 @@ def read_pod(cluster, namespace, resource_name):
         resource_type='pods',
         namespace=namespace,
         resource_name=resource_name,
-        label_selector={'application': 'spilo'},
+        label_selector=COMMON_CLUSTER_LABEL,
     )
 
 
@@ -159,7 +169,17 @@ def read_service(cluster, namespace, resource_name):
         resource_type='services',
         namespace=namespace,
         resource_name=resource_name,
-        label_selector={'application': 'spilo'},
+        label_selector=COMMON_CLUSTER_LABEL,
+    )
+
+
+def read_pooler(cluster, namespace, resource_name):
+    return kubernetes_get(
+        cluster=cluster,
+        resource_type='deployments',
+        namespace=namespace,
+        resource_name=resource_name,
+        label_selector=COMMON_POOLER_LABEL,
     )
 
 
@@ -169,7 +189,7 @@ def read_statefulset(cluster, namespace, resource_name):
         resource_type='statefulsets',
         namespace=namespace,
         resource_name=resource_name,
-        label_selector={'application': 'spilo'},
+        label_selector=COMMON_CLUSTER_LABEL,
     )
 
 

ui/requirements.txt

@@ -1,5 +1,5 @@
 Flask-OAuthlib==0.9.5
-Flask==1.1.1
+Flask==1.1.2
 backoff==1.8.1
 boto3==1.10.4
 boto==2.49.0

ui/start_server.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/bin/python3 -m operator_ui