From 865d5b41a75a98bbc6c015b67a5f228f0dd2e652 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 29 Apr 2020 17:26:46 +0200 Subject: [PATCH 1/3] set event broadcasting to Infof and update rbac (#952) --- .../templates/clusterrole.yaml | 5 +++++ docs/user.md | 19 +++++++++++++++---- manifests/operator-service-account-rbac.yaml | 5 +++++ pkg/controller/controller.go | 2 +- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 0defcab41..bd34e803e 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -49,6 +49,11 @@ rules: - events verbs: - create + - get + - list + - patch + - update + - watch # to manage endpoints which are also used by Patroni - apiGroups: - "" diff --git a/docs/user.md b/docs/user.md index 2d9f2be6a..3683fdf61 100644 --- a/docs/user.md +++ b/docs/user.md @@ -53,8 +53,19 @@ them. ## Watch pods being created +Check if the database pods are coming up. Use the label `application=spilo` to +filter and list the label `spilo-role` to see when the master is promoted and +replicas get their labels. + ```bash -kubectl get pods -w --show-labels +kubectl get pods -l application=spilo -L spilo-role -w +``` + +The operator also emits K8s events to the Postgresql CRD which can be inspected +in the operator logs or with: + +```bash +kubectl describe postgresql acid-minimal-cluster ``` ## Connect to PostgreSQL @@ -736,14 +747,14 @@ spin up more instances). ## Custom TLS certificates -By default, the spilo image generates its own TLS certificate during startup. +By default, the Spilo image generates its own TLS certificate during startup. However, this certificate cannot be verified and thus doesn't protect from active MITM attacks. In this section we show how to specify a custom TLS certificate which is mounted in the database pods via a K8s Secret. Before applying these changes, in k8s the operator must also be configured with the `spilo_fsgroup` set to the GID matching the postgres user group. If you -don't know the value, use `103` which is the GID from the default spilo image +don't know the value, use `103` which is the GID from the default Spilo image (`spilo_fsgroup=103` in the cluster request spec). OpenShift allocates the users and groups dynamically (based on scc), and their @@ -805,5 +816,5 @@ spec: Alternatively, it is also possible to use [cert-manager](https://cert-manager.io/docs/) to generate these secrets. -Certificate rotation is handled in the spilo image which checks every 5 +Certificate rotation is handled in the Spilo image which checks every 5 minutes if the certificates have changed and reloads postgres accordingly. diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 667941a24..266df30c5 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -50,6 +50,11 @@ rules: - events verbs: - create + - get + - list + - patch + - update + - watch # to manage endpoints which are also used by Patroni - apiGroups: - "" diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 0b3fde5d9..26b6b1b87 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -76,7 +76,7 @@ func NewController(controllerConfig *spec.ControllerConfig, controllerId string) } eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(logger.Debugf) + eventBroadcaster.StartLogging(logger.Infof) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: myComponentName}) c := &Controller{ From 5af4379118a41664f404a95c40ff17a872f15e04 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 30 Apr 2020 09:58:07 +0200 Subject: [PATCH 2/3] [UI] add toggle for connection pooler (#953) * [UI] add toggle for connection pooler * remove team service logger * fix new.tag.pug and change port in Makefile --- ui/Dockerfile | 5 ++-- ui/Makefile | 2 +- ui/app/src/edit.tag.pug | 1 + ui/app/src/new.tag.pug | 25 ++++++++++++++++- ui/app/src/postgresql.tag.pug | 9 ++++++ ui/manifests/deployment.yaml | 2 ++ ui/manifests/ui-service-account-rbac.yaml | 1 + ui/operator_ui/main.py | 34 +++++++++++++++++++++-- ui/operator_ui/spiloutils.py | 28 ++++++++++++++++--- ui/requirements.txt | 2 +- ui/start_server.sh | 2 ++ 11 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 ui/start_server.sh diff --git a/ui/Dockerfile b/ui/Dockerfile index 3e1ae8756..5ea912dbc 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:3.6 MAINTAINER team-acid@zalando.de -EXPOSE 8080 +EXPOSE 8081 RUN \ apk add --no-cache \ @@ -29,6 +29,7 @@ RUN \ /var/cache/apk/* COPY requirements.txt / +COPY start_server.sh / RUN pip3 install -r /requirements.txt COPY operator_ui /operator_ui @@ -37,4 +38,4 @@ ARG VERSION=dev RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" /operator_ui/__init__.py WORKDIR / -ENTRYPOINT ["/usr/bin/python3", "-m", "operator_ui"] +CMD ["/usr/bin/python3", "-m", "operator_ui"] diff --git a/ui/Makefile b/ui/Makefile index e7d5df674..29c8d9409 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -36,4 +36,4 @@ push: docker push "$(IMAGE):$(TAG)$(CDP_TAG)" mock: - docker run -it -p 8080:8080 "$(IMAGE):$(TAG)" --mock + docker run -it -p 8081:8081 "$(IMAGE):$(TAG)" --mock diff --git a/ui/app/src/edit.tag.pug b/ui/app/src/edit.tag.pug index 9029594bd..c1d94e589 100644 --- a/ui/app/src/edit.tag.pug +++ b/ui/app/src/edit.tag.pug @@ -137,6 +137,7 @@ edit o.spec.numberOfInstances = i.spec.numberOfInstances o.spec.enableMasterLoadBalancer = i.spec.enableMasterLoadBalancer || false o.spec.enableReplicaLoadBalancer = i.spec.enableReplicaLoadBalancer || false + o.spec.enableConnectionPooler = i.spec.enableConnectionPooler || false o.spec.volume = { size: i.spec.volume.size } if ('users' in i.spec && typeof i.spec.users === 'object') { diff --git a/ui/app/src/new.tag.pug b/ui/app/src/new.tag.pug index fe9d78226..6293a6c7a 100644 --- a/ui/app/src/new.tag.pug +++ b/ui/app/src/new.tag.pug @@ -239,6 +239,18 @@ new | | Enable replica ELB + tr + td Enable Connection Pool + td + label + input( + type='checkbox' + value='{ enableConnectionPooler }' + onchange='{ toggleEnableConnectionPooler }' + ) + | + | Enable Connection Pool (using PGBouncer) + tr td Volume size td @@ -493,6 +505,9 @@ new {{#if enableReplicaLoadBalancer}} enableReplicaLoadBalancer: true {{/if}} + {{#if enableConnectionPooler}} + enableConnectionPooler: true + {{/if}} volume: size: "{{ volumeSize }}Gi" {{#if users}} @@ -516,13 +531,14 @@ new - {{ odd }}/32 {{/if}} + {{#if resourcesVisible}} resources: requests: cpu: {{ cpu.state.request.state }}m memory: {{ memory.state.request.state }}Mi limits: cpu: {{ cpu.state.limit.state }}m - memory: {{ memory.state.limit.state }}Mi{{#if restoring}} + memory: {{ memory.state.limit.state }}Mi{{/if}}{{#if restoring}} clone: cluster: "{{ backup.state.name.state }}" @@ -542,6 +558,7 @@ new instanceCount: this.instanceCount, enableMasterLoadBalancer: this.enableMasterLoadBalancer, enableReplicaLoadBalancer: this.enableReplicaLoadBalancer, + enableConnectionPooler: this.enableConnectionPooler, volumeSize: this.volumeSize, users: this.users.valids, databases: this.databases.valids, @@ -552,6 +569,7 @@ new memory: this.memory, backup: this.backup, namespace: this.namespace, + resourcesVisible: this.config.resources_visible, restoring: this.backup.state.type.state !== 'empty', pitr: this.backup.state.type.state === 'pitr', } @@ -598,6 +616,10 @@ new this.enableReplicaLoadBalancer = !this.enableReplicaLoadBalancer } + this.toggleEnableConnectionPooler = e => { + this.enableConnectionPooler = !this.enableConnectionPooler + } + this.volumeChange = e => { this.volumeSize = +e.target.value } @@ -892,6 +914,7 @@ new this.odd = '' this.enableMasterLoadBalancer = false this.enableReplicaLoadBalancer = false + this.enableConnectionPooler = false this.postgresqlVersion = this.postgresqlVersion = ( this.config.postgresql_versions[0] diff --git a/ui/app/src/postgresql.tag.pug b/ui/app/src/postgresql.tag.pug index be7173dbe..9edae99d3 100644 --- a/ui/app/src/postgresql.tag.pug +++ b/ui/app/src/postgresql.tag.pug @@ -92,6 +92,8 @@ postgresql .alert.alert-success(if='{ progress.masterLabel }') PostgreSQL master available, label is attached .alert.alert-success(if='{ progress.masterLabel && progress.dnsName }') PostgreSQL ready: { progress.dnsName } + .alert.alert-success(if='{ progress.pooler }') Connection pooler deployment created + .col-lg-3 help-general(config='{ opts.config }') @@ -122,9 +124,11 @@ postgresql jQuery.get( '/postgresqls/' + this.cluster_path, ).done(data => { + this.progress.pooler = false this.progress.postgresql = true this.progress.postgresqlManifest = data this.progress.createdTimestamp = data.metadata.creationTimestamp + this.progress.poolerEnabled = data.spec.enableConnectionPooler this.uid = this.progress.postgresqlManifest.metadata.uid this.update() @@ -160,6 +164,11 @@ postgresql this.progress.dnsName = data.metadata.name + '.' + data.metadata.namespace } + jQuery.get('/pooler/' + this.cluster_path).done(data => { + this.progress.pooler = {"url": ""} + this.update() + }) + this.update() }) }) diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index 6138ca1a8..ccaecd312 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -44,6 +44,8 @@ spec: value: "http://postgres-operator:8080" - name: "OPERATOR_CLUSTER_NAME_LABEL" value: "cluster-name" + - name: "RESOURCES_VISIBLE" + value: "False" - name: "TARGET_NAMESPACE" value: "default" - name: "TEAMS" diff --git a/ui/manifests/ui-service-account-rbac.yaml b/ui/manifests/ui-service-account-rbac.yaml index 2e09797a0..d4937b5a2 100644 --- a/ui/manifests/ui-service-account-rbac.yaml +++ b/ui/manifests/ui-service-account-rbac.yaml @@ -39,6 +39,7 @@ rules: - apiGroups: - apps resources: + - deployments - statefulsets verbs: - get diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index 5a3054f0e..a294ae081 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -25,7 +25,7 @@ from flask import ( from flask_oauthlib.client import OAuth from functools import wraps from gevent import sleep, spawn -from gevent.wsgi import WSGIServer +from gevent.pywsgi import WSGIServer from jq import jq from json import dumps, loads from logging import DEBUG, ERROR, INFO, basicConfig, exception, getLogger @@ -44,6 +44,7 @@ from .spiloutils import ( create_postgresql, read_basebackups, read_namespaces, + read_pooler, read_pods, read_postgresql, read_postgresqls, @@ -80,6 +81,7 @@ OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-nam OPERATOR_UI_CONFIG = getenv('OPERATOR_UI_CONFIG', '{}') OPERATOR_UI_MAINTENANCE_CHECK = getenv('OPERATOR_UI_MAINTENANCE_CHECK', '{}') READ_ONLY_MODE = getenv('READ_ONLY_MODE', False) in [True, 'true'] +RESOURCES_VISIBLE = getenv('RESOURCES_VISIBLE', True) SPILO_S3_BACKUP_PREFIX = getenv('SPILO_S3_BACKUP_PREFIX', 'spilo/') SUPERUSER_TEAM = getenv('SUPERUSER_TEAM', 'acid') TARGET_NAMESPACE = getenv('TARGET_NAMESPACE') @@ -312,6 +314,7 @@ DEFAULT_UI_CONFIG = { def get_config(): config = loads(OPERATOR_UI_CONFIG) or DEFAULT_UI_CONFIG config['read_only_mode'] = READ_ONLY_MODE + config['resources_visible'] = RESOURCES_VISIBLE config['superuser_team'] = SUPERUSER_TEAM config['target_namespace'] = TARGET_NAMESPACE @@ -397,6 +400,22 @@ def get_service(namespace: str, cluster: str): ) +@app.route('/pooler//') +@authorize +def get_list_poolers(namespace: str, cluster: str): + + if TARGET_NAMESPACE not in ['', '*', namespace]: + return wrong_namespace() + + return respond( + read_pooler( + get_cluster(), + namespace, + "{}-pooler".format(cluster), + ), + ) + + @app.route('/statefulsets//') @authorize def get_list_clusters(namespace: str, cluster: str): @@ -587,6 +606,17 @@ def update_postgresql(namespace: str, cluster: str): spec['volume'] = {'size': size} + if 'enableConnectionPooler' in postgresql['spec']: + cp = postgresql['spec']['enableConnectionPooler'] + if not cp: + if 'enableConnectionPooler' in o['spec']: + del o['spec']['enableConnectionPooler'] + else: + spec['enableConnectionPooler'] = True + else: + if 'enableConnectionPooler' in o['spec']: + del o['spec']['enableConnectionPooler'] + if 'enableReplicaLoadBalancer' in postgresql['spec']: rlb = postgresql['spec']['enableReplicaLoadBalancer'] if not rlb: @@ -1006,7 +1036,7 @@ def init_cluster(): def main(port, secret_key, debug, clusters: list): global TARGET_NAMESPACE - basicConfig(level=DEBUG if debug else INFO) + basicConfig(stream=sys.stdout, level=(DEBUG if debug else INFO), format='%(asctime)s %(levelname)s: %(message)s',) init_cluster() diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py index 33d07d88a..8d1996fb5 100644 --- a/ui/operator_ui/spiloutils.py +++ b/ui/operator_ui/spiloutils.py @@ -1,7 +1,7 @@ from boto3 import client from datetime import datetime, timezone from furl import furl -from json import dumps +from json import dumps, loads from logging import getLogger from os import environ, getenv from requests import Session @@ -18,6 +18,15 @@ session = Session() OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-name') +COMMON_CLUSTER_LABEL = getenv('COMMON_CLUSTER_LABEL', '{"application":"spilo"}') +COMMON_POOLER_LABEL = getenv('COMMONG_POOLER_LABEL', '{"application":"db-connection-pooler"}') + +logger.info("Common Cluster Label: {}".format(COMMON_CLUSTER_LABEL)) +logger.info("Common Pooler Label: {}".format(COMMON_POOLER_LABEL)) + +COMMON_CLUSTER_LABEL = loads(COMMON_CLUSTER_LABEL) +COMMON_POOLER_LABEL = loads(COMMON_POOLER_LABEL) + def request(cluster, path, **kwargs): if 'timeout' not in kwargs: @@ -85,6 +94,7 @@ def resource_api_version(resource_type): return { 'postgresqls': 'apis/acid.zalan.do/v1', 'statefulsets': 'apis/apps/v1', + 'deployments': 'apis/apps/v1', }.get(resource_type, 'api/v1') @@ -149,7 +159,7 @@ def read_pod(cluster, namespace, resource_name): resource_type='pods', namespace=namespace, resource_name=resource_name, - label_selector={'application': 'spilo'}, + label_selector=COMMON_CLUSTER_LABEL, ) @@ -159,7 +169,17 @@ def read_service(cluster, namespace, resource_name): resource_type='services', namespace=namespace, resource_name=resource_name, - label_selector={'application': 'spilo'}, + label_selector=COMMON_CLUSTER_LABEL, + ) + + +def read_pooler(cluster, namespace, resource_name): + return kubernetes_get( + cluster=cluster, + resource_type='deployments', + namespace=namespace, + resource_name=resource_name, + label_selector=COMMON_POOLER_LABEL, ) @@ -169,7 +189,7 @@ def read_statefulset(cluster, namespace, resource_name): resource_type='statefulsets', namespace=namespace, resource_name=resource_name, - label_selector={'application': 'spilo'}, + label_selector=COMMON_CLUSTER_LABEL, ) diff --git a/ui/requirements.txt b/ui/requirements.txt index 5d987416c..7dc49eb3d 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -1,5 +1,5 @@ Flask-OAuthlib==0.9.5 -Flask==1.1.1 +Flask==1.1.2 backoff==1.8.1 boto3==1.10.4 boto==2.49.0 diff --git a/ui/start_server.sh b/ui/start_server.sh new file mode 100644 index 000000000..e2c3980cc --- /dev/null +++ b/ui/start_server.sh @@ -0,0 +1,2 @@ +#!/bin/bash +/usr/bin/python3 -m operator_ui From be208b61f1f795bd79490832dc798cf262ba0cf8 Mon Sep 17 00:00:00 2001 From: Petr Barborka Date: Thu, 30 Apr 2020 17:10:16 +0200 Subject: [PATCH 3/3] Fix S3 backup list (#880) * Fix S3 backup list Co-authored-by: Petr Barborka --- ui/operator_ui/spiloutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py index 8d1996fb5..ea347a84d 100644 --- a/ui/operator_ui/spiloutils.py +++ b/ui/operator_ui/spiloutils.py @@ -322,7 +322,7 @@ def read_basebackups( f=configure_backup_cxt, aws_instance_profile=use_aws_instance_profile, s3_prefix=f's3://{bucket}/{prefix}{pg_cluster}{suffix}/wal/', - )._backup_list(detail=True) + )._backup_list(detail=True)._backup_list(prefix=f"{prefix}{pg_cluster}{suffix}/wal/") ]