From d643ad5e212a5a63d4bc93f01f85ccbfb5e83271 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 29 Mar 2022 15:25:59 +0200 Subject: [PATCH] add code to sync config maps --- .../templates/clusterrole-postgres-pod.yaml | 8 +- .../templates/clusterrole.yaml | 6 - docs/quickstart.md | 14 +- e2e/tests/test_e2e.py | 2 + manifests/configmap.yaml | 2 +- ...erator-service-account-rbac-openshift.yaml | 283 ++++++++++++++++++ pkg/cluster/cluster.go | 27 +- pkg/cluster/k8sres.go | 21 +- pkg/cluster/resources.go | 36 ++- pkg/cluster/sync.go | 40 ++- pkg/cluster/types.go | 2 + pkg/controller/postgresql.go | 3 +- 12 files changed, 409 insertions(+), 35 deletions(-) create mode 100644 manifests/operator-service-account-rbac-openshift.yaml diff --git a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml index 33c43822f..fdccf16d3 100644 --- a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml +++ b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} rules: -# Patroni needs to watch and manage endpoints +# Patroni needs to watch and manage config maps or endpoints {{- if toString .Values.configGeneral.kubernetes_use_configmaps | eq "true" }} - apiGroups: - "" @@ -24,12 +24,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get {{- else }} - apiGroups: - "" diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 8b2e9136e..199086acc 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -89,12 +89,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get {{- else }} # to read configuration from ConfigMaps - apiGroups: diff --git a/docs/quickstart.md b/docs/quickstart.md index ed01367b7..7049a6ef9 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -37,7 +37,7 @@ The Postgres Operator can be deployed in the following ways: * Kustomization * Helm chart -### Manual deployment setup +### Manual deployment setup on Kubernetes The Postgres Operator can be installed simply by applying yaml manifests. Note, we provide the `/manifests` directory as an example only; you should consider @@ -71,6 +71,18 @@ manifest. ./run_operator_locally.sh ``` +### Manual deployment setup on OpenShift + +To install the Postgres Operator in OpenShift you have to change the config +parameter `kubernetes_use_configmaps` to `"true"`. Otherwise, the operator +and Patroni will store leader and config keys in `Endpoints` that are not +supported in OpenShift. This requires also a slightly different set of rules +for the `postgres-operator` and `postgres-pod` cluster roles. + +```bash +oc create -f manifests/operator-service-account-rbac-openshift.yaml +``` + ### Helm chart Alternatively, the operator can be installed by using the provided [Helm](https://helm.sh/) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index b2977b687..0f9515896 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1759,6 +1759,8 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyEqual(lambda: len(k8s.api.custom_objects_api.list_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", label_selector="cluster-name=acid-minimal-cluster")["items"]), 0, "Manifest not deleted") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + # check if everything has been deleted self.eventuallyEqual(lambda: k8s.count_pods_with_label(cluster_label), 0, "Pods not deleted") self.eventuallyEqual(lambda: k8s.count_services_with_label(cluster_label), 0, "Service not deleted") diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 130a35176..8b392544b 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -63,13 +63,13 @@ data: # etcd_host: "" external_traffic_policy: "Cluster" # gcp_credentials: "" - # kubernetes_use_configmaps: "false" # ignored_annotations: "" # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" # infrastructure_roles_secrets: "secretname:monitoring-roles,userkey:user,passwordkey:password,rolekey:inrole" # inherited_annotations: owned-by # inherited_labels: application,environment # kube_iam_role: "" + # kubernetes_use_configmaps: "false" # log_s3_bucket: "" logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1" # logical_backup_google_application_credentials: "" diff --git a/manifests/operator-service-account-rbac-openshift.yaml b/manifests/operator-service-account-rbac-openshift.yaml new file mode 100644 index 000000000..e0e45cc54 --- /dev/null +++ b/manifests/operator-service-account-rbac-openshift.yaml @@ -0,0 +1,283 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: postgres-operator + namespace: default + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: postgres-operator +rules: +# all verbs allowed for custom operator resources +- apiGroups: + - acid.zalan.do + resources: + - postgresqls + - postgresqls/status + - operatorconfigurations + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# operator only reads PostgresTeams +- apiGroups: + - acid.zalan.do + resources: + - postgresteams + verbs: + - get + - list + - watch +# all verbs allowed for event streams (Zalando-internal feature) +# - apiGroups: +# - zalando.org +# resources: +# - fabriceventstreams +# verbs: +# - create +# - delete +# - deletecollection +# - get +# - list +# - patch +# - update +# - watch +# to create or get/update CRDs when starting up +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - patch + - update +# to read configuration and manage ConfigMaps used by Patroni +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# to send events to the CRs +- apiGroups: + - "" + resources: + - events + verbs: + - create + - get + - list + - patch + - update + - watch +# to CRUD secrets for database access +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - update +# to check nodes for node readiness label +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +# to read or delete existing PVCs. Creation via StatefulSet +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - delete + - get + - list + - patch + - update + # to read existing PVs. Creation should be done via dynamic provisioning +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update # only for resizing AWS volumes +# to watch Spilo pods and do rolling updates. Creation via StatefulSet +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - patch + - update + - watch +# to resize the filesystem in Spilo pods when increasing volume size +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create +# to CRUD services to point to Postgres cluster instances +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - patch + - update +# to CRUD the StatefulSet which controls the Postgres cluster instances +- apiGroups: + - apps + resources: + - statefulsets + - deployments + verbs: + - create + - delete + - get + - list + - patch +# to CRUD cron jobs for logical backups +- apiGroups: + - batch + resources: + - cronjobs + verbs: + - create + - delete + - get + - list + - patch + - update +# to get namespaces operator resources can run in +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +# to define PDBs. Update happens via delete/create +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - create + - delete + - get +# to create ServiceAccounts in each namespace the operator watches +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - create +# to create role bindings to the postgres-pod service account +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + verbs: + - get + - create +# to grant privilege to run privileged pods (not needed by default) +#- apiGroups: +# - extensions +# resources: +# - podsecuritypolicies +# resourceNames: +# - privileged +# verbs: +# - use + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: postgres-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: postgres-operator +subjects: +- kind: ServiceAccount + name: postgres-operator + namespace: default + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: postgres-pod +rules: +# Patroni needs to watch and manage config maps +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# Patroni needs to watch pods +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch +# to let Patroni create a headless service +- apiGroups: + - "" + resources: + - services + verbs: + - create +# to grant privilege to run privileged pods (not needed by default) +#- apiGroups: +# - extensions +# resources: +# - podsecuritypolicies +# resourceNames: +# - privileged +# verbs: +# - use diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index c4b7fd831..3475a8958 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -59,6 +59,7 @@ type Config struct { type kubeResources struct { Services map[PostgresRole]*v1.Service Endpoints map[PostgresRole]*v1.Endpoints + ConfigMaps map[PostgresRole]*v1.ConfigMap Secrets map[types.UID]*v1.Secret Statefulset *appsv1.StatefulSet PodDisruptionBudget *policybeta1.PodDisruptionBudget @@ -1484,22 +1485,29 @@ func (c *Cluster) GetCurrentProcess() Process { // GetStatus provides status of the cluster func (c *Cluster) GetStatus() *ClusterStatus { - return &ClusterStatus{ - Cluster: c.Spec.ClusterName, - Team: c.Spec.TeamID, - Status: c.Status, - Spec: c.Spec, - + status := &ClusterStatus{ + Cluster: c.Spec.ClusterName, + Team: c.Spec.TeamID, + Status: c.Status, + Spec: c.Spec, MasterService: c.GetServiceMaster(), ReplicaService: c.GetServiceReplica(), - MasterEndpoint: c.GetEndpointMaster(), - ReplicaEndpoint: c.GetEndpointReplica(), StatefulSet: c.GetStatefulSet(), PodDisruptionBudget: c.GetPodDisruptionBudget(), CurrentProcess: c.GetCurrentProcess(), Error: fmt.Errorf("error: %s", c.Error), } + + if c.patroniKubernetesUseConfigMaps() { + status.MasterEndpoint = c.GetEndpointMaster() + status.ReplicaEndpoint = c.GetEndpointReplica() + } else { + status.MasterConfigMap = c.GetConfigMapMaster() + status.ReplicaConfigMap = c.GetConfigMapReplica() + } + + return status } // Switchover does a switchover (via Patroni) to a candidate pod @@ -1579,10 +1587,11 @@ func (c *Cluster) deletePatroniClusterObjects() error { } if c.patroniKubernetesUseConfigMaps() { - actionsList = append(actionsList, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps) + actionsList = append(actionsList, c.deletePatroniClusterConfigMaps) } else { actionsList = append(actionsList, c.deletePatroniClusterEndpoints) } + actionsList = append(actionsList, c.deletePatroniClusterServices) c.logger.Debugf("removing leftover Patroni objects (endpoints / services and configmaps)") for _, deleter := range actionsList { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 625c6a813..cd47358c0 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -76,13 +76,12 @@ func (c *Cluster) statefulSetName() string { return c.Name } -func (c *Cluster) endpointName(role PostgresRole) string { - name := c.Name - if role == Replica { - name = name + "-repl" - } +func (c *Cluster) configMapName(role PostgresRole) string { + return c.serviceName(role) +} - return name +func (c *Cluster) endpointName(role PostgresRole) string { + return c.serviceName(role) } func (c *Cluster) serviceName(role PostgresRole) string { @@ -1821,6 +1820,16 @@ func (c *Cluster) generateEndpoint(role PostgresRole, subsets []v1.EndpointSubse return endpoints } +func (c *Cluster) generateConfigMap(role PostgresRole) *v1.ConfigMap { + return &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.configMapName(role), + Namespace: c.Namespace, + Labels: c.roleLabelsSet(true, role), + }, + } +} + func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) []v1.EnvVar { result := make([]v1.EnvVar, 0) diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 5e5c6156e..b2d8d672a 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -35,8 +35,14 @@ func (c *Cluster) listResources() error { c.logger.Infof("found secret: %q (uid: %q) namesapce: %s", util.NameFromMeta(obj.ObjectMeta), obj.UID, obj.ObjectMeta.Namespace) } - for role, endpoint := range c.Endpoints { - c.logger.Infof("found %s endpoint: %q (uid: %q)", role, util.NameFromMeta(endpoint.ObjectMeta), endpoint.UID) + if c.patroniKubernetesUseConfigMaps() { + for role, configMap := range c.ConfigMaps { + c.logger.Infof("found %s config map: %q (uid: %q)", role, util.NameFromMeta(configMap.ObjectMeta), configMap.UID) + } + } else { + for role, endpoint := range c.Endpoints { + c.logger.Infof("found %s endpoint: %q (uid: %q)", role, util.NameFromMeta(endpoint.ObjectMeta), endpoint.UID) + } } for role, service := range c.Services { @@ -402,6 +408,20 @@ func (c *Cluster) generateEndpointSubsets(role PostgresRole) []v1.EndpointSubset return result } +func (c *Cluster) createConfigMap(role PostgresRole) (*v1.ConfigMap, error) { + c.setProcessName("creating config map") + configMapSpec := c.generateConfigMap(role) + + configMap, err := c.KubeClient.ConfigMaps(configMapSpec.Namespace).Create(context.TODO(), configMapSpec, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("could not create %s config map: %v", role, err) + } + + c.ConfigMaps[role] = configMap + + return configMap, nil +} + func (c *Cluster) createPodDisruptionBudget() (*policybeta1.PodDisruptionBudget, error) { podDisruptionBudgetSpec := c.generatePodDisruptionBudget() podDisruptionBudget, err := c.KubeClient. @@ -589,11 +609,21 @@ func (c *Cluster) GetEndpointMaster() *v1.Endpoints { return c.Endpoints[Master] } -// GetEndpointReplica returns cluster's kubernetes master Endpoint +// GetEndpointReplica returns cluster's kubernetes replica Endpoint func (c *Cluster) GetEndpointReplica() *v1.Endpoints { return c.Endpoints[Replica] } +// GetConfigMapMaster returns cluster's kubernetes master ConfigMap +func (c *Cluster) GetConfigMapMaster() *v1.ConfigMap { + return c.ConfigMaps[Master] +} + +// GetConfigMapReplica returns cluster's kubernetes replica ConfigMap +func (c *Cluster) GetConfigMapReplica() *v1.ConfigMap { + return c.ConfigMaps[Replica] +} + // GetStatefulSet returns cluster's kubernetes StatefulSet func (c *Cluster) GetStatefulSet() *appsv1.StatefulSet { return c.Statefulset diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 7ff021ceb..f3f811e23 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -144,7 +144,11 @@ func (c *Cluster) syncServices() error { for _, role := range []PostgresRole{Master, Replica} { c.logger.Debugf("syncing %s service", role) - if !c.patroniKubernetesUseConfigMaps() { + if c.patroniKubernetesUseConfigMaps() { + if err := c.syncConfigMap(role); err != nil { + return fmt.Errorf("could not sync %s config map: %v", role, err) + } + } else { if err := c.syncEndpoint(role); err != nil { return fmt.Errorf("could not sync %s endpoint: %v", role, err) } @@ -234,6 +238,40 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error { return nil } +func (c *Cluster) syncConfigMap(role PostgresRole) error { + var ( + cm *v1.ConfigMap + err error + ) + c.setProcessName("syncing %s config map", role) + + if cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get(context.TODO(), c.configMapName(role), metav1.GetOptions{}); err == nil { + // TODO: No syncing of config map here, is this covered completely by updateService? + c.ConfigMaps[role] = cm + return nil + } + if !k8sutil.ResourceNotFound(err) { + return fmt.Errorf("could not get %s config map: %v", role, err) + } + // no existing config map, create new one + c.ConfigMaps[role] = nil + c.logger.Infof("could not find the cluster's %s config map", role) + + if cm, err = c.createConfigMap(role); err == nil { + c.logger.Infof("created missing %s config map %q", role, util.NameFromMeta(cm.ObjectMeta)) + } else { + if !k8sutil.ResourceAlreadyExists(err) { + return fmt.Errorf("could not create missing %s config map: %v", role, err) + } + c.logger.Infof("%s config map %q already exists", role, util.NameFromMeta(cm.ObjectMeta)) + if cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get(context.TODO(), c.configMapName(role), metav1.GetOptions{}); err != nil { + return fmt.Errorf("could not fetch existing %s config map: %v", role, err) + } + } + c.ConfigMaps[role] = cm + return nil +} + func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error { var ( pdb *policybeta1.PodDisruptionBudget diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 67b4ee395..c271fd019 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -63,6 +63,8 @@ type ClusterStatus struct { ReplicaService *v1.Service MasterEndpoint *v1.Endpoints ReplicaEndpoint *v1.Endpoints + MasterConfigMap *v1.ConfigMap + ReplicaConfigMap *v1.ConfigMap StatefulSet *appsv1.StatefulSet PodDisruptionBudget *policybeta1.PodDisruptionBudget diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 590494412..aac078933 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -544,7 +544,8 @@ func (c *Controller) postgresqlCheck(obj interface{}) *acidv1.Postgresql { Ensures the pod service account and role bindings exists in a namespace before a PG cluster is created there so that a user does not have to deploy these credentials manually. StatefulSets require the service account to - create pods; Patroni requires relevant RBAC bindings to access endpoints. + create pods; Patroni requires relevant RBAC bindings to access endpoints + or config maps. The operator does not sync accounts/role bindings after creation. */