From 417f13c0bdda2fb67c1f5ce4c5cb7640e33ecadb Mon Sep 17 00:00:00 2001 From: zerg-junior Date: Thu, 19 Jul 2018 16:40:40 +0200 Subject: [PATCH] Submit RBAC credentials during initial Event processing (#344) * During initial Event processing submit the service account for pods and bind it to a cluster role that allows Patroni to successfully start. The cluster role is assumed to be created by the k8s cluster administrator. --- docs/administrator.md | 4 +- docs/reference/operator_parameters.md | 6 +- manifests/operator-service-account-rbac.yaml | 15 ++++ pkg/cluster/cluster.go | 48 ++---------- pkg/controller/controller.go | 54 ++++++++++++- pkg/controller/operator_config.go | 4 +- pkg/controller/postgresql.go | 81 ++++++++++++++++++++ pkg/util/config/config.go | 27 +++---- pkg/util/config/crd_config.go | 23 +++--- pkg/util/k8sutil/k8sutil.go | 3 + 10 files changed, 193 insertions(+), 72 deletions(-) diff --git a/docs/administrator.md b/docs/administrator.md index bb775ed02..5fbae8fe4 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -90,13 +90,13 @@ namespace. The operator performs **no** further syncing of this account. ## Role-based access control for the operator -The `manifests/operator-rbac.yaml` defines cluster roles and bindings needed +The `manifests/operator-service-account-rbac.yaml` defines cluster roles and bindings needed for the operator to function under access control restrictions. To deploy the operator with this RBAC policy use: ```bash $ kubectl create -f manifests/configmap.yaml - $ kubectl create -f manifests/operator-rbac.yaml + $ kubectl create -f manifests/operator-service-account-rbac.yaml $ kubectl create -f manifests/postgres-operator.yaml $ kubectl create -f manifests/minimal-postgres-manifest.yaml ``` diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 331a77dbd..fd8e797b3 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -110,8 +110,10 @@ configuration they are grouped under the `kubernetes` key. * **pod_service_account_definition** The operator tries to create the pod Service Account in the namespace that doesn't define such an account using the YAML definition provided by this - option. If not defined, a simple definition that contains only the name will - be used. The default is empty. + option. If not defined, a simple definition that contains only the name will be used. The default is empty. + +* **pod_service_account_role_binding_definition** + This definition must bind pod service account to a role with permission sufficient for the pods to start and for Patroni to access k8s endpoints; service account on its own lacks any such rights starting with k8s v1.8. If not excplicitly defined by the user, a simple definition that binds the account to the operator's own 'zalando-postgres-operator' cluster role will be used. The default is empty. * **pod_terminate_grace_period** Patroni pods are [terminated diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 2be3cc4d2..8a1bfb857 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -123,6 +123,21 @@ rules: verbs: - get - create +- apiGroups: + - "rbac.authorization.k8s.io" + resources: + - rolebindings + verbs: + - get + - create +- apiGroups: + - "rbac.authorization.k8s.io" + resources: + - clusterroles + verbs: + - bind + resourceNames: + - zalando-postgres-operator --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 1f97aae0d..f1979ab8a 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -28,6 +28,7 @@ import ( "github.com/zalando-incubator/postgres-operator/pkg/util/patroni" "github.com/zalando-incubator/postgres-operator/pkg/util/teams" "github.com/zalando-incubator/postgres-operator/pkg/util/users" + rbacv1beta1 "k8s.io/client-go/pkg/apis/rbac/v1beta1" ) var ( @@ -39,10 +40,11 @@ var ( // Config contains operator-wide clients and configuration used from a cluster. TODO: remove struct duplication. type Config struct { - OpConfig config.Config - RestConfig *rest.Config - InfrastructureRoles map[string]spec.PgUser // inherited from the controller - PodServiceAccount *v1.ServiceAccount + OpConfig config.Config + RestConfig *rest.Config + InfrastructureRoles map[string]spec.PgUser // inherited from the controller + PodServiceAccount *v1.ServiceAccount + PodServiceAccountRoleBinding *rbacv1beta1.RoleBinding } type kubeResources struct { @@ -199,39 +201,6 @@ func (c *Cluster) initUsers() error { return nil } -/* - Ensures the service account required by StatefulSets to create pods exists in a namespace before a PG cluster is created there so that a user does not have to deploy the account manually. - - The operator does not sync these accounts after creation. -*/ -func (c *Cluster) createPodServiceAccounts() error { - - podServiceAccountName := c.Config.OpConfig.PodServiceAccountName - _, err := c.KubeClient.ServiceAccounts(c.Namespace).Get(podServiceAccountName, metav1.GetOptions{}) - - if err != nil { - - c.setProcessName(fmt.Sprintf("creating pod service account in the namespace %v", c.Namespace)) - - c.logger.Infof("the pod service account %q cannot be retrieved in the namespace %q. Trying to deploy the account.", podServiceAccountName, c.Namespace) - - // get a separate copy of service account - // to prevent a race condition when setting a namespace for many clusters - sa := *c.PodServiceAccount - _, err = c.KubeClient.ServiceAccounts(c.Namespace).Create(&sa) - if err != nil { - return fmt.Errorf("cannot deploy the pod service account %q defined in the config map to the %q namespace: %v", podServiceAccountName, c.Namespace, err) - } - - c.logger.Infof("successfully deployed the pod service account %q to the %q namespace", podServiceAccountName, c.Namespace) - - } else { - c.logger.Infof("successfully found the service account %q used to create pods to the namespace %q", podServiceAccountName, c.Namespace) - } - - return nil -} - // Create creates the new kubernetes objects associated with the cluster. func (c *Cluster) Create() error { c.mu.Lock() @@ -298,11 +267,6 @@ func (c *Cluster) Create() error { } c.logger.Infof("pod disruption budget %q has been successfully created", util.NameFromMeta(pdb.ObjectMeta)) - if err = c.createPodServiceAccounts(); err != nil { - return fmt.Errorf("could not create pod service account %v : %v", c.OpConfig.PodServiceAccountName, err) - } - c.logger.Infof("pod service accounts have been successfully synced") - if c.Statefulset != nil { return fmt.Errorf("statefulset already exists in the cluster") } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index d02d5ea8a..7d1a6ed2f 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/pkg/api/v1" + rbacv1beta1 "k8s.io/client-go/pkg/apis/rbac/v1beta1" "k8s.io/client-go/tools/cache" "github.com/zalando-incubator/postgres-operator/pkg/apiserver" @@ -52,7 +53,9 @@ type Controller struct { workerLogs map[uint32]ringlog.RingLogger - PodServiceAccount *v1.ServiceAccount + PodServiceAccount *v1.ServiceAccount + PodServiceAccountRoleBinding *rbacv1beta1.RoleBinding + namespacesWithDefinedRBAC sync.Map } // NewController creates a new controller @@ -162,6 +165,53 @@ func (c *Controller) initPodServiceAccount() { // actual service accounts are deployed at the time of Postgres/Spilo cluster creation } +func (c *Controller) initRoleBinding() { + + // service account on its own lacks any rights starting with k8s v1.8 + // operator binds it to the cluster role with sufficient priviliges + // we assume the role is created by the k8s administrator + if c.opConfig.PodServiceAccountRoleBindingDefinition == "" { + c.opConfig.PodServiceAccountRoleBindingDefinition = ` + { + "apiVersion": "rbac.authorization.k8s.io/v1beta1", + "kind": "RoleBinding", + "metadata": { + "name": "zalando-postgres-operator" + }, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "zalando-postgres-operator" + }, + "subjects": [ + { + "kind": "ServiceAccount", + "name": "operator" + } + ] + }` + } + c.logger.Info("Parse role bindings") + // re-uses k8s internal parsing. See k8s client-go issue #193 for explanation + decode := scheme.Codecs.UniversalDeserializer().Decode + obj, groupVersionKind, err := decode([]byte(c.opConfig.PodServiceAccountRoleBindingDefinition), nil, nil) + + switch { + case err != nil: + panic(fmt.Errorf("Unable to parse the definiton of the role binding for the pod service account definiton from the operator config map: %v", err)) + case groupVersionKind.Kind != "RoleBinding": + panic(fmt.Errorf("role binding definiton in the operator config map defines another type of resource: %v", groupVersionKind.Kind)) + default: + c.PodServiceAccountRoleBinding = obj.(*rbacv1beta1.RoleBinding) + c.PodServiceAccountRoleBinding.Namespace = "" + c.PodServiceAccountRoleBinding.Subjects[0].Name = c.PodServiceAccount.Name + c.logger.Info("successfully parsed") + + } + + // actual roles bindings are deployed at the time of Postgres/Spilo cluster creation +} + func (c *Controller) initController() { c.initClients() @@ -176,6 +226,8 @@ func (c *Controller) initController() { } } else { c.initOperatorConfig() + c.initPodServiceAccount() + c.initRoleBinding() } c.modifyConfigFromEnvironment() diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 11ad32959..fb448105b 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -4,10 +4,11 @@ import ( "encoding/json" "fmt" + "time" + "github.com/zalando-incubator/postgres-operator/pkg/util/config" "github.com/zalando-incubator/postgres-operator/pkg/util/constants" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "time" ) func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, configObjectName string) (*config.OperatorConfiguration, error) { @@ -49,6 +50,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *config.OperatorConfigur result.PodServiceAccountName = fromCRD.Kubernetes.PodServiceAccountName result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition + result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition result.PodTerminateGracePeriod = time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod) result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index bf7fe8889..9f42075ed 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -20,6 +20,7 @@ import ( "github.com/zalando-incubator/postgres-operator/pkg/spec" "github.com/zalando-incubator/postgres-operator/pkg/util" "github.com/zalando-incubator/postgres-operator/pkg/util/constants" + "github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil" "github.com/zalando-incubator/postgres-operator/pkg/util/ringlog" ) @@ -179,6 +180,11 @@ func (c *Controller) processEvent(event spec.ClusterEvent) { c.warnOnDeprecatedPostgreSQLSpecParameters(&event.NewSpec.Spec) c.mergeDeprecatedPostgreSQLSpecParameters(&event.NewSpec.Spec) } + + if err := c.submitRBACCredentials(event); err != nil { + c.logger.Warnf("Pods and/or Patroni may misfunction due to the lack of permissions: %v", err) + } + } switch event.EventType { @@ -457,3 +463,78 @@ func (c *Controller) postgresqlDelete(obj interface{}) { c.queueClusterEvent(pg, nil, spec.EventDelete) } + +/* + Ensures the pod service account and role bindings exists in a namespace before a PG cluster is created there so that a user does not have to deploy these credentials manually. + StatefulSets require the service account to create pods; Patroni requires relevant RBAC bindings to access endpoints. + + The operator does not sync accounts/role bindings after creation. +*/ +func (c *Controller) submitRBACCredentials(event spec.ClusterEvent) error { + + namespace := event.NewSpec.GetNamespace() + if _, ok := c.namespacesWithDefinedRBAC.Load(namespace); ok { + return nil + } + + if err := c.createPodServiceAccount(namespace); err != nil { + return fmt.Errorf("could not create pod service account %v : %v", c.opConfig.PodServiceAccountName, err) + } + + if err := c.createRoleBindings(namespace); err != nil { + return fmt.Errorf("could not create role binding %v : %v", c.PodServiceAccountRoleBinding.Name, err) + } + + c.namespacesWithDefinedRBAC.Store(namespace, true) + return nil +} + +func (c *Controller) createPodServiceAccount(namespace string) error { + + podServiceAccountName := c.opConfig.PodServiceAccountName + _, err := c.KubeClient.ServiceAccounts(namespace).Get(podServiceAccountName, metav1.GetOptions{}) + if k8sutil.ResourceNotFound(err) { + + c.logger.Infof(fmt.Sprintf("creating pod service account in the namespace %v", namespace)) + + // get a separate copy of service account + // to prevent a race condition when setting a namespace for many clusters + sa := *c.PodServiceAccount + if _, err = c.KubeClient.ServiceAccounts(namespace).Create(&sa); err != nil { + return fmt.Errorf("cannot deploy the pod service account %v defined in the config map to the %v namespace: %v", podServiceAccountName, namespace, err) + } + + c.logger.Infof("successfully deployed the pod service account %v to the %v namespace", podServiceAccountName, namespace) + } else if k8sutil.ResourceAlreadyExists(err) { + return nil + } + + return err +} + +func (c *Controller) createRoleBindings(namespace string) error { + + podServiceAccountName := c.opConfig.PodServiceAccountName + podServiceAccountRoleBindingName := c.PodServiceAccountRoleBinding.Name + + _, err := c.KubeClient.RoleBindings(namespace).Get(podServiceAccountRoleBindingName, metav1.GetOptions{}) + if k8sutil.ResourceNotFound(err) { + + c.logger.Infof("Creating the role binding %v in the namespace %v", podServiceAccountRoleBindingName, namespace) + + // get a separate copy of role binding + // to prevent a race condition when setting a namespace for many clusters + rb := *c.PodServiceAccountRoleBinding + _, err = c.KubeClient.RoleBindings(namespace).Create(&rb) + if err != nil { + return fmt.Errorf("cannot bind the pod service account %q defined in the config map to the cluster role in the %q namespace: %v", podServiceAccountName, namespace, err) + } + + c.logger.Infof("successfully deployed the role binding for the pod service account %q to the %q namespace", podServiceAccountName, namespace) + + } else if k8sutil.ResourceAlreadyExists(err) { + return nil + } + + return err +} diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 26b4d378b..e9017bfab 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -75,19 +75,20 @@ type Config struct { // default name `operator` enables backward compatibility with the older ServiceAccountName field PodServiceAccountName string `name:"pod_service_account_name" default:"operator"` // value of this string must be valid JSON or YAML; see initPodServiceAccount - PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""` - DbHostedZone string `name:"db_hosted_zone" default:"db.example.com"` - AWSRegion string `name:"aws_region" default:"eu-central-1"` - WALES3Bucket string `name:"wal_s3_bucket"` - LogS3Bucket string `name:"log_s3_bucket"` - KubeIAMRole string `name:"kube_iam_role"` - DebugLogging bool `name:"debug_logging" default:"true"` - EnableDBAccess bool `name:"enable_database_access" default:"true"` - EnableTeamsAPI bool `name:"enable_teams_api" default:"true"` - EnableTeamSuperuser bool `name:"enable_team_superuser" default:"false"` - TeamAdminRole string `name:"team_admin_role" default:"admin"` - EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"` - EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"` + PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""` + PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""` + DbHostedZone string `name:"db_hosted_zone" default:"db.example.com"` + AWSRegion string `name:"aws_region" default:"eu-central-1"` + WALES3Bucket string `name:"wal_s3_bucket"` + LogS3Bucket string `name:"log_s3_bucket"` + KubeIAMRole string `name:"kube_iam_role"` + DebugLogging bool `name:"debug_logging" default:"true"` + EnableDBAccess bool `name:"enable_database_access" default:"true"` + EnableTeamsAPI bool `name:"enable_teams_api" default:"true"` + EnableTeamSuperuser bool `name:"enable_team_superuser" default:"false"` + TeamAdminRole string `name:"team_admin_role" default:"admin"` + EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"` + EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"` // deprecated and kept for backward compatibility EnableLoadBalancer *bool `name:"enable_load_balancer"` MasterDNSNameFormat stringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` diff --git a/pkg/util/config/crd_config.go b/pkg/util/config/crd_config.go index e268c41f6..ee3c4b712 100644 --- a/pkg/util/config/crd_config.go +++ b/pkg/util/config/crd_config.go @@ -31,17 +31,18 @@ type PostgresUsersConfiguration struct { type KubernetesMetaConfiguration struct { PodServiceAccountName string `json:"pod_service_account_name,omitempty"` // TODO: change it to the proper json - PodServiceAccountDefinition string `json:"pod_service_account_definition,omitempty"` - PodTerminateGracePeriod spec.Duration `json:"pod_terminate_grace_period,omitempty"` - WatchedNamespace string `json:"watched_namespace,omitempty"` - PDBNameFormat stringTemplate `json:"pdb_name_format,omitempty"` - SecretNameTemplate stringTemplate `json:"secret_name_template,omitempty"` - OAuthTokenSecretName spec.NamespacedName `json:"oauth_token_secret_name,omitempty"` - InfrastructureRolesSecretName spec.NamespacedName `json:"infrastructure_roles_secret_name,omitempty"` - PodRoleLabel string `json:"pod_role_label,omitempty"` - ClusterLabels map[string]string `json:"cluster_labels,omitempty"` - ClusterNameLabel string `json:"cluster_name_label,omitempty"` - NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` + PodServiceAccountDefinition string `json:"pod_service_account_definition,omitempty"` + PodServiceAccountRoleBindingDefinition string `json:"pod_service_account_role_binding_definition,omitempty"` + PodTerminateGracePeriod spec.Duration `json:"pod_terminate_grace_period,omitempty"` + WatchedNamespace string `json:"watched_namespace,omitempty"` + PDBNameFormat stringTemplate `json:"pdb_name_format,omitempty"` + SecretNameTemplate stringTemplate `json:"secret_name_template,omitempty"` + OAuthTokenSecretName spec.NamespacedName `json:"oauth_token_secret_name,omitempty"` + InfrastructureRolesSecretName spec.NamespacedName `json:"infrastructure_roles_secret_name,omitempty"` + PodRoleLabel string `json:"pod_role_label,omitempty"` + ClusterLabels map[string]string `json:"cluster_labels,omitempty"` + ClusterNameLabel string `json:"cluster_name_label,omitempty"` + NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` // TODO: use a proper toleration structure? PodToleration map[string]string `json:"toleration,omitempty"` // TODO: use namespacedname diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 142d4f822..dd96aa5a7 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -13,6 +13,7 @@ import ( "k8s.io/client-go/kubernetes/typed/apps/v1beta1" v1core "k8s.io/client-go/kubernetes/typed/core/v1" policyv1beta1 "k8s.io/client-go/kubernetes/typed/policy/v1beta1" + rbacv1beta1 "k8s.io/client-go/kubernetes/typed/rbac/v1beta1" "k8s.io/client-go/pkg/api" "k8s.io/client-go/pkg/api/v1" policybeta1 "k8s.io/client-go/pkg/apis/policy/v1beta1" @@ -35,6 +36,7 @@ type KubernetesClient struct { v1core.NamespacesGetter v1core.ServiceAccountsGetter v1beta1.StatefulSetsGetter + rbacv1beta1.RoleBindingsGetter policyv1beta1.PodDisruptionBudgetsGetter apiextbeta1.CustomResourceDefinitionsGetter @@ -83,6 +85,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.StatefulSetsGetter = client.AppsV1beta1() kubeClient.PodDisruptionBudgetsGetter = client.PolicyV1beta1() kubeClient.RESTClient = client.CoreV1().RESTClient() + kubeClient.RoleBindingsGetter = client.RbacV1beta1() cfg2 := *cfg cfg2.GroupVersion = &schema.GroupVersion{