Submit RBAC credentials during initial Event processing (#344)

* During initial Event processing submit the service account for pods and bind it to a cluster role that allows Patroni to successfully start. The cluster role is assumed to be created by the k8s cluster administrator.

docs/administrator.md

@@ -90,13 +90,13 @@ namespace. The operator  performs **no** further syncing of this account.
 
 ## Role-based access control for the operator
 
-The `manifests/operator-rbac.yaml` defines cluster roles and bindings needed
+The `manifests/operator-service-account-rbac.yaml` defines cluster roles and bindings needed
 for the operator to function under access control restrictions. To deploy the
 operator with this RBAC policy use:
 
 ```bash
     $ kubectl create -f manifests/configmap.yaml
-    $ kubectl create -f manifests/operator-rbac.yaml
+    $ kubectl create -f manifests/operator-service-account-rbac.yaml
     $ kubectl create -f manifests/postgres-operator.yaml
     $ kubectl create -f manifests/minimal-postgres-manifest.yaml
 ```

docs/reference/operator_parameters.md

@@ -110,8 +110,10 @@ configuration they are grouped under the `kubernetes` key.
 * **pod_service_account_definition**
   The operator tries to create the pod Service Account in the namespace that
   doesn't define such an account using the YAML definition provided by this
-  option. If not defined, a simple definition that contains only the name will
+  option. If not defined, a simple definition that contains only the name will be used. The default is empty.
-  be used. The default is empty.
+
+* **pod_service_account_role_binding_definition**
+  This definition must bind pod service account to a role with permission sufficient for the pods to start and for Patroni to access k8s endpoints; service account on its own lacks any such rights starting with k8s v1.8. If not excplicitly defined by the user, a simple definition that binds the account to the operator's own 'zalando-postgres-operator' cluster role will be used. The default is empty.
 
 * **pod_terminate_grace_period**
   Patroni pods are [terminated

manifests/operator-service-account-rbac.yaml

@@ -123,6 +123,21 @@ rules:
   verbs:
   - get
   - create
+- apiGroups:
+  - "rbac.authorization.k8s.io"
+  resources:
+  - rolebindings
+  verbs:
+  - get
+  - create
+- apiGroups:
+  - "rbac.authorization.k8s.io"
+  resources:
+  - clusterroles
+  verbs:
+  - bind
+  resourceNames: 
+  - zalando-postgres-operator
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

pkg/cluster/cluster.go

@@ -28,6 +28,7 @@ import (
 	"github.com/zalando-incubator/postgres-operator/pkg/util/patroni"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/teams"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/users"
+	rbacv1beta1 "k8s.io/client-go/pkg/apis/rbac/v1beta1"
 )
 
 var (
@@ -39,10 +40,11 @@ var (
 
 // Config contains operator-wide clients and configuration used from a cluster. TODO: remove struct duplication.
 type Config struct {
-	OpConfig            config.Config
+	OpConfig                     config.Config
-	RestConfig          *rest.Config
+	RestConfig                   *rest.Config
-	InfrastructureRoles map[string]spec.PgUser // inherited from the controller
+	InfrastructureRoles          map[string]spec.PgUser // inherited from the controller
-	PodServiceAccount   *v1.ServiceAccount
+	PodServiceAccount            *v1.ServiceAccount
+	PodServiceAccountRoleBinding *rbacv1beta1.RoleBinding
 }
 
 type kubeResources struct {
@@ -199,39 +201,6 @@ func (c *Cluster) initUsers() error {
 	return nil
 }
 
-/*
-  Ensures the service account required by StatefulSets to create pods exists in a namespace before a PG cluster is created there so that a user does not have to deploy the account manually.
-
-  The operator does not sync these accounts after creation.
-*/
-func (c *Cluster) createPodServiceAccounts() error {
-
-	podServiceAccountName := c.Config.OpConfig.PodServiceAccountName
-	_, err := c.KubeClient.ServiceAccounts(c.Namespace).Get(podServiceAccountName, metav1.GetOptions{})
-
-	if err != nil {
-
-		c.setProcessName(fmt.Sprintf("creating pod service account in the namespace %v", c.Namespace))
-
-		c.logger.Infof("the pod service account %q cannot be retrieved in the namespace %q. Trying to deploy the account.", podServiceAccountName, c.Namespace)
-
-		// get a separate copy of service account
-		// to prevent a race condition when setting a namespace for many clusters
-		sa := *c.PodServiceAccount
-		_, err = c.KubeClient.ServiceAccounts(c.Namespace).Create(&sa)
-		if err != nil {
-			return fmt.Errorf("cannot deploy the pod service account %q defined in the config map to the %q namespace: %v", podServiceAccountName, c.Namespace, err)
-		}
-
-		c.logger.Infof("successfully deployed the pod service account %q to the %q namespace", podServiceAccountName, c.Namespace)
-
-	} else {
-		c.logger.Infof("successfully found the service account %q used to create pods to the namespace %q", podServiceAccountName, c.Namespace)
-	}
-
-	return nil
-}
-
 // Create creates the new kubernetes objects associated with the cluster.
 func (c *Cluster) Create() error {
 	c.mu.Lock()
@@ -298,11 +267,6 @@ func (c *Cluster) Create() error {
 	}
 	c.logger.Infof("pod disruption budget %q has been successfully created", util.NameFromMeta(pdb.ObjectMeta))
 
-	if err = c.createPodServiceAccounts(); err != nil {
-		return fmt.Errorf("could not create pod service account %v : %v", c.OpConfig.PodServiceAccountName, err)
-	}
-	c.logger.Infof("pod service accounts have been successfully synced")
-
 	if c.Statefulset != nil {
 		return fmt.Errorf("statefulset already exists in the cluster")
 	}

pkg/controller/controller.go

@@ -10,6 +10,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/pkg/api/v1"
+	rbacv1beta1 "k8s.io/client-go/pkg/apis/rbac/v1beta1"
 	"k8s.io/client-go/tools/cache"
 
 	"github.com/zalando-incubator/postgres-operator/pkg/apiserver"
@@ -52,7 +53,9 @@ type Controller struct {
 
 	workerLogs map[uint32]ringlog.RingLogger
 
-	PodServiceAccount *v1.ServiceAccount
+	PodServiceAccount            *v1.ServiceAccount
+	PodServiceAccountRoleBinding *rbacv1beta1.RoleBinding
+	namespacesWithDefinedRBAC    sync.Map
 }
 
 // NewController creates a new controller
@@ -162,6 +165,53 @@ func (c *Controller) initPodServiceAccount() {
 	// actual service accounts are deployed at the time of Postgres/Spilo cluster creation
 }
 
+func (c *Controller) initRoleBinding() {
+
+	// service account on its own lacks any rights starting with k8s v1.8
+	// operator binds it to the cluster role with sufficient priviliges
+	// we assume the role is created by the k8s administrator
+	if c.opConfig.PodServiceAccountRoleBindingDefinition == "" {
+		c.opConfig.PodServiceAccountRoleBindingDefinition = `
+		{ 
+			"apiVersion": "rbac.authorization.k8s.io/v1beta1",
+			"kind": "RoleBinding", 
+			"metadata": { 
+				   "name": "zalando-postgres-operator"
+			},
+			"roleRef": {
+				"apiGroup": "rbac.authorization.k8s.io",
+				"kind": "ClusterRole",
+				"name": "zalando-postgres-operator"
+			},
+			"subjects": [
+				{
+					"kind": "ServiceAccount",
+					"name": "operator"
+				}
+			]	
+		}`
+	}
+	c.logger.Info("Parse role bindings")
+	// re-uses k8s internal parsing. See k8s client-go issue #193 for explanation
+	decode := scheme.Codecs.UniversalDeserializer().Decode
+	obj, groupVersionKind, err := decode([]byte(c.opConfig.PodServiceAccountRoleBindingDefinition), nil, nil)
+
+	switch {
+	case err != nil:
+		panic(fmt.Errorf("Unable to parse the definiton of the role binding  for the pod service account definiton from the operator config map: %v", err))
+	case groupVersionKind.Kind != "RoleBinding":
+		panic(fmt.Errorf("role binding definiton in the operator config map defines another type of resource: %v", groupVersionKind.Kind))
+	default:
+		c.PodServiceAccountRoleBinding = obj.(*rbacv1beta1.RoleBinding)
+		c.PodServiceAccountRoleBinding.Namespace = ""
+		c.PodServiceAccountRoleBinding.Subjects[0].Name = c.PodServiceAccount.Name
+		c.logger.Info("successfully parsed")
+
+	}
+
+	// actual roles bindings are deployed at the time of Postgres/Spilo cluster creation
+}
+
 func (c *Controller) initController() {
 	c.initClients()
 
@@ -176,6 +226,8 @@ func (c *Controller) initController() {
 		}
 	} else {
 		c.initOperatorConfig()
+		c.initPodServiceAccount()
+		c.initRoleBinding()
 	}
 
 	c.modifyConfigFromEnvironment()

pkg/controller/operator_config.go

@@ -4,10 +4,11 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"time"
+
 	"github.com/zalando-incubator/postgres-operator/pkg/util/config"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"time"
 )
 
 func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, configObjectName string) (*config.OperatorConfiguration, error) {
@@ -49,6 +50,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *config.OperatorConfigur
 
 	result.PodServiceAccountName = fromCRD.Kubernetes.PodServiceAccountName
 	result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition
+	result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition
 	result.PodTerminateGracePeriod = time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod)
 	result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace
 	result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat

pkg/controller/postgresql.go

@@ -20,6 +20,7 @@ import (
 	"github.com/zalando-incubator/postgres-operator/pkg/spec"
 	"github.com/zalando-incubator/postgres-operator/pkg/util"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
+	"github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/ringlog"
 )
 
@@ -179,6 +180,11 @@ func (c *Controller) processEvent(event spec.ClusterEvent) {
 			c.warnOnDeprecatedPostgreSQLSpecParameters(&event.NewSpec.Spec)
 			c.mergeDeprecatedPostgreSQLSpecParameters(&event.NewSpec.Spec)
 		}
+
+		if err := c.submitRBACCredentials(event); err != nil {
+			c.logger.Warnf("Pods and/or Patroni may misfunction due to the lack of permissions: %v", err)
+		}
+
 	}
 
 	switch event.EventType {
@@ -457,3 +463,78 @@ func (c *Controller) postgresqlDelete(obj interface{}) {
 
 	c.queueClusterEvent(pg, nil, spec.EventDelete)
 }
+
+/*
+  Ensures the pod service account and role bindings exists in a namespace before a PG cluster is created there so that a user does not have to deploy these credentials manually.
+  StatefulSets require the service account to create pods; Patroni requires relevant RBAC bindings to access endpoints.
+
+  The operator does not sync accounts/role bindings after creation.
+*/
+func (c *Controller) submitRBACCredentials(event spec.ClusterEvent) error {
+
+	namespace := event.NewSpec.GetNamespace()
+	if _, ok := c.namespacesWithDefinedRBAC.Load(namespace); ok {
+		return nil
+	}
+
+	if err := c.createPodServiceAccount(namespace); err != nil {
+		return fmt.Errorf("could not create pod service account %v : %v", c.opConfig.PodServiceAccountName, err)
+	}
+
+	if err := c.createRoleBindings(namespace); err != nil {
+		return fmt.Errorf("could not create role binding %v : %v", c.PodServiceAccountRoleBinding.Name, err)
+	}
+
+	c.namespacesWithDefinedRBAC.Store(namespace, true)
+	return nil
+}
+
+func (c *Controller) createPodServiceAccount(namespace string) error {
+
+	podServiceAccountName := c.opConfig.PodServiceAccountName
+	_, err := c.KubeClient.ServiceAccounts(namespace).Get(podServiceAccountName, metav1.GetOptions{})
+	if k8sutil.ResourceNotFound(err) {
+
+		c.logger.Infof(fmt.Sprintf("creating pod service account in the namespace %v", namespace))
+
+		// get a separate copy of service account
+		// to prevent a race condition when setting a namespace for many clusters
+		sa := *c.PodServiceAccount
+		if _, err = c.KubeClient.ServiceAccounts(namespace).Create(&sa); err != nil {
+			return fmt.Errorf("cannot deploy the pod service account %v defined in the config map to the %v namespace: %v", podServiceAccountName, namespace, err)
+		}
+
+		c.logger.Infof("successfully deployed the pod service account %v to the %v namespace", podServiceAccountName, namespace)
+	} else if k8sutil.ResourceAlreadyExists(err) {
+		return nil
+	}
+
+	return err
+}
+
+func (c *Controller) createRoleBindings(namespace string) error {
+
+	podServiceAccountName := c.opConfig.PodServiceAccountName
+	podServiceAccountRoleBindingName := c.PodServiceAccountRoleBinding.Name
+
+	_, err := c.KubeClient.RoleBindings(namespace).Get(podServiceAccountRoleBindingName, metav1.GetOptions{})
+	if k8sutil.ResourceNotFound(err) {
+
+		c.logger.Infof("Creating the role binding %v in the namespace %v", podServiceAccountRoleBindingName, namespace)
+
+		// get a separate copy of role binding
+		// to prevent a race condition when setting a namespace for many clusters
+		rb := *c.PodServiceAccountRoleBinding
+		_, err = c.KubeClient.RoleBindings(namespace).Create(&rb)
+		if err != nil {
+			return fmt.Errorf("cannot bind the pod service account %q defined in the config map to the cluster role in the %q namespace: %v", podServiceAccountName, namespace, err)
+		}
+
+		c.logger.Infof("successfully deployed the role binding for the pod service account %q to the %q namespace", podServiceAccountName, namespace)
+
+	} else if k8sutil.ResourceAlreadyExists(err) {
+		return nil
+	}
+
+	return err
+}

pkg/util/config/config.go

@@ -75,19 +75,20 @@ type Config struct {
 	// default name `operator` enables backward compatibility with the older ServiceAccountName field
 	PodServiceAccountName string `name:"pod_service_account_name" default:"operator"`
 	// value of this string must be valid JSON or YAML; see initPodServiceAccount
-	PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""`
+	PodServiceAccountDefinition            string `name:"pod_service_account_definition" default:""`
-	DbHostedZone                string `name:"db_hosted_zone" default:"db.example.com"`
+	PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""`
-	AWSRegion                   string `name:"aws_region" default:"eu-central-1"`
+	DbHostedZone                           string `name:"db_hosted_zone" default:"db.example.com"`
-	WALES3Bucket                string `name:"wal_s3_bucket"`
+	AWSRegion                              string `name:"aws_region" default:"eu-central-1"`
-	LogS3Bucket                 string `name:"log_s3_bucket"`
+	WALES3Bucket                           string `name:"wal_s3_bucket"`
-	KubeIAMRole                 string `name:"kube_iam_role"`
+	LogS3Bucket                            string `name:"log_s3_bucket"`
-	DebugLogging                bool   `name:"debug_logging" default:"true"`
+	KubeIAMRole                            string `name:"kube_iam_role"`
-	EnableDBAccess              bool   `name:"enable_database_access" default:"true"`
+	DebugLogging                           bool   `name:"debug_logging" default:"true"`
-	EnableTeamsAPI              bool   `name:"enable_teams_api" default:"true"`
+	EnableDBAccess                         bool   `name:"enable_database_access" default:"true"`
-	EnableTeamSuperuser         bool   `name:"enable_team_superuser" default:"false"`
+	EnableTeamsAPI                         bool   `name:"enable_teams_api" default:"true"`
-	TeamAdminRole               string `name:"team_admin_role" default:"admin"`
+	EnableTeamSuperuser                    bool   `name:"enable_team_superuser" default:"false"`
-	EnableMasterLoadBalancer    bool   `name:"enable_master_load_balancer" default:"true"`
+	TeamAdminRole                          string `name:"team_admin_role" default:"admin"`
-	EnableReplicaLoadBalancer   bool   `name:"enable_replica_load_balancer" default:"false"`
+	EnableMasterLoadBalancer               bool   `name:"enable_master_load_balancer" default:"true"`
+	EnableReplicaLoadBalancer              bool   `name:"enable_replica_load_balancer" default:"false"`
 	// deprecated and kept for backward compatibility
 	EnableLoadBalancer       *bool             `name:"enable_load_balancer"`
 	MasterDNSNameFormat      stringTemplate    `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"`

pkg/util/config/crd_config.go

@@ -31,17 +31,18 @@ type PostgresUsersConfiguration struct {
 type KubernetesMetaConfiguration struct {
 	PodServiceAccountName string `json:"pod_service_account_name,omitempty"`
 	// TODO: change it to the proper json
-	PodServiceAccountDefinition   string              `json:"pod_service_account_definition,omitempty"`
+	PodServiceAccountDefinition            string              `json:"pod_service_account_definition,omitempty"`
-	PodTerminateGracePeriod       spec.Duration       `json:"pod_terminate_grace_period,omitempty"`
+	PodServiceAccountRoleBindingDefinition string              `json:"pod_service_account_role_binding_definition,omitempty"`
-	WatchedNamespace              string              `json:"watched_namespace,omitempty"`
+	PodTerminateGracePeriod                spec.Duration       `json:"pod_terminate_grace_period,omitempty"`
-	PDBNameFormat                 stringTemplate      `json:"pdb_name_format,omitempty"`
+	WatchedNamespace                       string              `json:"watched_namespace,omitempty"`
-	SecretNameTemplate            stringTemplate      `json:"secret_name_template,omitempty"`
+	PDBNameFormat                          stringTemplate      `json:"pdb_name_format,omitempty"`
-	OAuthTokenSecretName          spec.NamespacedName `json:"oauth_token_secret_name,omitempty"`
+	SecretNameTemplate                     stringTemplate      `json:"secret_name_template,omitempty"`
-	InfrastructureRolesSecretName spec.NamespacedName `json:"infrastructure_roles_secret_name,omitempty"`
+	OAuthTokenSecretName                   spec.NamespacedName `json:"oauth_token_secret_name,omitempty"`
-	PodRoleLabel                  string              `json:"pod_role_label,omitempty"`
+	InfrastructureRolesSecretName          spec.NamespacedName `json:"infrastructure_roles_secret_name,omitempty"`
-	ClusterLabels                 map[string]string   `json:"cluster_labels,omitempty"`
+	PodRoleLabel                           string              `json:"pod_role_label,omitempty"`
-	ClusterNameLabel              string              `json:"cluster_name_label,omitempty"`
+	ClusterLabels                          map[string]string   `json:"cluster_labels,omitempty"`
-	NodeReadinessLabel            map[string]string   `json:"node_readiness_label,omitempty"`
+	ClusterNameLabel                       string              `json:"cluster_name_label,omitempty"`
+	NodeReadinessLabel                     map[string]string   `json:"node_readiness_label,omitempty"`
 	// TODO: use a proper toleration structure?
 	PodToleration map[string]string `json:"toleration,omitempty"`
 	// TODO: use namespacedname

pkg/util/k8sutil/k8sutil.go

@@ -13,6 +13,7 @@ import (
 	"k8s.io/client-go/kubernetes/typed/apps/v1beta1"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	policyv1beta1 "k8s.io/client-go/kubernetes/typed/policy/v1beta1"
+	rbacv1beta1 "k8s.io/client-go/kubernetes/typed/rbac/v1beta1"
 	"k8s.io/client-go/pkg/api"
 	"k8s.io/client-go/pkg/api/v1"
 	policybeta1 "k8s.io/client-go/pkg/apis/policy/v1beta1"
@@ -35,6 +36,7 @@ type KubernetesClient struct {
 	v1core.NamespacesGetter
 	v1core.ServiceAccountsGetter
 	v1beta1.StatefulSetsGetter
+	rbacv1beta1.RoleBindingsGetter
 	policyv1beta1.PodDisruptionBudgetsGetter
 	apiextbeta1.CustomResourceDefinitionsGetter
 
@@ -83,6 +85,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) {
 	kubeClient.StatefulSetsGetter = client.AppsV1beta1()
 	kubeClient.PodDisruptionBudgetsGetter = client.PolicyV1beta1()
 	kubeClient.RESTClient = client.CoreV1().RESTClient()
+	kubeClient.RoleBindingsGetter = client.RbacV1beta1()
 
 	cfg2 := *cfg
 	cfg2.GroupVersion = &schema.GroupVersion{