merge with master

pkg/controller/controller.go

@@ -1,41 +1,54 @@
 package controller
 
 import (
+	"bytes"
+	"context"
+	"encoding/json"
 	"fmt"
 	"os"
+	"strings"
 	"sync"
+	"time"
 
 	"github.com/sirupsen/logrus"
-	v1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/client-go/tools/cache"
-
+	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando/postgres-operator/pkg/apiserver"
 	"github.com/zalando/postgres-operator/pkg/cluster"
+	acidv1informer "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do/v1"
 	"github.com/zalando/postgres-operator/pkg/spec"
+	"github.com/zalando/postgres-operator/pkg/teams"
 	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
 	"github.com/zalando/postgres-operator/pkg/util/ringlog"
-
-	acidv1informer "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do/v1"
+	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/tools/reference"
 )
 
 // Controller represents operator controller
 type Controller struct {
-	config   spec.ControllerConfig
-	opConfig *config.Config
+	config    spec.ControllerConfig
+	opConfig  *config.Config
+	pgTeamMap teams.PostgresTeamMap
 
 	logger     *logrus.Entry
 	KubeClient k8sutil.KubernetesClient
 	apiserver  *apiserver.Server
 
+	eventRecorder    record.EventRecorder
+	eventBroadcaster record.EventBroadcaster
+
 	stopCh chan struct{}
 
+	controllerID     string
 	curWorkerID      uint32 //initialized with 0
 	curWorkerCluster sync.Map
 	clusterWorkers   map[spec.NamespacedName]uint32
@@ -45,10 +58,11 @@ type Controller struct {
 	clusterHistory   map[spec.NamespacedName]ringlog.RingLogger // history of the cluster changes
 	teamClusters     map[string][]spec.NamespacedName
 
-	postgresqlInformer cache.SharedIndexInformer
-	podInformer        cache.SharedIndexInformer
-	nodesInformer      cache.SharedIndexInformer
-	podCh              chan cluster.PodEvent
+	postgresqlInformer   cache.SharedIndexInformer
+	postgresTeamInformer cache.SharedIndexInformer
+	podInformer          cache.SharedIndexInformer
+	nodesInformer        cache.SharedIndexInformer
+	podCh                chan cluster.PodEvent
 
 	clusterEventQueues    []*cache.FIFO // [workerID]Queue
 	lastClusterSyncTime   int64
@@ -62,13 +76,35 @@ type Controller struct {
 }
 
 // NewController creates a new controller
-func NewController(controllerConfig *spec.ControllerConfig) *Controller {
+func NewController(controllerConfig *spec.ControllerConfig, controllerId string) *Controller {
 	logger := logrus.New()
+	if controllerConfig.EnableJsonLogging {
+		logger.SetFormatter(&logrus.JSONFormatter{})
+	} else {
+		if os.Getenv("LOG_NOQUOTE") != "" {
+			logger.SetFormatter(&logrus.TextFormatter{PadLevelText: true, DisableQuote: true})
+		}
+	}
+
+	var myComponentName = "postgres-operator"
+	if controllerId != "" {
+		myComponentName += "/" + controllerId
+	}
+
+	eventBroadcaster := record.NewBroadcaster()
+
+	// disabling the sending of events also to the logoutput
+	// the operator currently duplicates a lot of log entries with this setup
+	// eventBroadcaster.StartLogging(logger.Infof)
+	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: myComponentName})
 
 	c := &Controller{
 		config:           *controllerConfig,
 		opConfig:         &config.Config{},
 		logger:           logger.WithField("pkg", "controller"),
+		eventRecorder:    recorder,
+		eventBroadcaster: eventBroadcaster,
+		controllerID:     controllerId,
 		curWorkerCluster: sync.Map{},
 		clusterWorkers:   make(map[spec.NamespacedName]uint32),
 		clusters:         make(map[spec.NamespacedName]*cluster.Cluster),
@@ -90,6 +126,11 @@ func (c *Controller) initClients() {
 	if err != nil {
 		c.logger.Fatalf("could not create kubernetes clients: %v", err)
 	}
+	c.eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: c.KubeClient.EventsGetter.Events("")})
+	if err != nil {
+		c.logger.Fatalf("could not setup kubernetes event sink: %v", err)
+	}
+
 }
 
 func (c *Controller) initOperatorConfig() {
@@ -97,7 +138,7 @@ func (c *Controller) initOperatorConfig() {
 
 	if c.config.ConfigMapName != (spec.NamespacedName{}) {
 		configMap, err := c.KubeClient.ConfigMaps(c.config.ConfigMapName.Namespace).
-			Get(c.config.ConfigMapName.Name, metav1.GetOptions{})
+			Get(context.TODO(), c.config.ConfigMapName.Name, metav1.GetOptions{})
 		if err != nil {
 			panic(err)
 		}
@@ -156,12 +197,25 @@ func (c *Controller) warnOnDeprecatedOperatorParameters() {
 		c.logger.Warningf("Operator configuration parameter 'enable_load_balancer' is deprecated and takes no effect. " +
 			"Consider using the 'enable_master_load_balancer' or 'enable_replica_load_balancer' instead.")
 	}
+
+	if len(c.opConfig.SidecarImages) > 0 {
+		c.logger.Warningf("Operator configuration parameter 'sidecar_docker_images' is deprecated. " +
+			"Consider using 'sidecars' instead.")
+	}
+}
+
+func compactValue(v string) string {
+	var compact bytes.Buffer
+	if err := json.Compact(&compact, []byte(v)); err != nil {
+		panic("Hard coded json strings broken!")
+	}
+	return compact.String()
 }
 
 func (c *Controller) initPodServiceAccount() {
 
 	if c.opConfig.PodServiceAccountDefinition == "" {
-		c.opConfig.PodServiceAccountDefinition = `
+		stringValue := `
 		{
 			"apiVersion": "v1",
 			"kind": "ServiceAccount",
@@ -169,6 +223,9 @@ func (c *Controller) initPodServiceAccount() {
 				"name": "postgres-pod"
 			}
 		}`
+
+		c.opConfig.PodServiceAccountDefinition = compactValue(stringValue)
+
 	}
 
 	// re-uses k8s internal parsing. See k8s client-go issue #193 for explanation
@@ -292,7 +349,7 @@ func (c *Controller) initRoleBinding() {
 	// operator binds it to the cluster role with sufficient privileges
 	// we assume the role is created by the k8s administrator
 	if c.opConfig.PodServiceAccountRoleBindingDefinition == "" {
-		c.opConfig.PodServiceAccountRoleBindingDefinition = fmt.Sprintf(`
+		stringValue := fmt.Sprintf(`
 		{
 			"apiVersion": "rbac.authorization.k8s.io/v1",
 			"kind": "RoleBinding",
@@ -311,6 +368,7 @@ func (c *Controller) initRoleBinding() {
 				}
 			]
 		}`, c.PodServiceAccount.Name, c.PodServiceAccount.Name, c.PodServiceAccount.Name)
+		c.opConfig.PodServiceAccountRoleBindingDefinition = compactValue(stringValue)
 	}
 	c.logger.Info("Parse role bindings")
 	// re-uses k8s internal parsing. See k8s client-go issue #193 for explanation
@@ -329,11 +387,19 @@ func (c *Controller) initRoleBinding() {
 
 	}
 
-	// actual roles bindings are deployed at the time of Postgres/Spilo cluster creation
+	// actual roles bindings ar*logrus.Entrye deployed at the time of Postgres/Spilo cluster creation
+}
+
+func logMultiLineConfig(log *logrus.Entry, config string) {
+	lines := strings.Split(config, "\n")
+	for _, l := range lines {
+		log.Infof("%s", l)
+	}
 }
 
 func (c *Controller) initController() {
 	c.initClients()
+	c.controllerID = os.Getenv("CONTROLLER_ID")
 
 	if configObjectName := os.Getenv("POSTGRES_OPERATOR_CONFIGURATION_OBJECT"); configObjectName != "" {
 		if err := c.createConfigurationCRD(c.opConfig.EnableCRDValidation); err != nil {
@@ -360,16 +426,22 @@ func (c *Controller) initController() {
 		c.logger.Fatalf("could not register Postgres CustomResourceDefinition: %v", err)
 	}
 
-	c.initPodServiceAccount()
 	c.initSharedInformers()
 
+	if c.opConfig.EnablePostgresTeamCRD {
+		c.loadPostgresTeams()
+	} else {
+		c.pgTeamMap = teams.PostgresTeamMap{}
+	}
+
 	if c.opConfig.DebugLogging {
 		c.logger.Logger.Level = logrus.DebugLevel
 	}
 
-	c.logger.Infof("config: %s", c.opConfig.MustMarshal())
+	logMultiLineConfig(c.logger, c.opConfig.MustMarshal())
 
-	if infraRoles, err := c.getInfrastructureRoles(&c.opConfig.InfrastructureRolesSecretName); err != nil {
+	roleDefs := c.getInfrastructureRoleDefinitions()
+	if infraRoles, err := c.getInfrastructureRoles(roleDefs); err != nil {
 		c.logger.Warningf("could not get infrastructure roles: %v", err)
 	} else {
 		c.config.InfrastructureRoles = infraRoles
@@ -393,6 +465,7 @@ func (c *Controller) initController() {
 
 func (c *Controller) initSharedInformers() {
 
+	// Postgresqls
 	c.postgresqlInformer = acidv1informer.NewPostgresqlInformer(
 		c.KubeClient.AcidV1ClientSet,
 		c.opConfig.WatchedNamespace,
@@ -405,6 +478,20 @@ func (c *Controller) initSharedInformers() {
 		DeleteFunc: c.postgresqlDelete,
 	})
 
+	// PostgresTeams
+	if c.opConfig.EnablePostgresTeamCRD {
+		c.postgresTeamInformer = acidv1informer.NewPostgresTeamInformer(
+			c.KubeClient.AcidV1ClientSet,
+			c.opConfig.WatchedNamespace,
+			constants.QueueResyncPeriodTPR*6, // 30 min
+			cache.Indexers{})
+
+		c.postgresTeamInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+			AddFunc:    c.postgresTeamAdd,
+			UpdateFunc: c.postgresTeamUpdate,
+		})
+	}
+
 	// Pods
 	podLw := &cache.ListWatch{
 		ListFunc:  c.podListFunc,
@@ -465,6 +552,10 @@ func (c *Controller) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) {
 	go c.apiserver.Run(stopCh, wg)
 	go c.kubeNodesInformer(stopCh, wg)
 
+	if c.opConfig.EnablePostgresTeamCRD {
+		go c.runPostgresTeamInformer(stopCh, wg)
+	}
+
 	c.logger.Info("started working in background")
 }
 
@@ -480,6 +571,12 @@ func (c *Controller) runPostgresqlInformer(stopCh <-chan struct{}, wg *sync.Wait
 	c.postgresqlInformer.Run(stopCh)
 }
 
+func (c *Controller) runPostgresTeamInformer(stopCh <-chan struct{}, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	c.postgresTeamInformer.Run(stopCh)
+}
+
 func queueClusterKey(eventType EventType, uid types.UID) string {
 	return fmt.Sprintf("%s-%s", eventType, uid)
 }
@@ -501,7 +598,7 @@ func (c *Controller) getEffectiveNamespace(namespaceFromEnvironment, namespaceFr
 
 	} else {
 
-		if _, err := c.KubeClient.Namespaces().Get(namespace, metav1.GetOptions{}); err != nil {
+		if _, err := c.KubeClient.Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil {
 			c.logger.Fatalf("Could not find the watched namespace %q", namespace)
 		} else {
 			c.logger.Infof("Listenting to the specific namespace %q", namespace)
@@ -511,3 +608,57 @@ func (c *Controller) getEffectiveNamespace(namespaceFromEnvironment, namespaceFr
 
 	return namespace
 }
+
+// GetReference of Postgres CR object
+// i.e. required to emit events to this resource
+func (c *Controller) GetReference(postgresql *acidv1.Postgresql) *v1.ObjectReference {
+	ref, err := reference.GetReference(scheme.Scheme, postgresql)
+	if err != nil {
+		c.logger.Errorf("could not get reference for Postgresql CR %v/%v: %v", postgresql.Namespace, postgresql.Name, err)
+	}
+	return ref
+}
+
+func (c *Controller) meetsClusterDeleteAnnotations(postgresql *acidv1.Postgresql) error {
+
+	deleteAnnotationDateKey := c.opConfig.DeleteAnnotationDateKey
+	currentTime := time.Now()
+	currentDate := currentTime.Format("2006-01-02") // go's reference date
+
+	if deleteAnnotationDateKey != "" {
+		if deleteDate, ok := postgresql.Annotations[deleteAnnotationDateKey]; ok {
+			if deleteDate != currentDate {
+				return fmt.Errorf("annotation %s not matching the current date: got %s, expected %s", deleteAnnotationDateKey, deleteDate, currentDate)
+			}
+		} else {
+			return fmt.Errorf("annotation %s not set in manifest to allow cluster deletion", deleteAnnotationDateKey)
+		}
+	}
+
+	deleteAnnotationNameKey := c.opConfig.DeleteAnnotationNameKey
+
+	if deleteAnnotationNameKey != "" {
+		if clusterName, ok := postgresql.Annotations[deleteAnnotationNameKey]; ok {
+			if clusterName != postgresql.Name {
+				return fmt.Errorf("annotation %s not matching the cluster name: got %s, expected %s", deleteAnnotationNameKey, clusterName, postgresql.Name)
+			}
+		} else {
+			return fmt.Errorf("annotation %s not set in manifest to allow cluster deletion", deleteAnnotationNameKey)
+		}
+	}
+
+	return nil
+}
+
+// hasOwnership returns true if the controller is the "owner" of the postgresql.
+// Whether it's owner is determined by the value of 'acid.zalan.do/controller'
+// annotation. If the value matches the controllerID then it owns it, or if the
+// controllerID is "" and there's no annotation set.
+func (c *Controller) hasOwnership(postgresql *acidv1.Postgresql) bool {
+	if postgresql.Annotations != nil {
+		if owner, ok := postgresql.Annotations[constants.PostgresqlControllerAnnotationKey]; ok {
+			return owner == c.controllerID
+		}
+	}
+	return c.controllerID == ""
+}

pkg/controller/node.go

@@ -1,11 +1,12 @@
 package controller
 
 import (
+	"context"
 	"fmt"
 	"time"
 
 	"github.com/zalando/postgres-operator/pkg/util/retryutil"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -22,7 +23,7 @@ func (c *Controller) nodeListFunc(options metav1.ListOptions) (runtime.Object, e
 		TimeoutSeconds:  options.TimeoutSeconds,
 	}
 
-	return c.KubeClient.Nodes().List(opts)
+	return c.KubeClient.Nodes().List(context.TODO(), opts)
 }
 
 func (c *Controller) nodeWatchFunc(options metav1.ListOptions) (watch.Interface, error) {
@@ -32,7 +33,7 @@ func (c *Controller) nodeWatchFunc(options metav1.ListOptions) (watch.Interface,
 		TimeoutSeconds:  options.TimeoutSeconds,
 	}
 
-	return c.KubeClient.Nodes().Watch(opts)
+	return c.KubeClient.Nodes().Watch(context.TODO(), opts)
 }
 
 func (c *Controller) nodeAdd(obj interface{}) {
@@ -41,7 +42,7 @@ func (c *Controller) nodeAdd(obj interface{}) {
 		return
 	}
 
-	c.logger.Debugf("new node has been added: %q (%s)", util.NameFromMeta(node.ObjectMeta), node.Spec.ProviderID)
+	c.logger.Debugf("new node has been added: %s (%s)", util.NameFromMeta(node.ObjectMeta), node.Spec.ProviderID)
 
 	// check if the node became not ready while the operator was down (otherwise we would have caught it in nodeUpdate)
 	if !c.nodeIsReady(node) {
@@ -75,7 +76,7 @@ func (c *Controller) nodeUpdate(prev, cur interface{}) {
 }
 
 func (c *Controller) nodeIsReady(node *v1.Node) bool {
-	return (!node.Spec.Unschedulable || util.MapContains(node.Labels, c.opConfig.NodeReadinessLabel) ||
+	return (!node.Spec.Unschedulable || (len(c.opConfig.NodeReadinessLabel) > 0 && util.MapContains(node.Labels, c.opConfig.NodeReadinessLabel)) ||
 		util.MapContains(node.Labels, map[string]string{"master": "true"}))
 }
 
@@ -87,7 +88,7 @@ func (c *Controller) attemptToMoveMasterPodsOffNode(node *v1.Node) error {
 	opts := metav1.ListOptions{
 		LabelSelector: labels.Set(c.opConfig.ClusterLabels).String(),
 	}
-	podList, err := c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(opts)
+	podList, err := c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(context.TODO(), opts)
 	if err != nil {
 		c.logger.Errorf("could not fetch list of the pods: %v", err)
 		return err
@@ -172,19 +173,19 @@ func (c *Controller) nodeDelete(obj interface{}) {
 }
 
 func (c *Controller) moveMasterPodsOffNode(node *v1.Node) {
-
+	// retry to move master until configured timeout is reached
 	err := retryutil.Retry(1*time.Minute, c.opConfig.MasterPodMoveTimeout,
 		func() (bool, error) {
 			err := c.attemptToMoveMasterPodsOffNode(node)
 			if err != nil {
-				return false, fmt.Errorf("unable to move master pods off the unschedulable node; will retry after delay of 1 minute")
+				return false, err
 			}
 			return true, nil
 		},
 	)
 
 	if err != nil {
-		c.logger.Warningf("failed to move master pods from the node %q: timeout of %v minutes expired", node.Name, c.opConfig.MasterPodMoveTimeout)
+		c.logger.Warningf("failed to move master pods from the node %q: %v", node.Name, err)
 	}
 
 }

pkg/controller/node_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 
 	"github.com/zalando/postgres-operator/pkg/spec"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -13,10 +13,9 @@ const (
 	readyValue = "ready"
 )
 
-func initializeController() *Controller {
-	var c = NewController(&spec.ControllerConfig{})
-	c.opConfig.NodeReadinessLabel = map[string]string{readyLabel: readyValue}
-	return c
+func newNodeTestController() *Controller {
+	var controller = NewController(&spec.ControllerConfig{}, "node-test")
+	return controller
 }
 
 func makeNode(labels map[string]string, isSchedulable bool) *v1.Node {
@@ -31,34 +30,65 @@ func makeNode(labels map[string]string, isSchedulable bool) *v1.Node {
 	}
 }
 
-var c = initializeController()
+var nodeTestController = newNodeTestController()
 
 func TestNodeIsReady(t *testing.T) {
 	testName := "TestNodeIsReady"
 	var testTable = []struct {
-		in  *v1.Node
-		out bool
+		in             *v1.Node
+		out            bool
+		readinessLabel map[string]string
 	}{
 		{
-			in:  makeNode(map[string]string{"foo": "bar"}, true),
-			out: true,
+			in:             makeNode(map[string]string{"foo": "bar"}, true),
+			out:            true,
+			readinessLabel: map[string]string{readyLabel: readyValue},
 		},
 		{
-			in:  makeNode(map[string]string{"foo": "bar"}, false),
-			out: false,
+			in:             makeNode(map[string]string{"foo": "bar"}, false),
+			out:            false,
+			readinessLabel: map[string]string{readyLabel: readyValue},
 		},
 		{
-			in:  makeNode(map[string]string{readyLabel: readyValue}, false),
-			out: true,
+			in:             makeNode(map[string]string{readyLabel: readyValue}, false),
+			out:            true,
+			readinessLabel: map[string]string{readyLabel: readyValue},
 		},
 		{
-			in:  makeNode(map[string]string{"foo": "bar", "master": "true"}, false),
-			out: true,
+			in:             makeNode(map[string]string{"foo": "bar", "master": "true"}, false),
+			out:            true,
+			readinessLabel: map[string]string{readyLabel: readyValue},
+		},
+		{
+			in:             makeNode(map[string]string{"foo": "bar", "master": "true"}, false),
+			out:            true,
+			readinessLabel: map[string]string{readyLabel: readyValue},
+		},
+		{
+			in:             makeNode(map[string]string{"foo": "bar"}, true),
+			out:            true,
+			readinessLabel: map[string]string{},
+		},
+		{
+			in:             makeNode(map[string]string{"foo": "bar"}, false),
+			out:            false,
+			readinessLabel: map[string]string{},
+		},
+		{
+			in:             makeNode(map[string]string{readyLabel: readyValue}, false),
+			out:            false,
+			readinessLabel: map[string]string{},
+		},
+		{
+			in:             makeNode(map[string]string{"foo": "bar", "master": "true"}, false),
+			out:            true,
+			readinessLabel: map[string]string{},
 		},
 	}
 	for _, tt := range testTable {
-		if isReady := c.nodeIsReady(tt.in); isReady != tt.out {
-			t.Errorf("%s: expected response %t doesn't match the actual %t for the node %#v",
+		nodeTestController.opConfig.NodeReadinessLabel = tt.readinessLabel
+		if isReady := nodeTestController.nodeIsReady(tt.in); isReady != tt.out {
+			t.Errorf("%s: expected response %t does not match the actual %t for the node %#v",
 				testName, tt.out, isReady, tt.in)
 		}
 	}

pkg/controller/operator_config.go

@@ -1,18 +1,22 @@
 package controller
 
 import (
+	"context"
 	"fmt"
 
 	"time"
 
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/config"
+	"github.com/zalando/postgres-operator/pkg/util/constants"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, configObjectName string) (*acidv1.OperatorConfiguration, error) {
 
-	config, err := c.KubeClient.AcidV1ClientSet.AcidV1().OperatorConfigurations(configObjectNamespace).Get(configObjectName, metav1.GetOptions{})
+	config, err := c.KubeClient.OperatorConfigurationsGetter.OperatorConfigurations(configObjectNamespace).Get(
+		context.TODO(), configObjectName, metav1.GetOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("could not get operator configuration object %q: %v", configObjectName, err)
 	}
@@ -20,98 +24,138 @@ func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, con
 	return config, nil
 }
 
+func int32ToPointer(value int32) *int32 {
+	return &value
+}
+
 // importConfigurationFromCRD is a transitional function that converts CRD configuration to the one based on the configmap
 func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigurationData) *config.Config {
 	result := &config.Config{}
 
 	// general config
-	result.EnableCRDValidation = fromCRD.EnableCRDValidation
+	result.EnableCRDValidation = util.CoalesceBool(fromCRD.EnableCRDValidation, util.True())
+	result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade
+	result.EnablePgVersionEnvVar = fromCRD.EnablePgVersionEnvVar
+	result.EnableSpiloWalPathCompat = fromCRD.EnableSpiloWalPathCompat
 	result.EtcdHost = fromCRD.EtcdHost
-	result.DockerImage = fromCRD.DockerImage
-	result.Workers = fromCRD.Workers
+	result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps
+	result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-13:2.0-p2")
+	result.Workers = util.CoalesceUInt32(fromCRD.Workers, 8)
 	result.MinInstances = fromCRD.MinInstances
 	result.MaxInstances = fromCRD.MaxInstances
-	result.ResyncPeriod = time.Duration(fromCRD.ResyncPeriod)
-	result.RepairPeriod = time.Duration(fromCRD.RepairPeriod)
+	result.ResyncPeriod = util.CoalesceDuration(time.Duration(fromCRD.ResyncPeriod), "30m")
+	result.RepairPeriod = util.CoalesceDuration(time.Duration(fromCRD.RepairPeriod), "5m")
 	result.SetMemoryRequestToLimit = fromCRD.SetMemoryRequestToLimit
-	result.ShmVolume = fromCRD.ShmVolume
-	result.Sidecars = fromCRD.Sidecars
+	result.ShmVolume = util.CoalesceBool(fromCRD.ShmVolume, util.True())
+	result.SidecarImages = fromCRD.SidecarImages
+	result.SidecarContainers = fromCRD.SidecarContainers
 
 	// user config
-	result.SuperUsername = fromCRD.PostgresUsersConfiguration.SuperUsername
-	result.ReplicationUsername = fromCRD.PostgresUsersConfiguration.ReplicationUsername
+	result.SuperUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.SuperUsername, "postgres")
+	result.ReplicationUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.ReplicationUsername, "standby")
 
 	// kubernetes config
 	result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations
-	result.PodServiceAccountName = fromCRD.Kubernetes.PodServiceAccountName
+	result.PodServiceAccountName = util.Coalesce(fromCRD.Kubernetes.PodServiceAccountName, "postgres-pod")
 	result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition
 	result.PodServiceAccountRoleDefinition = fromCRD.Kubernetes.PodServiceAccountRoleDefinition
 	result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition
 	result.PodEnvironmentConfigMap = fromCRD.Kubernetes.PodEnvironmentConfigMap
-	result.PodTerminateGracePeriod = time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod)
+	result.PodEnvironmentSecret = fromCRD.Kubernetes.PodEnvironmentSecret
+	result.PodTerminateGracePeriod = util.CoalesceDuration(time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod), "5m")
 	result.SpiloPrivileged = fromCRD.Kubernetes.SpiloPrivileged
+	result.SpiloRunAsUser = fromCRD.Kubernetes.SpiloRunAsUser
+	result.SpiloRunAsGroup = fromCRD.Kubernetes.SpiloRunAsGroup
 	result.SpiloFSGroup = fromCRD.Kubernetes.SpiloFSGroup
-	result.ClusterDomain = fromCRD.Kubernetes.ClusterDomain
+	result.ClusterDomain = util.Coalesce(fromCRD.Kubernetes.ClusterDomain, "cluster.local")
 	result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace
 	result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat
-	result.EnablePodDisruptionBudget = fromCRD.Kubernetes.EnablePodDisruptionBudget
-	result.EnableInitContainers = fromCRD.Kubernetes.EnableInitContainers
-	result.EnableSidecars = fromCRD.Kubernetes.EnableSidecars
+	result.EnablePodDisruptionBudget = util.CoalesceBool(fromCRD.Kubernetes.EnablePodDisruptionBudget, util.True())
+	result.StorageResizeMode = util.Coalesce(fromCRD.Kubernetes.StorageResizeMode, "pvc")
+	result.EnableInitContainers = util.CoalesceBool(fromCRD.Kubernetes.EnableInitContainers, util.True())
+	result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True())
 	result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate
 	result.OAuthTokenSecretName = fromCRD.Kubernetes.OAuthTokenSecretName
+
 	result.InfrastructureRolesSecretName = fromCRD.Kubernetes.InfrastructureRolesSecretName
-	result.PodRoleLabel = fromCRD.Kubernetes.PodRoleLabel
-	result.ClusterLabels = fromCRD.Kubernetes.ClusterLabels
+	if fromCRD.Kubernetes.InfrastructureRolesDefs != nil {
+		result.InfrastructureRoles = []*config.InfrastructureRole{}
+		for _, secret := range fromCRD.Kubernetes.InfrastructureRolesDefs {
+			result.InfrastructureRoles = append(
+				result.InfrastructureRoles,
+				&config.InfrastructureRole{
+					SecretName:  secret.SecretName,
+					UserKey:     secret.UserKey,
+					RoleKey:     secret.RoleKey,
+					PasswordKey: secret.PasswordKey,
+				})
+		}
+	}
+
+	result.PodRoleLabel = util.Coalesce(fromCRD.Kubernetes.PodRoleLabel, "spilo-role")
+	result.ClusterLabels = util.CoalesceStrMap(fromCRD.Kubernetes.ClusterLabels, map[string]string{"application": "spilo"})
 	result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels
-	result.ClusterNameLabel = fromCRD.Kubernetes.ClusterNameLabel
+	result.InheritedAnnotations = fromCRD.Kubernetes.InheritedAnnotations
+	result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations
+	result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name")
+	result.DeleteAnnotationDateKey = fromCRD.Kubernetes.DeleteAnnotationDateKey
+	result.DeleteAnnotationNameKey = fromCRD.Kubernetes.DeleteAnnotationNameKey
 	result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel
 	result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName
-	result.PodManagementPolicy = fromCRD.Kubernetes.PodManagementPolicy
-	result.MasterPodMoveTimeout = time.Duration(fromCRD.Kubernetes.MasterPodMoveTimeout)
+	result.PodManagementPolicy = util.Coalesce(fromCRD.Kubernetes.PodManagementPolicy, "ordered_ready")
+	result.MasterPodMoveTimeout = util.CoalesceDuration(time.Duration(fromCRD.Kubernetes.MasterPodMoveTimeout), "10m")
 	result.EnablePodAntiAffinity = fromCRD.Kubernetes.EnablePodAntiAffinity
-	result.PodAntiAffinityTopologyKey = fromCRD.Kubernetes.PodAntiAffinityTopologyKey
+	result.PodAntiAffinityTopologyKey = util.Coalesce(fromCRD.Kubernetes.PodAntiAffinityTopologyKey, "kubernetes.io/hostname")
 
 	// Postgres Pod resources
-	result.DefaultCPURequest = fromCRD.PostgresPodResources.DefaultCPURequest
-	result.DefaultMemoryRequest = fromCRD.PostgresPodResources.DefaultMemoryRequest
-	result.DefaultCPULimit = fromCRD.PostgresPodResources.DefaultCPULimit
-	result.DefaultMemoryLimit = fromCRD.PostgresPodResources.DefaultMemoryLimit
-	result.MinCPULimit = fromCRD.PostgresPodResources.MinCPULimit
-	result.MinMemoryLimit = fromCRD.PostgresPodResources.MinMemoryLimit
+	result.DefaultCPURequest = util.Coalesce(fromCRD.PostgresPodResources.DefaultCPURequest, "100m")
+	result.DefaultMemoryRequest = util.Coalesce(fromCRD.PostgresPodResources.DefaultMemoryRequest, "100Mi")
+	result.DefaultCPULimit = util.Coalesce(fromCRD.PostgresPodResources.DefaultCPULimit, "1")
+	result.DefaultMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.DefaultMemoryLimit, "500Mi")
+	result.MinCPULimit = util.Coalesce(fromCRD.PostgresPodResources.MinCPULimit, "250m")
+	result.MinMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.MinMemoryLimit, "250Mi")
 
 	// timeout config
-	result.ResourceCheckInterval = time.Duration(fromCRD.Timeouts.ResourceCheckInterval)
-	result.ResourceCheckTimeout = time.Duration(fromCRD.Timeouts.ResourceCheckTimeout)
-	result.PodLabelWaitTimeout = time.Duration(fromCRD.Timeouts.PodLabelWaitTimeout)
-	result.PodDeletionWaitTimeout = time.Duration(fromCRD.Timeouts.PodDeletionWaitTimeout)
-	result.ReadyWaitInterval = time.Duration(fromCRD.Timeouts.ReadyWaitInterval)
-	result.ReadyWaitTimeout = time.Duration(fromCRD.Timeouts.ReadyWaitTimeout)
+	result.ResourceCheckInterval = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ResourceCheckInterval), "3s")
+	result.ResourceCheckTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ResourceCheckTimeout), "10m")
+	result.PodLabelWaitTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PodLabelWaitTimeout), "10m")
+	result.PodDeletionWaitTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PodDeletionWaitTimeout), "10m")
+	result.ReadyWaitInterval = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ReadyWaitInterval), "4s")
+	result.ReadyWaitTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ReadyWaitTimeout), "30s")
 
 	// load balancer config
-	result.DbHostedZone = fromCRD.LoadBalancer.DbHostedZone
+	result.DbHostedZone = util.Coalesce(fromCRD.LoadBalancer.DbHostedZone, "db.example.com")
 	result.EnableMasterLoadBalancer = fromCRD.LoadBalancer.EnableMasterLoadBalancer
 	result.EnableReplicaLoadBalancer = fromCRD.LoadBalancer.EnableReplicaLoadBalancer
 	result.CustomServiceAnnotations = fromCRD.LoadBalancer.CustomServiceAnnotations
 	result.MasterDNSNameFormat = fromCRD.LoadBalancer.MasterDNSNameFormat
 	result.ReplicaDNSNameFormat = fromCRD.LoadBalancer.ReplicaDNSNameFormat
+	result.ExternalTrafficPolicy = util.Coalesce(fromCRD.LoadBalancer.ExternalTrafficPolicy, "Cluster")
 
 	// AWS or GCP config
 	result.WALES3Bucket = fromCRD.AWSGCP.WALES3Bucket
 	result.AWSRegion = fromCRD.AWSGCP.AWSRegion
 	result.LogS3Bucket = fromCRD.AWSGCP.LogS3Bucket
 	result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole
+	result.WALGSBucket = fromCRD.AWSGCP.WALGSBucket
+	result.GCPCredentials = fromCRD.AWSGCP.GCPCredentials
 	result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount
-	result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath
+	result.AdditionalSecretMountPath = util.Coalesce(fromCRD.AWSGCP.AdditionalSecretMountPath, "/meta/credentials")
+	result.EnableEBSGp3Migration = fromCRD.AWSGCP.EnableEBSGp3Migration
+	result.EnableEBSGp3MigrationMaxSize = util.CoalesceInt64(fromCRD.AWSGCP.EnableEBSGp3MigrationMaxSize, 1000)
 
 	// logical backup config
-	result.LogicalBackupSchedule = fromCRD.LogicalBackup.Schedule
-	result.LogicalBackupDockerImage = fromCRD.LogicalBackup.DockerImage
+	result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *")
+	result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup:v1.6.0")
+	result.LogicalBackupProvider = util.Coalesce(fromCRD.LogicalBackup.BackupProvider, "s3")
 	result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket
 	result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region
 	result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint
 	result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID
 	result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey
 	result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE
+	result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials
+	result.LogicalBackupJobPrefix = util.Coalesce(fromCRD.LogicalBackup.JobPrefix, "logical-backup-")
 
 	// debug config
 	result.DebugLogging = fromCRD.OperatorDebug.DebugLogging
@@ -119,20 +163,22 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 
 	// Teams API config
 	result.EnableTeamsAPI = fromCRD.TeamsAPI.EnableTeamsAPI
-	result.TeamsAPIUrl = fromCRD.TeamsAPI.TeamsAPIUrl
-	result.TeamAPIRoleConfiguration = fromCRD.TeamsAPI.TeamAPIRoleConfiguration
+	result.TeamsAPIUrl = util.Coalesce(fromCRD.TeamsAPI.TeamsAPIUrl, "https://teams.example.com/api/")
+	result.TeamAPIRoleConfiguration = util.CoalesceStrMap(fromCRD.TeamsAPI.TeamAPIRoleConfiguration, map[string]string{"log_statement": "all"})
 	result.EnableTeamSuperuser = fromCRD.TeamsAPI.EnableTeamSuperuser
 	result.EnableAdminRoleForUsers = fromCRD.TeamsAPI.EnableAdminRoleForUsers
 	result.TeamAdminRole = fromCRD.TeamsAPI.TeamAdminRole
-	result.PamRoleName = fromCRD.TeamsAPI.PamRoleName
-	result.PamConfiguration = fromCRD.TeamsAPI.PamConfiguration
-	result.ProtectedRoles = fromCRD.TeamsAPI.ProtectedRoles
+	result.PamRoleName = util.Coalesce(fromCRD.TeamsAPI.PamRoleName, "zalandos")
+	result.PamConfiguration = util.Coalesce(fromCRD.TeamsAPI.PamConfiguration, "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees")
+	result.ProtectedRoles = util.CoalesceStrArr(fromCRD.TeamsAPI.ProtectedRoles, []string{"admin"})
 	result.PostgresSuperuserTeams = fromCRD.TeamsAPI.PostgresSuperuserTeams
+	result.EnablePostgresTeamCRD = fromCRD.TeamsAPI.EnablePostgresTeamCRD
+	result.EnablePostgresTeamCRDSuperusers = fromCRD.TeamsAPI.EnablePostgresTeamCRDSuperusers
 
 	// logging REST API config
-	result.APIPort = fromCRD.LoggingRESTAPI.APIPort
-	result.RingLogLines = fromCRD.LoggingRESTAPI.RingLogLines
-	result.ClusterHistoryEntries = fromCRD.LoggingRESTAPI.ClusterHistoryEntries
+	result.APIPort = util.CoalesceInt(fromCRD.LoggingRESTAPI.APIPort, 8080)
+	result.RingLogLines = util.CoalesceInt(fromCRD.LoggingRESTAPI.RingLogLines, 100)
+	result.ClusterHistoryEntries = util.CoalesceInt(fromCRD.LoggingRESTAPI.ClusterHistoryEntries, 1000)
 
 	// Scalyr config
 	result.ScalyrAPIKey = fromCRD.Scalyr.ScalyrAPIKey
@@ -143,5 +189,56 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.ScalyrCPULimit = fromCRD.Scalyr.ScalyrCPULimit
 	result.ScalyrMemoryLimit = fromCRD.Scalyr.ScalyrMemoryLimit
 
+	// Connection pooler. Looks like we can't use defaulting in CRD before 1.17,
+	// so ensure default values here.
+	result.ConnectionPooler.NumberOfInstances = util.CoalesceInt32(
+		fromCRD.ConnectionPooler.NumberOfInstances,
+		int32ToPointer(2))
+
+	result.ConnectionPooler.NumberOfInstances = util.MaxInt32(
+		result.ConnectionPooler.NumberOfInstances,
+		int32ToPointer(2))
+
+	result.ConnectionPooler.Schema = util.Coalesce(
+		fromCRD.ConnectionPooler.Schema,
+		constants.ConnectionPoolerSchemaName)
+
+	result.ConnectionPooler.User = util.Coalesce(
+		fromCRD.ConnectionPooler.User,
+		constants.ConnectionPoolerUserName)
+
+	if result.ConnectionPooler.User == result.SuperUsername {
+		msg := "Connection pool user is not allowed to be the same as super user, username: %s"
+		panic(fmt.Errorf(msg, result.ConnectionPooler.User))
+	}
+
+	result.ConnectionPooler.Image = util.Coalesce(
+		fromCRD.ConnectionPooler.Image,
+		"registry.opensource.zalan.do/acid/pgbouncer")
+
+	result.ConnectionPooler.Mode = util.Coalesce(
+		fromCRD.ConnectionPooler.Mode,
+		constants.ConnectionPoolerDefaultMode)
+
+	result.ConnectionPooler.ConnectionPoolerDefaultCPURequest = util.Coalesce(
+		fromCRD.ConnectionPooler.DefaultCPURequest,
+		constants.ConnectionPoolerDefaultCpuRequest)
+
+	result.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest = util.Coalesce(
+		fromCRD.ConnectionPooler.DefaultMemoryRequest,
+		constants.ConnectionPoolerDefaultMemoryRequest)
+
+	result.ConnectionPooler.ConnectionPoolerDefaultCPULimit = util.Coalesce(
+		fromCRD.ConnectionPooler.DefaultCPULimit,
+		constants.ConnectionPoolerDefaultCpuLimit)
+
+	result.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit = util.Coalesce(
+		fromCRD.ConnectionPooler.DefaultMemoryLimit,
+		constants.ConnectionPoolerDefaultMemoryLimit)
+
+	result.ConnectionPooler.MaxDBConnections = util.CoalesceInt32(
+		fromCRD.ConnectionPooler.MaxDBConnections,
+		int32ToPointer(constants.ConnectionPoolerMaxDBConnections))
+
 	return result
 }

pkg/controller/pod.go

@@ -1,7 +1,9 @@
 package controller
 
 import (
-	"k8s.io/api/core/v1"
+	"context"
+
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/watch"
@@ -19,7 +21,7 @@ func (c *Controller) podListFunc(options metav1.ListOptions) (runtime.Object, er
 		TimeoutSeconds:  options.TimeoutSeconds,
 	}
 
-	return c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(opts)
+	return c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(context.TODO(), opts)
 }
 
 func (c *Controller) podWatchFunc(options metav1.ListOptions) (watch.Interface, error) {
@@ -29,7 +31,7 @@ func (c *Controller) podWatchFunc(options metav1.ListOptions) (watch.Interface,
 		TimeoutSeconds:  options.TimeoutSeconds,
 	}
 
-	return c.KubeClient.Pods(c.opConfig.WatchedNamespace).Watch(opts)
+	return c.KubeClient.Pods(c.opConfig.WatchedNamespace).Watch(context.TODO(), opts)
 }
 
 func (c *Controller) dispatchPodEvent(clusterName spec.NamespacedName, event cluster.PodEvent) {

pkg/controller/postgresql.go

@@ -1,6 +1,8 @@
 package controller
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"reflect"
 	"strings"
@@ -10,6 +12,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/cache"
@@ -40,12 +43,23 @@ func (c *Controller) clusterResync(stopCh <-chan struct{}, wg *sync.WaitGroup) {
 
 // clusterListFunc obtains a list of all PostgreSQL clusters
 func (c *Controller) listClusters(options metav1.ListOptions) (*acidv1.PostgresqlList, error) {
+	var pgList acidv1.PostgresqlList
+
 	// TODO: use the SharedInformer cache instead of quering Kubernetes API directly.
-	list, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.opConfig.WatchedNamespace).List(options)
+	list, err := c.KubeClient.PostgresqlsGetter.Postgresqls(c.opConfig.WatchedNamespace).List(context.TODO(), options)
 	if err != nil {
 		c.logger.Errorf("could not list postgresql objects: %v", err)
 	}
-	return list, err
+	if c.controllerID != "" {
+		c.logger.Debugf("watch only clusters with controllerID %q", c.controllerID)
+	}
+	for _, pg := range list.Items {
+		if pg.Error == "" && c.hasOwnership(&pg) {
+			pgList.Items = append(pgList.Items, pg)
+		}
+	}
+
+	return &pgList, err
 }
 
 // clusterListAndSync lists all manifests and decides whether to run the sync or repair.
@@ -145,7 +159,7 @@ func (c *Controller) acquireInitialListOfClusters() error {
 }
 
 func (c *Controller) addCluster(lg *logrus.Entry, clusterName spec.NamespacedName, pgSpec *acidv1.Postgresql) *cluster.Cluster {
-	cl := cluster.New(c.makeClusterConfig(), c.KubeClient, *pgSpec, lg)
+	cl := cluster.New(c.makeClusterConfig(), c.KubeClient, *pgSpec, lg, c.eventRecorder)
 	cl.Run(c.stopCh)
 	teamName := strings.ToLower(cl.Spec.TeamID)
 
@@ -211,11 +225,11 @@ func (c *Controller) processEvent(event ClusterEvent) {
 	switch event.EventType {
 	case EventAdd:
 		if clusterFound {
-			lg.Debugf("cluster already exists")
+			lg.Infof("Recieved add event for already existing Postgres cluster")
 			return
 		}
 
-		lg.Infof("creation of the cluster started")
+		lg.Infof("creating a new Postgres cluster")
 
 		cl = c.addCluster(lg, clusterName, event.NewSpec)
 
@@ -224,6 +238,7 @@ func (c *Controller) processEvent(event ClusterEvent) {
 		if err := cl.Create(); err != nil {
 			cl.Error = fmt.Sprintf("could not create cluster: %v", err)
 			lg.Error(cl.Error)
+			c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Create", "%v", cl.Error)
 
 			return
 		}
@@ -262,6 +277,8 @@ func (c *Controller) processEvent(event ClusterEvent) {
 
 		c.curWorkerCluster.Store(event.WorkerID, cl)
 		cl.Delete()
+		// Fixme - no error handling for delete ?
+		// c.eventRecorder.Eventf(cl.GetReference, v1.EventTypeWarning, "Delete", "%v", cl.Error)
 
 		func() {
 			defer c.clustersMu.Unlock()
@@ -292,6 +309,7 @@ func (c *Controller) processEvent(event ClusterEvent) {
 		c.curWorkerCluster.Store(event.WorkerID, cl)
 		if err := cl.Sync(event.NewSpec); err != nil {
 			cl.Error = fmt.Sprintf("could not sync cluster: %v", err)
+			c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Sync", "%v", cl.Error)
 			lg.Error(cl.Error)
 			return
 		}
@@ -403,15 +421,42 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1.
 		clusterError = informerNewSpec.Error
 	}
 
+	// only allow deletion if delete annotations are set and conditions are met
+	if eventType == EventDelete {
+		if err := c.meetsClusterDeleteAnnotations(informerOldSpec); err != nil {
+			c.logger.WithField("cluster-name", clusterName).Warnf(
+				"ignoring %q event for cluster %q - manifest does not fulfill delete requirements: %s", eventType, clusterName, err)
+			c.logger.WithField("cluster-name", clusterName).Warnf(
+				"please, recreate Postgresql resource %q and set annotations to delete properly", clusterName)
+			if currentManifest, marshalErr := json.Marshal(informerOldSpec); marshalErr != nil {
+				c.logger.WithField("cluster-name", clusterName).Warnf("could not marshal current manifest:\n%+v", informerOldSpec)
+			} else {
+				c.logger.WithField("cluster-name", clusterName).Warnf("%s\n", string(currentManifest))
+			}
+			return
+		}
+	}
+
 	if clusterError != "" && eventType != EventDelete {
-		c.logger.
-			WithField("cluster-name", clusterName).
-			Debugf("skipping %q event for the invalid cluster: %s", eventType, clusterError)
+		c.logger.WithField("cluster-name", clusterName).Debugf("skipping %q event for the invalid cluster: %s", eventType, clusterError)
+
+		switch eventType {
+		case EventAdd:
+			c.KubeClient.SetPostgresCRDStatus(clusterName, acidv1.ClusterStatusAddFailed)
+			c.eventRecorder.Eventf(c.GetReference(informerNewSpec), v1.EventTypeWarning, "Create", "%v", clusterError)
+		case EventUpdate:
+			c.KubeClient.SetPostgresCRDStatus(clusterName, acidv1.ClusterStatusUpdateFailed)
+			c.eventRecorder.Eventf(c.GetReference(informerNewSpec), v1.EventTypeWarning, "Update", "%v", clusterError)
+		default:
+			c.KubeClient.SetPostgresCRDStatus(clusterName, acidv1.ClusterStatusSyncFailed)
+			c.eventRecorder.Eventf(c.GetReference(informerNewSpec), v1.EventTypeWarning, "Sync", "%v", clusterError)
+		}
+
 		return
 	}
 
 	// Don't pass the spec directly from the informer, since subsequent modifications of it would be reflected
-	// in the informer internal state, making it incohherent with the actual Kubernetes object (and, as a side
+	// in the informer internal state, making it incoherent with the actual Kubernetes object (and, as a side
 	// effect, the modified state will be returned together with subsequent events).
 
 	workerID := c.clusterWorkerID(clusterName)
@@ -428,7 +473,7 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1.
 	if err := c.clusterEventQueues[workerID].Add(clusterEvent); err != nil {
 		lg.Errorf("error while queueing cluster event: %v", clusterEvent)
 	}
-	lg.Infof("%q event has been queued", eventType)
+	lg.Infof("%s event has been queued", eventType)
 
 	if eventType != EventDelete {
 		return
@@ -449,47 +494,56 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1.
 		if err != nil {
 			lg.Warningf("could not delete event from the queue: %v", err)
 		} else {
-			lg.Debugf("event %q has been discarded for the cluster", evType)
+			lg.Debugf("event %s has been discarded for the cluster", evType)
 		}
 	}
 }
 
 func (c *Controller) postgresqlAdd(obj interface{}) {
-	pg, ok := obj.(*acidv1.Postgresql)
-	if !ok {
-		c.logger.Errorf("could not cast to postgresql spec")
-		return
+	pg := c.postgresqlCheck(obj)
+	if pg != nil {
+		// We will not get multiple Add events for the same cluster
+		c.queueClusterEvent(nil, pg, EventAdd)
 	}
 
-	// We will not get multiple Add events for the same cluster
-	c.queueClusterEvent(nil, pg, EventAdd)
+	return
 }
 
 func (c *Controller) postgresqlUpdate(prev, cur interface{}) {
-	pgOld, ok := prev.(*acidv1.Postgresql)
-	if !ok {
-		c.logger.Errorf("could not cast to postgresql spec")
-	}
-	pgNew, ok := cur.(*acidv1.Postgresql)
-	if !ok {
-		c.logger.Errorf("could not cast to postgresql spec")
-	}
-	// Avoid the inifinite recursion for status updates
-	if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) {
-		return
+	pgOld := c.postgresqlCheck(prev)
+	pgNew := c.postgresqlCheck(cur)
+	if pgOld != nil && pgNew != nil {
+		// Avoid the inifinite recursion for status updates
+		if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) {
+			if reflect.DeepEqual(pgNew.Annotations, pgOld.Annotations) {
+				return
+			}
+		}
+		c.queueClusterEvent(pgOld, pgNew, EventUpdate)
 	}
 
-	c.queueClusterEvent(pgOld, pgNew, EventUpdate)
+	return
 }
 
 func (c *Controller) postgresqlDelete(obj interface{}) {
+	pg := c.postgresqlCheck(obj)
+	if pg != nil {
+		c.queueClusterEvent(pg, nil, EventDelete)
+	}
+
+	return
+}
+
+func (c *Controller) postgresqlCheck(obj interface{}) *acidv1.Postgresql {
 	pg, ok := obj.(*acidv1.Postgresql)
 	if !ok {
 		c.logger.Errorf("could not cast to postgresql spec")
-		return
+		return nil
 	}
-
-	c.queueClusterEvent(pg, nil, EventDelete)
+	if !c.hasOwnership(pg) {
+		return nil
+	}
+	return pg
 }
 
 /*
@@ -525,15 +579,14 @@ func (c *Controller) submitRBACCredentials(event ClusterEvent) error {
 func (c *Controller) createPodServiceAccount(namespace string) error {
 
 	podServiceAccountName := c.opConfig.PodServiceAccountName
-
-	_, err := c.KubeClient.ServiceAccounts(namespace).Get(podServiceAccountName, metav1.GetOptions{})
+	_, err := c.KubeClient.ServiceAccounts(namespace).Get(context.TODO(), podServiceAccountName, metav1.GetOptions{})
 	if k8sutil.ResourceNotFound(err) {
 		c.logger.Infof(fmt.Sprintf("creating pod service account %q in the %q namespace", podServiceAccountName, namespace))
 
 		// get a separate copy of service account
 		// to prevent a race condition when setting a namespace for many clusters
 		sa := *c.PodServiceAccount
-		if _, err = c.KubeClient.ServiceAccounts(namespace).Create(&sa); err != nil {
+		if _, err = c.KubeClient.ServiceAccounts(namespace).Create(context.TODO(), &sa, metav1.CreateOptions{}); err != nil {
 			return fmt.Errorf("cannot deploy the pod service account %q defined in the configuration to the %q namespace: %v", podServiceAccountName, namespace, err)
 		}
 		c.logger.Infof("successfully deployed the pod service account %q to the %q namespace", podServiceAccountName, namespace)
@@ -572,14 +625,14 @@ func (c *Controller) createRoleBindings(namespace string) error {
 	podServiceAccountName := c.opConfig.PodServiceAccountName
 	podServiceAccountRoleBindingName := c.PodServiceAccountRoleBinding.Name
 
-	_, err := c.KubeClient.RoleBindings(namespace).Get(podServiceAccountRoleBindingName, metav1.GetOptions{})
+	_, err := c.KubeClient.RoleBindings(namespace).Get(context.TODO(), podServiceAccountRoleBindingName, metav1.GetOptions{})
 	if k8sutil.ResourceNotFound(err) {
 		c.logger.Infof("Creating the role binding %q in the %q namespace", podServiceAccountRoleBindingName, namespace)
 
 		// get a separate copy of role binding
 		// to prevent a race condition when setting a namespace for many clusters
 		rb := *c.PodServiceAccountRoleBinding
-		_, err = c.KubeClient.RoleBindings(namespace).Create(&rb)
+		_, err = c.KubeClient.RoleBindings(namespace).Create(context.TODO(), &rb, metav1.CreateOptions{})
 		if err != nil {
 			return fmt.Errorf("cannot bind the pod service account %q defined in the configuration to the cluster role in the %q namespace: %v", podServiceAccountName, namespace, err)
 		}

pkg/controller/postgresql_test.go

@@ -1,10 +1,14 @@
 package controller
 
 import (
-	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
-	"github.com/zalando/postgres-operator/pkg/spec"
+	"fmt"
 	"reflect"
 	"testing"
+	"time"
+
+	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	"github.com/zalando/postgres-operator/pkg/spec"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var (
@@ -12,9 +16,55 @@ var (
 	False = false
 )
 
-func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) {
-	c := NewController(&spec.ControllerConfig{})
+func newPostgresqlTestController() *Controller {
+	controller := NewController(&spec.ControllerConfig{}, "postgresql-test")
+	return controller
+}
 
+var postgresqlTestController = newPostgresqlTestController()
+
+func TestControllerOwnershipOnPostgresql(t *testing.T) {
+	tests := []struct {
+		name  string
+		pg    *acidv1.Postgresql
+		owned bool
+		error string
+	}{
+		{
+			"Postgres cluster with defined ownership of mocked controller",
+			&acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{"acid.zalan.do/controller": "postgresql-test"},
+				},
+			},
+			True,
+			"Postgres cluster should be owned by operator, but controller says no",
+		},
+		{
+			"Postgres cluster with defined ownership of another controller",
+			&acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{"acid.zalan.do/controller": "stups-test"},
+				},
+			},
+			False,
+			"Postgres cluster should be owned by another operator, but controller say yes",
+		},
+		{
+			"Test Postgres cluster without defined ownership",
+			&acidv1.Postgresql{},
+			False,
+			"Postgres cluster should be owned by operator with empty controller ID, but controller says yes",
+		},
+	}
+	for _, tt := range tests {
+		if postgresqlTestController.hasOwnership(tt.pg) != tt.owned {
+			t.Errorf("%s: %v", tt.name, tt.error)
+		}
+	}
+}
+
+func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) {
 	tests := []struct {
 		name  string
 		in    *acidv1.PostgresSpec
@@ -36,9 +86,94 @@ func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) {
 		},
 	}
 	for _, tt := range tests {
-		result := c.mergeDeprecatedPostgreSQLSpecParameters(tt.in)
+		result := postgresqlTestController.mergeDeprecatedPostgreSQLSpecParameters(tt.in)
 		if !reflect.DeepEqual(result, tt.out) {
 			t.Errorf("%s: %v", tt.name, tt.error)
 		}
 	}
 }
+
+func TestMeetsClusterDeleteAnnotations(t *testing.T) {
+	// set delete annotations in configuration
+	postgresqlTestController.opConfig.DeleteAnnotationDateKey = "delete-date"
+	postgresqlTestController.opConfig.DeleteAnnotationNameKey = "delete-clustername"
+
+	currentTime := time.Now()
+	today := currentTime.Format("2006-01-02") // go's reference date
+	clusterName := "acid-test-cluster"
+
+	tests := []struct {
+		name  string
+		pg    *acidv1.Postgresql
+		error string
+	}{
+		{
+			"Postgres cluster with matching delete annotations",
+			&acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: clusterName,
+					Annotations: map[string]string{
+						"delete-date":        today,
+						"delete-clustername": clusterName,
+					},
+				},
+			},
+			"",
+		},
+		{
+			"Postgres cluster with violated delete date annotation",
+			&acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: clusterName,
+					Annotations: map[string]string{
+						"delete-date":        "2020-02-02",
+						"delete-clustername": clusterName,
+					},
+				},
+			},
+			fmt.Sprintf("annotation delete-date not matching the current date: got 2020-02-02, expected %s", today),
+		},
+		{
+			"Postgres cluster with violated delete cluster name annotation",
+			&acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: clusterName,
+					Annotations: map[string]string{
+						"delete-date":        today,
+						"delete-clustername": "acid-minimal-cluster",
+					},
+				},
+			},
+			fmt.Sprintf("annotation delete-clustername not matching the cluster name: got acid-minimal-cluster, expected %s", clusterName),
+		},
+		{
+			"Postgres cluster with missing delete annotations",
+			&acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        clusterName,
+					Annotations: map[string]string{},
+				},
+			},
+			"annotation delete-date not set in manifest to allow cluster deletion",
+		},
+		{
+			"Postgres cluster with missing delete cluster name annotation",
+			&acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: clusterName,
+					Annotations: map[string]string{
+						"delete-date": today,
+					},
+				},
+			},
+			"annotation delete-clustername not set in manifest to allow cluster deletion",
+		},
+	}
+	for _, tt := range tests {
+		if err := postgresqlTestController.meetsClusterDeleteAnnotations(tt.pg); err != nil {
+			if !reflect.DeepEqual(err.Error(), tt.error) {
+				t.Errorf("Expected error %q, got: %v", tt.error, err)
+			}
+		}
+	}
+}

pkg/controller/types.go

@@ -1,9 +1,10 @@
 package controller
 
 import (
-	"k8s.io/apimachinery/pkg/types"
 	"time"
 
+	"k8s.io/apimachinery/pkg/types"
+
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 )
 

pkg/controller/util.go

@@ -1,11 +1,13 @@
 package controller
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	v1 "k8s.io/api/core/v1"
-	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -13,6 +15,8 @@ import (
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando/postgres-operator/pkg/cluster"
 	"github.com/zalando/postgres-operator/pkg/spec"
+	"github.com/zalando/postgres-operator/pkg/teams"
+	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
 	"gopkg.in/yaml.v2"
@@ -27,6 +31,7 @@ func (c *Controller) makeClusterConfig() cluster.Config {
 	return cluster.Config{
 		RestConfig:          c.config.RestConfig,
 		OpConfig:            config.Copy(c.opConfig),
+		PgTeamMap:           c.pgTeamMap,
 		InfrastructureRoles: infrastructureRoles,
 		PodServiceAccount:   c.PodServiceAccount,
 	}
@@ -49,8 +54,8 @@ func (c *Controller) clusterWorkerID(clusterName spec.NamespacedName) uint32 {
 	return c.clusterWorkers[clusterName]
 }
 
-func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefinition) error {
-	if _, err := c.KubeClient.CustomResourceDefinitions().Create(crd); err != nil {
+func (c *Controller) createOperatorCRD(crd *apiextv1.CustomResourceDefinition) error {
+	if _, err := c.KubeClient.CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}); err != nil {
 		if k8sutil.ResourceAlreadyExists(err) {
 			c.logger.Infof("customResourceDefinition %q is already registered and will only be updated", crd.Name)
 
@@ -58,7 +63,8 @@ func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefiniti
 			if err != nil {
 				return fmt.Errorf("could not marshal new customResourceDefintion: %v", err)
 			}
-			if _, err := c.KubeClient.CustomResourceDefinitions().Patch(crd.Name, types.MergePatchType, patch); err != nil {
+			if _, err := c.KubeClient.CustomResourceDefinitions().Patch(
+				context.TODO(), crd.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
 				return fmt.Errorf("could not update customResourceDefinition: %v", err)
 			}
 		} else {
@@ -69,19 +75,19 @@ func (c *Controller) createOperatorCRD(crd *apiextv1beta1.CustomResourceDefiniti
 	}
 
 	return wait.Poll(c.config.CRDReadyWaitInterval, c.config.CRDReadyWaitTimeout, func() (bool, error) {
-		c, err := c.KubeClient.CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{})
+		c, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{})
 		if err != nil {
 			return false, err
 		}
 
 		for _, cond := range c.Status.Conditions {
 			switch cond.Type {
-			case apiextv1beta1.Established:
-				if cond.Status == apiextv1beta1.ConditionTrue {
+			case apiextv1.Established:
+				if cond.Status == apiextv1.ConditionTrue {
 					return true, err
 				}
-			case apiextv1beta1.NamesAccepted:
-				if cond.Status == apiextv1beta1.ConditionFalse {
+			case apiextv1.NamesAccepted:
+				if cond.Status == apiextv1.ConditionFalse {
 					return false, fmt.Errorf("name conflict: %v", cond.Reason)
 				}
 			}
@@ -107,61 +113,262 @@ func readDecodedRole(s string) (*spec.PgUser, error) {
 	return &result, nil
 }
 
-func (c *Controller) getInfrastructureRoles(rolesSecret *spec.NamespacedName) (map[string]spec.PgUser, error) {
-	if *rolesSecret == (spec.NamespacedName{}) {
+var emptyName = (spec.NamespacedName{})
+
+// Return information about what secrets we need to use to create
+// infrastructure roles and in which format are they. This is done in
+// compatible way, so that the previous logic is not changed, and handles both
+// configuration in ConfigMap & CRD.
+func (c *Controller) getInfrastructureRoleDefinitions() []*config.InfrastructureRole {
+	var roleDef config.InfrastructureRole
+
+	// take from CRD configuration
+	rolesDefs := c.opConfig.InfrastructureRoles
+
+	// check if we can extract something from the configmap config option
+	if c.opConfig.InfrastructureRolesDefs != "" {
+		// The configmap option could contain either a role description (in the
+		// form key1: value1, key2: value2), which has to be used together with
+		// an old secret name.
+
+		var secretName spec.NamespacedName
+		var err error
+		propertySep := ","
+		valueSep := ":"
+
+		// The field contains the format in which secret is written, let's
+		// convert it to a proper definition
+		properties := strings.Split(c.opConfig.InfrastructureRolesDefs, propertySep)
+		roleDef = config.InfrastructureRole{Template: false}
+
+		for _, property := range properties {
+			values := strings.Split(property, valueSep)
+			if len(values) < 2 {
+				continue
+			}
+			name := strings.TrimSpace(values[0])
+			value := strings.TrimSpace(values[1])
+
+			switch name {
+			case "secretname":
+				if err = secretName.DecodeWorker(value, "default"); err != nil {
+					c.logger.Warningf("Could not marshal secret name %s: %v", value, err)
+				} else {
+					roleDef.SecretName = secretName
+				}
+			case "userkey":
+				roleDef.UserKey = value
+			case "passwordkey":
+				roleDef.PasswordKey = value
+			case "rolekey":
+				roleDef.RoleKey = value
+			case "defaultuservalue":
+				roleDef.DefaultUserValue = value
+			case "defaultrolevalue":
+				roleDef.DefaultRoleValue = value
+			default:
+				c.logger.Warningf("Role description is not known: %s", properties)
+			}
+		}
+
+		if roleDef.SecretName != emptyName &&
+			(roleDef.UserKey != "" || roleDef.DefaultUserValue != "") &&
+			roleDef.PasswordKey != "" {
+			rolesDefs = append(rolesDefs, &roleDef)
+		}
+	}
+
+	if c.opConfig.InfrastructureRolesSecretName != emptyName {
+		// At this point we deal with the old format, let's replicate it
+		// via existing definition structure and remember that it's just a
+		// template, the real values are in user1,password1,inrole1 etc.
+		rolesDefs = append(rolesDefs, &config.InfrastructureRole{
+			SecretName:  c.opConfig.InfrastructureRolesSecretName,
+			UserKey:     "user",
+			PasswordKey: "password",
+			RoleKey:     "inrole",
+			Template:    true,
+		})
+	}
+
+	return rolesDefs
+}
+
+func (c *Controller) getInfrastructureRoles(
+	rolesSecrets []*config.InfrastructureRole) (
+	map[string]spec.PgUser, []error) {
+
+	var errors []error
+	var noRolesProvided = true
+
+	roles := []spec.PgUser{}
+	uniqRoles := map[string]spec.PgUser{}
+
+	// To be compatible with the legacy implementation we need to return nil if
+	// the provided secret name is empty. The equivalent situation in the
+	// current implementation is an empty rolesSecrets slice or all its items
+	// are empty.
+	for _, role := range rolesSecrets {
+		if role.SecretName != emptyName {
+			noRolesProvided = false
+		}
+	}
+
+	if noRolesProvided {
+		return nil, nil
+	}
+
+	for _, secret := range rolesSecrets {
+		infraRoles, err := c.getInfrastructureRole(secret)
+
+		if err != nil || infraRoles == nil {
+			c.logger.Debugf("Cannot get infrastructure role: %+v", *secret)
+
+			if err != nil {
+				errors = append(errors, err)
+			}
+
+			continue
+		}
+
+		for _, r := range infraRoles {
+			roles = append(roles, r)
+		}
+	}
+
+	for _, r := range roles {
+		if _, exists := uniqRoles[r.Name]; exists {
+			msg := "Conflicting infrastructure roles: roles[%s] = (%q, %q)"
+			c.logger.Debugf(msg, r.Name, uniqRoles[r.Name], r)
+		}
+
+		uniqRoles[r.Name] = r
+	}
+
+	return uniqRoles, errors
+}
+
+// Generate list of users representing one infrastructure role based on its
+// description in various K8S objects. An infrastructure role could be
+// described by a secret and optionally a config map. The former should contain
+// the secret information, i.e. username, password, role. The latter could
+// contain an extensive description of the role and even override an
+// information obtained from the secret (except a password).
+//
+// This function returns a list of users to be compatible with the previous
+// behaviour, since we don't know how many users are actually encoded in the
+// secret if it's a "template" role. If the provided role is not a template
+// one, the result would be a list with just one user in it.
+//
+// FIXME: This dependency on two different objects is rather unnecessary
+// complicated, so let's get rid of it via deprecation process.
+func (c *Controller) getInfrastructureRole(
+	infraRole *config.InfrastructureRole) (
+	[]spec.PgUser, error) {
+
+	rolesSecret := infraRole.SecretName
+	roles := []spec.PgUser{}
+
+	if rolesSecret == emptyName {
 		// we don't have infrastructure roles defined, bail out
 		return nil, nil
 	}
 
 	infraRolesSecret, err := c.KubeClient.
 		Secrets(rolesSecret.Namespace).
-		Get(rolesSecret.Name, metav1.GetOptions{})
+		Get(context.TODO(), rolesSecret.Name, metav1.GetOptions{})
 	if err != nil {
-		c.logger.Debugf("infrastructure roles secret name: %q", *rolesSecret)
-		return nil, fmt.Errorf("could not get infrastructure roles secret: %v", err)
+		msg := "could not get infrastructure roles secret %s/%s: %v"
+		return nil, fmt.Errorf(msg, rolesSecret.Namespace, rolesSecret.Name, err)
 	}
 
 	secretData := infraRolesSecret.Data
-	result := make(map[string]spec.PgUser)
-Users:
-	// in worst case we would have one line per user
-	for i := 1; i <= len(secretData); i++ {
-		properties := []string{"user", "password", "inrole"}
-		t := spec.PgUser{Origin: spec.RoleOriginInfrastructure}
-		for _, p := range properties {
-			key := fmt.Sprintf("%s%d", p, i)
-			if val, present := secretData[key]; !present {
-				if p == "user" {
-					// exit when the user name with the next sequence id is absent
-					break Users
-				}
-			} else {
-				s := string(val)
-				switch p {
-				case "user":
-					t.Name = s
-				case "password":
-					t.Password = s
-				case "inrole":
-					t.MemberOf = append(t.MemberOf, s)
-				default:
-					c.logger.Warningf("unknown key %q", p)
-				}
+
+	if infraRole.Template {
+	Users:
+		for i := 1; i <= len(secretData); i++ {
+			properties := []string{
+				infraRole.UserKey,
+				infraRole.PasswordKey,
+				infraRole.RoleKey,
 			}
-			delete(secretData, key)
+			t := spec.PgUser{Origin: spec.RoleOriginInfrastructure}
+			for _, p := range properties {
+				key := fmt.Sprintf("%s%d", p, i)
+				if val, present := secretData[key]; !present {
+					if p == "user" {
+						// exit when the user name with the next sequence id is
+						// absent
+						break Users
+					}
+				} else {
+					s := string(val)
+					switch p {
+					case "user":
+						t.Name = s
+					case "password":
+						t.Password = s
+					case "inrole":
+						t.MemberOf = append(t.MemberOf, s)
+					default:
+						c.logger.Warningf("unknown key %q", p)
+					}
+				}
+				// XXX: This is a part of the original implementation, which is
+				// rather obscure. Why do we delete this key? Wouldn't it be
+				// used later in comparison for configmap?
+				delete(secretData, key)
+			}
+
+			if t.Valid() {
+				roles = append(roles, t)
+			} else {
+				msg := "infrastructure role %q is not complete and ignored"
+				c.logger.Warningf(msg, t)
+			}
+		}
+	} else {
+		roleDescr := &spec.PgUser{Origin: spec.RoleOriginInfrastructure}
+
+		if details, exists := secretData[infraRole.Details]; exists {
+			if err := yaml.Unmarshal(details, &roleDescr); err != nil {
+				return nil, fmt.Errorf("could not decode yaml role: %v", err)
+			}
+		} else {
+			roleDescr.Name = util.Coalesce(string(secretData[infraRole.UserKey]), infraRole.DefaultUserValue)
+			roleDescr.Password = string(secretData[infraRole.PasswordKey])
+			roleDescr.MemberOf = append(roleDescr.MemberOf,
+				util.Coalesce(string(secretData[infraRole.RoleKey]), infraRole.DefaultRoleValue))
 		}
 
-		if t.Name != "" {
-			if t.Password == "" {
-				c.logger.Warningf("infrastructure role %q has no password defined and is ignored", t.Name)
-				continue
-			}
-			result[t.Name] = t
+		if !roleDescr.Valid() {
+			msg := "infrastructure role %q is not complete and ignored"
+			c.logger.Warningf(msg, roleDescr)
+
+			return nil, nil
 		}
+
+		if roleDescr.Name == "" {
+			msg := "infrastructure role %q has no name defined and is ignored"
+			c.logger.Warningf(msg, roleDescr.Name)
+			return nil, nil
+		}
+
+		if roleDescr.Password == "" {
+			msg := "infrastructure role %q has no password defined and is ignored"
+			c.logger.Warningf(msg, roleDescr.Name)
+			return nil, nil
+		}
+
+		roles = append(roles, *roleDescr)
 	}
 
-	// perhaps we have some map entries with usernames, passwords, let's check if we have those users in the configmap
-	if infraRolesMap, err := c.KubeClient.ConfigMaps(rolesSecret.Namespace).Get(rolesSecret.Name, metav1.GetOptions{}); err == nil {
+	// Now plot twist. We need to check if there is a configmap with the same
+	// name and extract a role description if it exists.
+	infraRolesMap, err := c.KubeClient.
+		ConfigMaps(rolesSecret.Namespace).
+		Get(context.TODO(), rolesSecret.Name, metav1.GetOptions{})
+	if err == nil {
 		// we have a configmap with username - json description, let's read and decode it
 		for role, s := range infraRolesMap.Data {
 			roleDescr, err := readDecodedRole(s)
@@ -179,20 +386,43 @@ Users:
 			}
 			roleDescr.Name = role
 			roleDescr.Origin = spec.RoleOriginInfrastructure
-			result[role] = *roleDescr
+			roles = append(roles, *roleDescr)
 		}
 	}
 
-	if len(secretData) > 0 {
-		c.logger.Warningf("%d unprocessed entries in the infrastructure roles secret,"+
-			" checking configmap %v", len(secretData), rolesSecret.Name)
-		c.logger.Info(`infrastructure role entries should be in the {key}{id} format,` +
-			` where {key} can be either of "user", "password", "inrole" and the {id}` +
-			` a monotonically increasing integer starting with 1`)
-		c.logger.Debugf("unprocessed entries: %#v", secretData)
+	// TODO: check for role collisions
+	return roles, nil
+}
+
+func (c *Controller) loadPostgresTeams() {
+	// reset team map
+	c.pgTeamMap = teams.PostgresTeamMap{}
+
+	pgTeams, err := c.KubeClient.PostgresTeamsGetter.PostgresTeams(c.opConfig.WatchedNamespace).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		c.logger.Errorf("could not list postgres team objects: %v", err)
 	}
 
-	return result, nil
+	c.pgTeamMap.Load(pgTeams)
+	c.logger.Debugf("Internal Postgres Team Cache: %#v", c.pgTeamMap)
+}
+
+func (c *Controller) postgresTeamAdd(obj interface{}) {
+	pgTeam, ok := obj.(*acidv1.PostgresTeam)
+	if !ok {
+		c.logger.Errorf("could not cast to PostgresTeam spec")
+	}
+	c.logger.Debugf("PostgreTeam %q added. Reloading postgres team CRDs and overwriting cached map", pgTeam.Name)
+	c.loadPostgresTeams()
+}
+
+func (c *Controller) postgresTeamUpdate(prev, obj interface{}) {
+	pgTeam, ok := obj.(*acidv1.PostgresTeam)
+	if !ok {
+		c.logger.Errorf("could not cast to PostgresTeam spec")
+	}
+	c.logger.Debugf("PostgreTeam %q updated. Reloading postgres team CRDs and overwriting cached map", pgTeam.Name)
+	c.loadPostgresTeams()
 }
 
 func (c *Controller) podClusterName(pod *v1.Pod) spec.NamespacedName {

pkg/controller/util_test.go

@@ -8,26 +8,31 @@ import (
 	b64 "encoding/base64"
 
 	"github.com/zalando/postgres-operator/pkg/spec"
+	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const (
-	testInfrastructureRolesSecretName = "infrastructureroles-test"
+	testInfrastructureRolesOldSecretName = "infrastructureroles-old-test"
+	testInfrastructureRolesNewSecretName = "infrastructureroles-new-test"
 )
 
-func newMockController() *Controller {
-	controller := NewController(&spec.ControllerConfig{})
+func newUtilTestController() *Controller {
+	controller := NewController(&spec.ControllerConfig{}, "util-test")
 	controller.opConfig.ClusterNameLabel = "cluster-name"
 	controller.opConfig.InfrastructureRolesSecretName =
-		spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesSecretName}
+		spec.NamespacedName{
+			Namespace: v1.NamespaceDefault,
+			Name:      testInfrastructureRolesOldSecretName,
+		}
 	controller.opConfig.Workers = 4
 	controller.KubeClient = k8sutil.NewMockKubernetesClient()
 	return controller
 }
 
-var mockController = newMockController()
+var utilTestController = newUtilTestController()
 
 func TestPodClusterName(t *testing.T) {
 	var testTable = []struct {
@@ -43,7 +48,7 @@ func TestPodClusterName(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{
 					Namespace: v1.NamespaceDefault,
 					Labels: map[string]string{
-						mockController.opConfig.ClusterNameLabel: "testcluster",
+						utilTestController.opConfig.ClusterNameLabel: "testcluster",
 					},
 				},
 			},
@@ -51,7 +56,7 @@ func TestPodClusterName(t *testing.T) {
 		},
 	}
 	for _, test := range testTable {
-		resp := mockController.podClusterName(test.in)
+		resp := utilTestController.podClusterName(test.in)
 		if resp != test.expected {
 			t.Errorf("expected response %v does not match the actual %v", test.expected, resp)
 		}
@@ -73,31 +78,39 @@ func TestClusterWorkerID(t *testing.T) {
 		},
 	}
 	for _, test := range testTable {
-		resp := mockController.clusterWorkerID(test.in)
+		resp := utilTestController.clusterWorkerID(test.in)
 		if resp != test.expected {
 			t.Errorf("expected response %v does not match the actual %v", test.expected, resp)
 		}
 	}
 }
 
-func TestGetInfrastructureRoles(t *testing.T) {
+// Test functionality of getting infrastructure roles from their description in
+// corresponding secrets. Here we test only common stuff (e.g. when a secret do
+// not exist, or empty) and the old format.
+func TestOldInfrastructureRoleFormat(t *testing.T) {
 	var testTable = []struct {
-		secretName    spec.NamespacedName
-		expectedRoles map[string]spec.PgUser
-		expectedError error
+		secretName     spec.NamespacedName
+		expectedRoles  map[string]spec.PgUser
+		expectedErrors []error
 	}{
 		{
+			// empty secret name
 			spec.NamespacedName{},
 			nil,
 			nil,
 		},
 		{
+			// secret does not exist
 			spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: "null"},
-			nil,
-			fmt.Errorf(`could not get infrastructure roles secret: NotFound`),
+			map[string]spec.PgUser{},
+			[]error{fmt.Errorf(`could not get infrastructure roles secret default/null: NotFound`)},
 		},
 		{
-			spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesSecretName},
+			spec.NamespacedName{
+				Namespace: v1.NamespaceDefault,
+				Name:      testInfrastructureRolesOldSecretName,
+			},
 			map[string]spec.PgUser{
 				"testrole": {
 					Name:     "testrole",
@@ -116,15 +129,354 @@ func TestGetInfrastructureRoles(t *testing.T) {
 		},
 	}
 	for _, test := range testTable {
-		roles, err := mockController.getInfrastructureRoles(&test.secretName)
-		if err != test.expectedError {
-			if err != nil && test.expectedError != nil && err.Error() == test.expectedError.Error() {
-				continue
-			}
-			t.Errorf("expected error '%v' does not match the actual error '%v'", test.expectedError, err)
+		roles, errors := utilTestController.getInfrastructureRoles(
+			[]*config.InfrastructureRole{
+				&config.InfrastructureRole{
+					SecretName:  test.secretName,
+					UserKey:     "user",
+					PasswordKey: "password",
+					RoleKey:     "inrole",
+					Template:    true,
+				},
+			})
+
+		if len(errors) != len(test.expectedErrors) {
+			t.Errorf("expected error '%v' does not match the actual error '%v'",
+				test.expectedErrors, errors)
 		}
+
+		for idx := range errors {
+			err := errors[idx]
+			expectedErr := test.expectedErrors[idx]
+
+			if err != expectedErr {
+				if err != nil && expectedErr != nil && err.Error() == expectedErr.Error() {
+					continue
+				}
+				t.Errorf("expected error '%v' does not match the actual error '%v'",
+					expectedErr, err)
+			}
+		}
+
 		if !reflect.DeepEqual(roles, test.expectedRoles) {
-			t.Errorf("expected roles output %v does not match the actual %v", test.expectedRoles, roles)
+			t.Errorf("expected roles output %#v does not match the actual %#v",
+				test.expectedRoles, roles)
+		}
+	}
+}
+
+// Test functionality of getting infrastructure roles from their description in
+// corresponding secrets. Here we test the new format.
+func TestNewInfrastructureRoleFormat(t *testing.T) {
+	var testTable = []struct {
+		secrets        []spec.NamespacedName
+		expectedRoles  map[string]spec.PgUser
+		expectedErrors []error
+	}{
+		// one secret with one configmap
+		{
+			[]spec.NamespacedName{
+				spec.NamespacedName{
+					Namespace: v1.NamespaceDefault,
+					Name:      testInfrastructureRolesNewSecretName,
+				},
+			},
+			map[string]spec.PgUser{
+				"new-test-role": {
+					Name:     "new-test-role",
+					Origin:   spec.RoleOriginInfrastructure,
+					Password: "new-test-password",
+					MemberOf: []string{"new-test-inrole"},
+				},
+				"new-foobar": {
+					Name:     "new-foobar",
+					Origin:   spec.RoleOriginInfrastructure,
+					Password: b64.StdEncoding.EncodeToString([]byte("password")),
+					MemberOf: nil,
+					Flags:    []string{"createdb"},
+				},
+			},
+			nil,
+		},
+		// multiple standalone secrets
+		{
+			[]spec.NamespacedName{
+				spec.NamespacedName{
+					Namespace: v1.NamespaceDefault,
+					Name:      "infrastructureroles-new-test1",
+				},
+				spec.NamespacedName{
+					Namespace: v1.NamespaceDefault,
+					Name:      "infrastructureroles-new-test2",
+				},
+			},
+			map[string]spec.PgUser{
+				"new-test-role1": {
+					Name:     "new-test-role1",
+					Origin:   spec.RoleOriginInfrastructure,
+					Password: "new-test-password1",
+					MemberOf: []string{"new-test-inrole1"},
+				},
+				"new-test-role2": {
+					Name:     "new-test-role2",
+					Origin:   spec.RoleOriginInfrastructure,
+					Password: "new-test-password2",
+					MemberOf: []string{"new-test-inrole2"},
+				},
+			},
+			nil,
+		},
+	}
+	for _, test := range testTable {
+		definitions := []*config.InfrastructureRole{}
+		for _, secret := range test.secrets {
+			definitions = append(definitions, &config.InfrastructureRole{
+				SecretName:  secret,
+				UserKey:     "user",
+				PasswordKey: "password",
+				RoleKey:     "inrole",
+				Template:    false,
+			})
+		}
+
+		roles, errors := utilTestController.getInfrastructureRoles(definitions)
+		if len(errors) != len(test.expectedErrors) {
+			t.Errorf("expected error does not match the actual error:\n%+v\n%+v",
+				test.expectedErrors, errors)
+
+			// Stop and do not do any further checks
+			return
+		}
+
+		for idx := range errors {
+			err := errors[idx]
+			expectedErr := test.expectedErrors[idx]
+
+			if err != expectedErr {
+				if err != nil && expectedErr != nil && err.Error() == expectedErr.Error() {
+					continue
+				}
+				t.Errorf("expected error '%v' does not match the actual error '%v'",
+					expectedErr, err)
+			}
+		}
+
+		if !reflect.DeepEqual(roles, test.expectedRoles) {
+			t.Errorf("expected roles output/the actual:\n%#v\n%#v",
+				test.expectedRoles, roles)
+		}
+	}
+}
+
+// Tests for getting correct infrastructure roles definitions from present
+// configuration. E.g. in which secrets for which roles too look. The biggest
+// point here is compatibility of old and new formats of defining
+// infrastructure roles.
+func TestInfrastructureRoleDefinitions(t *testing.T) {
+	var testTable = []struct {
+		rolesDefs      []*config.InfrastructureRole
+		roleSecretName spec.NamespacedName
+		roleSecrets    string
+		expectedDefs   []*config.InfrastructureRole
+	}{
+		// only new CRD format
+		{
+			[]*config.InfrastructureRole{
+				&config.InfrastructureRole{
+					SecretName: spec.NamespacedName{
+						Namespace: v1.NamespaceDefault,
+						Name:      testInfrastructureRolesNewSecretName,
+					},
+					UserKey:     "test-user",
+					PasswordKey: "test-password",
+					RoleKey:     "test-role",
+					Template:    false,
+				},
+			},
+			spec.NamespacedName{},
+			"",
+			[]*config.InfrastructureRole{
+				&config.InfrastructureRole{
+					SecretName: spec.NamespacedName{
+						Namespace: v1.NamespaceDefault,
+						Name:      testInfrastructureRolesNewSecretName,
+					},
+					UserKey:     "test-user",
+					PasswordKey: "test-password",
+					RoleKey:     "test-role",
+					Template:    false,
+				},
+			},
+		},
+		// only new configmap format
+		{
+			[]*config.InfrastructureRole{},
+			spec.NamespacedName{},
+			"secretname: infrastructureroles-new-test, userkey: test-user, passwordkey: test-password, rolekey: test-role",
+			[]*config.InfrastructureRole{
+				&config.InfrastructureRole{
+					SecretName: spec.NamespacedName{
+						Namespace: v1.NamespaceDefault,
+						Name:      testInfrastructureRolesNewSecretName,
+					},
+					UserKey:     "test-user",
+					PasswordKey: "test-password",
+					RoleKey:     "test-role",
+					Template:    false,
+				},
+			},
+		},
+		// new configmap format with defaultRoleValue
+		{
+			[]*config.InfrastructureRole{},
+			spec.NamespacedName{},
+			"secretname: infrastructureroles-new-test, userkey: test-user, passwordkey: test-password, defaultrolevalue: test-role",
+			[]*config.InfrastructureRole{
+				&config.InfrastructureRole{
+					SecretName: spec.NamespacedName{
+						Namespace: v1.NamespaceDefault,
+						Name:      testInfrastructureRolesNewSecretName,
+					},
+					UserKey:          "test-user",
+					PasswordKey:      "test-password",
+					DefaultRoleValue: "test-role",
+					Template:         false,
+				},
+			},
+		},
+		// only old CRD and configmap format
+		{
+			[]*config.InfrastructureRole{},
+			spec.NamespacedName{
+				Namespace: v1.NamespaceDefault,
+				Name:      testInfrastructureRolesOldSecretName,
+			},
+			"",
+			[]*config.InfrastructureRole{
+				&config.InfrastructureRole{
+					SecretName: spec.NamespacedName{
+						Namespace: v1.NamespaceDefault,
+						Name:      testInfrastructureRolesOldSecretName,
+					},
+					UserKey:     "user",
+					PasswordKey: "password",
+					RoleKey:     "inrole",
+					Template:    true,
+				},
+			},
+		},
+		// both formats for CRD
+		{
+			[]*config.InfrastructureRole{
+				&config.InfrastructureRole{
+					SecretName: spec.NamespacedName{
+						Namespace: v1.NamespaceDefault,
+						Name:      testInfrastructureRolesNewSecretName,
+					},
+					UserKey:     "test-user",
+					PasswordKey: "test-password",
+					RoleKey:     "test-role",
+					Template:    false,
+				},
+			},
+			spec.NamespacedName{
+				Namespace: v1.NamespaceDefault,
+				Name:      testInfrastructureRolesOldSecretName,
+			},
+			"",
+			[]*config.InfrastructureRole{
+				&config.InfrastructureRole{
+					SecretName: spec.NamespacedName{
+						Namespace: v1.NamespaceDefault,
+						Name:      testInfrastructureRolesNewSecretName,
+					},
+					UserKey:     "test-user",
+					PasswordKey: "test-password",
+					RoleKey:     "test-role",
+					Template:    false,
+				},
+				&config.InfrastructureRole{
+					SecretName: spec.NamespacedName{
+						Namespace: v1.NamespaceDefault,
+						Name:      testInfrastructureRolesOldSecretName,
+					},
+					UserKey:     "user",
+					PasswordKey: "password",
+					RoleKey:     "inrole",
+					Template:    true,
+				},
+			},
+		},
+		// both formats for configmap
+		{
+			[]*config.InfrastructureRole{},
+			spec.NamespacedName{
+				Namespace: v1.NamespaceDefault,
+				Name:      testInfrastructureRolesOldSecretName,
+			},
+			"secretname: infrastructureroles-new-test, userkey: test-user, passwordkey: test-password, rolekey: test-role",
+			[]*config.InfrastructureRole{
+				&config.InfrastructureRole{
+					SecretName: spec.NamespacedName{
+						Namespace: v1.NamespaceDefault,
+						Name:      testInfrastructureRolesNewSecretName,
+					},
+					UserKey:     "test-user",
+					PasswordKey: "test-password",
+					RoleKey:     "test-role",
+					Template:    false,
+				},
+				&config.InfrastructureRole{
+					SecretName: spec.NamespacedName{
+						Namespace: v1.NamespaceDefault,
+						Name:      testInfrastructureRolesOldSecretName,
+					},
+					UserKey:     "user",
+					PasswordKey: "password",
+					RoleKey:     "inrole",
+					Template:    true,
+				},
+			},
+		},
+		// incorrect configmap format
+		{
+			[]*config.InfrastructureRole{},
+			spec.NamespacedName{},
+			"wrong-format",
+			[]*config.InfrastructureRole{},
+		},
+		// configmap without a secret
+		{
+			[]*config.InfrastructureRole{},
+			spec.NamespacedName{},
+			"userkey: test-user, passwordkey: test-password, rolekey: test-role",
+			[]*config.InfrastructureRole{},
+		},
+	}
+
+	for _, test := range testTable {
+		t.Logf("Test: %+v", test)
+		utilTestController.opConfig.InfrastructureRoles = test.rolesDefs
+		utilTestController.opConfig.InfrastructureRolesSecretName = test.roleSecretName
+		utilTestController.opConfig.InfrastructureRolesDefs = test.roleSecrets
+
+		defs := utilTestController.getInfrastructureRoleDefinitions()
+		if len(defs) != len(test.expectedDefs) {
+			t.Errorf("expected definitions does not match the actual:\n%#v\n%#v",
+				test.expectedDefs, defs)
+
+			// Stop and do not do any further checks
+			return
+		}
+
+		for idx := range defs {
+			def := defs[idx]
+			expectedDef := test.expectedDefs[idx]
+
+			if !reflect.DeepEqual(def, expectedDef) {
+				t.Errorf("expected definition/the actual:\n%#v\n%#v",
+					expectedDef, def)
+			}
 		}
 	}
 }