merge master

pkg/cluster/cluster.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"k8s.io/api/apps/v1beta1"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	policybeta1 "k8s.io/api/policy/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -494,7 +494,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 	defer func() {
 		if updateFailed {
 			c.setStatus(acidv1.ClusterStatusUpdateFailed)
-		} else if c.Status != acidv1.ClusterStatusRunning {
+		} else {
 			c.setStatus(acidv1.ClusterStatusRunning)
 		}
 	}()
@@ -709,11 +709,16 @@ func (c *Cluster) initRobotUsers() error {
 		if err != nil {
 			return fmt.Errorf("invalid flags for user %q: %v", username, err)
 		}
+		adminRole := ""
+		if c.OpConfig.EnableAdminRoleForUsers {
+			adminRole = c.OpConfig.TeamAdminRole
+		}
 		newRole := spec.PgUser{
-			Origin:   spec.RoleOriginManifest,
-			Name:     username,
-			Password: util.RandomPassword(constants.PasswordLength),
-			Flags:    flags,
+			Origin:    spec.RoleOriginManifest,
+			Name:      username,
+			Password:  util.RandomPassword(constants.PasswordLength),
+			Flags:     flags,
+			AdminRole: adminRole,
 		}
 		if currentRole, present := c.pgUsers[username]; present {
 			c.pgUsers[username] = c.resolveNameConflict(&currentRole, &newRole)
@@ -872,7 +877,7 @@ func (c *Cluster) GetStatus() *ClusterStatus {
 func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) error {
 
 	var err error
-	c.logger.Debugf("failing over from %q to %q", curMaster.Name, candidate)
+	c.logger.Debugf("switching over from %q to %q", curMaster.Name, candidate)
 
 	var wg sync.WaitGroup
 
@@ -898,12 +903,12 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e
 	}()
 
 	if err = c.patroni.Switchover(curMaster, candidate.Name); err == nil {
-		c.logger.Debugf("successfully failed over from %q to %q", curMaster.Name, candidate)
+		c.logger.Debugf("successfully switched over from %q to %q", curMaster.Name, candidate)
 		if err = <-podLabelErr; err != nil {
 			err = fmt.Errorf("could not get master pod label: %v", err)
 		}
 	} else {
-		err = fmt.Errorf("could not failover: %v", err)
+		err = fmt.Errorf("could not switch over: %v", err)
 	}
 
 	// signal the role label waiting goroutine to close the shop and go home

pkg/cluster/k8sres.go

@@ -18,6 +18,7 @@ import (
 	acidv1 "github.com/zalando-incubator/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando-incubator/postgres-operator/pkg/spec"
 	"github.com/zalando-incubator/postgres-operator/pkg/util"
+	"github.com/zalando-incubator/postgres-operator/pkg/util/config"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
 	"k8s.io/apimachinery/pkg/labels"
 )
@@ -92,18 +93,18 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources {
 	defaultRequests := acidv1.ResourceDescription{CPU: config.DefaultCPURequest, Memory: config.DefaultMemoryRequest}
 	defaultLimits := acidv1.ResourceDescription{CPU: config.DefaultCPULimit, Memory: config.DefaultMemoryLimit}
 
-	return acidv1.Resources{ResourceRequest: defaultRequests, ResourceLimits: defaultLimits}
+	return acidv1.Resources{ResourceRequests: defaultRequests, ResourceLimits: defaultLimits}
 }
 
 func generateResourceRequirements(resources acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) {
 	var err error
 
-	specRequests := resources.ResourceRequest
+	specRequests := resources.ResourceRequests
 	specLimits := resources.ResourceLimits
 
 	result := v1.ResourceRequirements{}
 
-	result.Requests, err = fillResourceList(specRequests, defaultResources.ResourceRequest)
+	result.Requests, err = fillResourceList(specRequests, defaultResources.ResourceRequests)
 	if err != nil {
 		return nil, fmt.Errorf("could not fill resource requests: %v", err)
 	}
@@ -289,6 +290,26 @@ func nodeAffinity(nodeReadinessLabel map[string]string) *v1.Affinity {
 	}
 }
 
+func generatePodAffinity(labels labels.Set, topologyKey string, nodeAffinity *v1.Affinity) *v1.Affinity {
+	// generate pod anti-affinity to avoid multiple pods of the same Postgres cluster in the same topology , e.g. node
+	podAffinity := v1.Affinity{
+		PodAntiAffinity: &v1.PodAntiAffinity{
+			RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{{
+				LabelSelector: &metav1.LabelSelector{
+					MatchLabels: labels,
+				},
+				TopologyKey: topologyKey,
+			}},
+		},
+	}
+
+	if nodeAffinity != nil && nodeAffinity.NodeAffinity != nil {
+		podAffinity.NodeAffinity = nodeAffinity.NodeAffinity
+	}
+
+	return &podAffinity
+}
+
 func tolerations(tolerationsSpec *[]v1.Toleration, podToleration map[string]string) []v1.Toleration {
 	// allow to override tolerations by postgresql manifest
 	if len(*tolerationsSpec) > 0 {
@@ -338,7 +359,6 @@ func generateSpiloContainer(
 	envVars []v1.EnvVar,
 	volumeMounts []v1.VolumeMount,
 ) *v1.Container {
-
 	privilegedMode := true
 	return &v1.Container{
 		Name:            name,
@@ -377,8 +397,8 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar,
 
 			resources, err := generateResourceRequirements(
 				makeResources(
-					sidecar.Resources.ResourceRequest.CPU,
-					sidecar.Resources.ResourceRequest.Memory,
+					sidecar.Resources.ResourceRequests.CPU,
+					sidecar.Resources.ResourceRequests.Memory,
 					sidecar.Resources.ResourceLimits.CPU,
 					sidecar.Resources.ResourceLimits.Memory,
 				),
@@ -396,10 +416,21 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar,
 	return nil, nil
 }
 
+// Check whether or not we're requested to mount an shm volume,
+// taking into account that PostgreSQL manifest has precedence.
+func mountShmVolumeNeeded(opConfig config.Config, pgSpec *acidv1.PostgresSpec) bool {
+	if pgSpec.ShmVolume != nil {
+		return *pgSpec.ShmVolume
+	}
+
+	return opConfig.ShmVolume
+}
+
 func generatePodTemplate(
 	namespace string,
 	labels labels.Set,
 	spiloContainer *v1.Container,
+	initContainers []v1.Container,
 	sidecarContainers []v1.Container,
 	tolerationsSpec *[]v1.Toleration,
 	nodeAffinity *v1.Affinity,
@@ -407,6 +438,9 @@ func generatePodTemplate(
 	podServiceAccountName string,
 	kubeIAMRole string,
 	priorityClassName string,
+	shmVolume bool,
+	podAntiAffinity bool,
+	podAntiAffinityTopologyKey string,
 ) (*v1.PodTemplateSpec, error) {
 
 	terminateGracePeriodSeconds := terminateGracePeriod
@@ -417,10 +451,17 @@ func generatePodTemplate(
 		ServiceAccountName:            podServiceAccountName,
 		TerminationGracePeriodSeconds: &terminateGracePeriodSeconds,
 		Containers:                    containers,
+		InitContainers:                initContainers,
 		Tolerations:                   *tolerationsSpec,
 	}
 
-	if nodeAffinity != nil {
+	if shmVolume {
+		addShmVolume(&podSpec)
+	}
+
+	if podAntiAffinity {
+		podSpec.Affinity = generatePodAffinity(labels, podAntiAffinityTopologyKey, nodeAffinity)
+	} else if nodeAffinity != nil {
 		podSpec.Affinity = nodeAffinity
 	}
 
@@ -475,6 +516,18 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri
 			Name:  "PGUSER_SUPERUSER",
 			Value: c.OpConfig.SuperUsername,
 		},
+		{
+			Name:  "KUBERNETES_SCOPE_LABEL",
+			Value: c.OpConfig.ClusterNameLabel,
+		},
+		{
+			Name:  "KUBERNETES_ROLE_LABEL",
+			Value: c.OpConfig.PodRoleLabel,
+		},
+		{
+			Name:  "KUBERNETES_LABELS",
+			Value: labels.Set(c.OpConfig.ClusterLabels).String(),
+		},
 		{
 			Name: "PGPASSWORD_SUPERUSER",
 			ValueFrom: &v1.EnvVarSource{
@@ -629,7 +682,7 @@ func getBucketScopeSuffix(uid string) string {
 
 func makeResources(cpuRequest, memoryRequest, cpuLimit, memoryLimit string) acidv1.Resources {
 	return acidv1.Resources{
-		ResourceRequest: acidv1.ResourceDescription{
+		ResourceRequests: acidv1.ResourceDescription{
 			CPU:    cpuRequest,
 			Memory: memoryRequest,
 		},
@@ -648,6 +701,61 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State
 		podTemplate         *v1.PodTemplateSpec
 		volumeClaimTemplate *v1.PersistentVolumeClaim
 	)
+
+	// Improve me. Please.
+	if c.OpConfig.SetMemoryRequestToLimit {
+
+		// controller adjusts the default memory request at operator startup
+
+		request := spec.Resources.ResourceRequests.Memory
+		if request == "" {
+			request = c.OpConfig.DefaultMemoryRequest
+		}
+
+		limit := spec.Resources.ResourceLimits.Memory
+		if limit == "" {
+			limit = c.OpConfig.DefaultMemoryLimit
+		}
+
+		isSmaller, err := util.RequestIsSmallerThanLimit(request, limit)
+		if err != nil {
+			return nil, err
+		}
+		if isSmaller {
+			c.logger.Warningf("The memory request of %v for the Postgres container is increased to match the memory limit of %v.", request, limit)
+			spec.Resources.ResourceRequests.Memory = limit
+
+		}
+
+		// controller adjusts the Scalyr sidecar request at operator startup
+		// as this sidecar is managed separately
+
+		// adjust sidecar containers defined for that particular cluster
+		for _, sidecar := range spec.Sidecars {
+
+			// TODO #413
+			sidecarRequest := sidecar.Resources.ResourceRequests.Memory
+			if request == "" {
+				request = c.OpConfig.DefaultMemoryRequest
+			}
+
+			sidecarLimit := sidecar.Resources.ResourceLimits.Memory
+			if limit == "" {
+				limit = c.OpConfig.DefaultMemoryLimit
+			}
+
+			isSmaller, err := util.RequestIsSmallerThanLimit(sidecarRequest, sidecarLimit)
+			if err != nil {
+				return nil, err
+			}
+			if isSmaller {
+				c.logger.Warningf("The memory request of %v for the %v sidecar container is increased to match the memory limit of %v.", sidecar.Resources.ResourceRequests.Memory, sidecar.Name, sidecar.Resources.ResourceLimits.Memory)
+				sidecar.Resources.ResourceRequests.Memory = sidecar.Resources.ResourceLimits.Memory
+			}
+		}
+
+	}
+
 	defaultResources := c.makeDefaultResources()
 
 	resourceRequirements, err := generateResourceRequirements(spec.Resources, defaultResources)
@@ -674,8 +782,8 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State
 
 	// generate environment variables for the spilo container
 	spiloEnvVars := deduplicateEnvVars(
-		c.generateSpiloPodEnvVars(c.Postgresql.GetUID(), spiloConfiguration, &spec.Clone, customPodEnvVarsList),
-		c.containerName(), c.logger)
+		c.generateSpiloPodEnvVars(c.Postgresql.GetUID(), spiloConfiguration, &spec.Clone,
+			customPodEnvVarsList), c.containerName(), c.logger)
 
 	// pickup the docker image for the spilo container
 	effectiveDockerImage := util.Coalesce(spec.DockerImage, c.OpConfig.DockerImage)
@@ -683,9 +791,15 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State
 	volumeMounts := generateVolumeMounts()
 
 	// generate the spilo container
-	spiloContainer := generateSpiloContainer(c.containerName(), &effectiveDockerImage, resourceRequirements, spiloEnvVars, volumeMounts)
+	c.logger.Debugf("Generating Spilo container, environment variables: %v", spiloEnvVars)
+	spiloContainer := generateSpiloContainer(c.containerName(),
+		&effectiveDockerImage,
+		resourceRequirements,
+		spiloEnvVars,
+		volumeMounts,
+	)
 
-	// resolve conflicts between operator-global and per-cluster sidecards
+	// resolve conflicts between operator-global and per-cluster sidecars
 	sideCars := c.mergeSidecars(spec.Sidecars)
 
 	resourceRequirementsScalyrSidecar := makeResources(
@@ -714,18 +828,22 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State
 	tolerationSpec := tolerations(&spec.Tolerations, c.OpConfig.PodToleration)
 	effectivePodPriorityClassName := util.Coalesce(spec.PodPriorityClassName, c.OpConfig.PodPriorityClassName)
 
-	// generate pod template for the statefulset, based on the spilo container and sidecards
+	// generate pod template for the statefulset, based on the spilo container and sidecars
 	if podTemplate, err = generatePodTemplate(
 		c.Namespace,
 		c.labelsSet(true),
 		spiloContainer,
+		spec.InitContainers,
 		sidecarContainers,
 		&tolerationSpec,
 		nodeAffinity(c.OpConfig.NodeReadinessLabel),
 		int64(c.OpConfig.PodTerminateGracePeriod.Seconds()),
 		c.OpConfig.PodServiceAccountName,
 		c.OpConfig.KubeIAMRole,
-		effectivePodPriorityClassName); err != nil {
+		effectivePodPriorityClassName,
+		mountShmVolumeNeeded(c.OpConfig, spec),
+		c.OpConfig.EnablePodAntiAffinity,
+		c.OpConfig.PodAntiAffinityTopologyKey); err != nil {
 		return nil, fmt.Errorf("could not generate pod template: %v", err)
 	}
 
@@ -832,6 +950,32 @@ func (c *Cluster) getNumberOfInstances(spec *acidv1.PostgresSpec) int32 {
 	return newcur
 }
 
+// To avoid issues with limited /dev/shm inside docker environment, when
+// PostgreSQL can't allocate enough of dsa segments from it, we can
+// mount an extra memory volume
+//
+// see https://docs.okd.io/latest/dev_guide/shared_memory.html
+func addShmVolume(podSpec *v1.PodSpec) {
+	volumes := append(podSpec.Volumes, v1.Volume{
+		Name: constants.ShmVolumeName,
+		VolumeSource: v1.VolumeSource{
+			EmptyDir: &v1.EmptyDirVolumeSource{
+				Medium: "Memory",
+			},
+		},
+	})
+
+	pgIdx := constants.PostgresContainerIdx
+	mounts := append(podSpec.Containers[pgIdx].VolumeMounts,
+		v1.VolumeMount{
+			Name:      constants.ShmVolumeName,
+			MountPath: constants.ShmVolumePath,
+		})
+
+	podSpec.Containers[0].VolumeMounts = mounts
+	podSpec.Volumes = volumes
+}
+
 func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string) (*v1.PersistentVolumeClaim, error) {
 
 	var storageClassName *string
@@ -959,7 +1103,7 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec)
 	}
 
 	if role == Replica {
-		serviceSpec.Selector = c.roleLabelsSet(role)
+		serviceSpec.Selector = c.roleLabelsSet(false, role)
 	}
 
 	var annotations map[string]string
@@ -982,6 +1126,13 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec)
 			constants.ZalandoDNSNameAnnotation: dnsName,
 			constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
 		}
+
+		if len(c.OpConfig.CustomServiceAnnotations) != 0 {
+			c.logger.Debugf("There are custom annotations defined, creating them.")
+			for customAnnotationKey, customAnnotationValue := range c.OpConfig.CustomServiceAnnotations {
+				annotations[customAnnotationKey] = customAnnotationValue
+			}
+		}
 	} else if role == Replica {
 		// before PR #258, the replica service was only created if allocated a LB
 		// now we always create the service but warn if the LB is absent
@@ -992,7 +1143,7 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec)
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        c.serviceName(role),
 			Namespace:   c.Namespace,
-			Labels:      c.roleLabelsSet(role),
+			Labels:      c.roleLabelsSet(true, role),
 			Annotations: annotations,
 		},
 		Spec: serviceSpec,
@@ -1006,7 +1157,7 @@ func (c *Cluster) generateEndpoint(role PostgresRole, subsets []v1.EndpointSubse
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      c.endpointName(role),
 			Namespace: c.Namespace,
-			Labels:    c.roleLabelsSet(role),
+			Labels:    c.roleLabelsSet(true, role),
 		},
 	}
 	if len(subsets) > 0 {
@@ -1070,7 +1221,7 @@ func (c *Cluster) generatePodDisruptionBudget() *policybeta1.PodDisruptionBudget
 		Spec: policybeta1.PodDisruptionBudgetSpec{
 			MinAvailable: &minAvailable,
 			Selector: &metav1.LabelSelector{
-				MatchLabels: c.roleLabelsSet(Master),
+				MatchLabels: c.roleLabelsSet(false, Master),
 			},
 		},
 	}

pkg/cluster/k8sres_test.go

@@ -1,8 +1,11 @@
 package cluster
 
 import (
+	"k8s.io/api/core/v1"
+
 	acidv1 "github.com/zalando-incubator/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/config"
+	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil"
 	"testing"
 )
@@ -75,3 +78,54 @@ func TestCreateLoadBalancerLogic(t *testing.T) {
 		}
 	}
 }
+
+func TestShmVolume(t *testing.T) {
+	testName := "TestShmVolume"
+	tests := []struct {
+		subTest string
+		podSpec *v1.PodSpec
+		shmPos  int
+	}{
+		{
+			subTest: "empty PodSpec",
+			podSpec: &v1.PodSpec{
+				Volumes: []v1.Volume{},
+				Containers: []v1.Container{
+					v1.Container{
+						VolumeMounts: []v1.VolumeMount{},
+					},
+				},
+			},
+			shmPos: 0,
+		},
+		{
+			subTest: "non empty PodSpec",
+			podSpec: &v1.PodSpec{
+				Volumes: []v1.Volume{v1.Volume{}},
+				Containers: []v1.Container{
+					v1.Container{
+						VolumeMounts: []v1.VolumeMount{
+							v1.VolumeMount{},
+						},
+					},
+				},
+			},
+			shmPos: 1,
+		},
+	}
+	for _, tt := range tests {
+		addShmVolume(tt.podSpec)
+
+		volumeName := tt.podSpec.Volumes[tt.shmPos].Name
+		volumeMountName := tt.podSpec.Containers[0].VolumeMounts[tt.shmPos].Name
+
+		if volumeName != constants.ShmVolumeName {
+			t.Errorf("%s %s: Expected volume %s was not created, have %s instead",
+				testName, tt.subTest, constants.ShmVolumeName, volumeName)
+		}
+		if volumeMountName != constants.ShmVolumeName {
+			t.Errorf("%s %s: Expected mount %s was not created, have %s instead",
+				testName, tt.subTest, constants.ShmVolumeName, volumeMountName)
+		}
+	}
+}

pkg/cluster/pod.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"math/rand"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/zalando-incubator/postgres-operator/pkg/spec"
@@ -27,7 +27,7 @@ func (c *Cluster) listPods() ([]v1.Pod, error) {
 
 func (c *Cluster) getRolePods(role PostgresRole) ([]v1.Pod, error) {
 	listOptions := metav1.ListOptions{
-		LabelSelector: c.roleLabelsSet(role).String(),
+		LabelSelector: c.roleLabelsSet(false, role).String(),
 	}
 
 	pods, err := c.KubeClient.Pods(c.Namespace).List(listOptions)
@@ -77,11 +77,7 @@ func (c *Cluster) deletePod(podName spec.NamespacedName) error {
 		return err
 	}
 
-	if err := c.waitForPodDeletion(ch); err != nil {
-		return err
-	}
-
-	return nil
+	return c.waitForPodDeletion(ch)
 }
 
 func (c *Cluster) unregisterPodSubscriber(podName spec.NamespacedName) {
@@ -122,7 +118,7 @@ func (c *Cluster) movePodFromEndOfLifeNode(pod *v1.Pod) (*v1.Pod, error) {
 	if eol, err = c.podIsEndOfLife(pod); err != nil {
 		return nil, fmt.Errorf("could not get node %q: %v", pod.Spec.NodeName, err)
 	} else if !eol {
-		c.logger.Infof("pod %q is already on a live node", podName)
+		c.logger.Infof("check failed: pod %q is already on a live node", podName)
 		return pod, nil
 	}
 
@@ -162,7 +158,7 @@ func (c *Cluster) masterCandidate(oldNodeName string) (*v1.Pod, error) {
 	}
 
 	if len(replicas) == 0 {
-		c.logger.Warningf("no available master candidates, migration will cause longer downtime of the master instance")
+		c.logger.Warningf("no available master candidates, migration will cause longer downtime of Postgres cluster")
 		return nil, nil
 	}
 
@@ -193,18 +189,18 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
 		return fmt.Errorf("could not get pod: %v", err)
 	}
 
-	c.logger.Infof("migrating master pod %q", podName)
+	c.logger.Infof("starting process to migrate master pod %q", podName)
 
 	if eol, err = c.podIsEndOfLife(oldMaster); err != nil {
 		return fmt.Errorf("could not get node %q: %v", oldMaster.Spec.NodeName, err)
 	}
 	if !eol {
-		c.logger.Debugf("pod is already on a live node")
+		c.logger.Debugf("no action needed: master pod is already on a live node")
 		return nil
 	}
 
 	if role := PostgresRole(oldMaster.Labels[c.OpConfig.PodRoleLabel]); role != Master {
-		c.logger.Warningf("pod %q is not a master", podName)
+		c.logger.Warningf("no action needed: pod %q is not the master (anymore)", podName)
 		return nil
 	}
 	// we must have a statefulset in the cluster for the migration to work
@@ -219,10 +215,10 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
 	// We may not have a cached statefulset if the initial cluster sync has aborted, revert to the spec in that case.
 	if *c.Statefulset.Spec.Replicas > 1 {
 		if masterCandidatePod, err = c.masterCandidate(oldMaster.Spec.NodeName); err != nil {
-			return fmt.Errorf("could not get new master candidate: %v", err)
+			return fmt.Errorf("could not find suitable replica pod as candidate for failover: %v", err)
 		}
 	} else {
-		c.logger.Warningf("single master pod for cluster %q, migration will cause longer downtime of the master instance", c.clusterName())
+		c.logger.Warningf("migrating single pod cluster %q, this will cause downtime of the Postgres cluster until pod is back", c.clusterName())
 	}
 
 	// there are two cases for each postgres cluster that has its master pod on the node to migrate from:
@@ -256,15 +252,15 @@ func (c *Cluster) MigrateReplicaPod(podName spec.NamespacedName, fromNodeName st
 		return fmt.Errorf("could not get pod: %v", err)
 	}
 
-	c.logger.Infof("migrating replica pod %q", podName)
+	c.logger.Infof("migrating replica pod %q to live node", podName)
 
 	if replicaPod.Spec.NodeName != fromNodeName {
-		c.logger.Infof("pod %q has already migrated to node %q", podName, replicaPod.Spec.NodeName)
+		c.logger.Infof("check failed: pod %q has already migrated to node %q", podName, replicaPod.Spec.NodeName)
 		return nil
 	}
 
 	if role := PostgresRole(replicaPod.Labels[c.OpConfig.PodRoleLabel]); role != Replica {
-		return fmt.Errorf("pod %q is not a replica", podName)
+		return fmt.Errorf("check failed: pod %q is not a replica", podName)
 	}
 
 	_, err = c.movePodFromEndOfLifeNode(replicaPod)
@@ -296,7 +292,7 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) {
 }
 
 func (c *Cluster) recreatePods() error {
-	c.setProcessName("recreating pods")
+	c.setProcessName("starting to recreate pods")
 	ls := c.labelsSet(false)
 	namespace := c.Namespace
 
@@ -337,10 +333,10 @@ func (c *Cluster) recreatePods() error {
 		// failover if we have not observed a master pod when re-creating former replicas.
 		if newMasterPod == nil && len(replicas) > 0 {
 			if err := c.Switchover(masterPod, masterCandidate(replicas)); err != nil {
-				c.logger.Warningf("could not perform failover: %v", err)
+				c.logger.Warningf("could not perform switch over: %v", err)
 			}
 		} else if newMasterPod == nil && len(replicas) == 0 {
-			c.logger.Warningf("cannot switch master role before re-creating the pod: no replicas")
+			c.logger.Warningf("cannot perform switch over before re-creating the pod: no replicas")
 		}
 		c.logger.Infof("recreating old master pod %q", util.NameFromMeta(masterPod.ObjectMeta))
 

pkg/cluster/resources.go

@@ -437,7 +437,11 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error
 func (c *Cluster) deleteService(role PostgresRole) error {
 	c.logger.Debugf("deleting service %s", role)
 
-	service := c.Services[role]
+	service, ok := c.Services[role]
+	if !ok {
+		c.logger.Debugf("No service for %s role was found, nothing to delete", role)
+		return nil
+	}
 
 	if err := c.KubeClient.Services(service.Namespace).Delete(service.Name, c.deleteOptions); err != nil {
 		return err

pkg/cluster/types.go

@@ -1,12 +1,13 @@
 package cluster
 
 import (
+	"time"
+
 	acidv1 "github.com/zalando-incubator/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"k8s.io/api/apps/v1beta1"
 	"k8s.io/api/core/v1"
 	policybeta1 "k8s.io/api/policy/v1beta1"
 	"k8s.io/apimachinery/pkg/types"
-	"time"
 )
 
 // PostgresRole describes role of the node
@@ -20,6 +21,7 @@ const (
 	Replica PostgresRole = "replica"
 )
 
+// PodEventType represents the type of a pod-related event
 type PodEventType string
 
 // Possible values for the EventType

pkg/cluster/util.go

@@ -389,6 +389,19 @@ func (c *Cluster) labelsSet(shouldAddExtraLabels bool) labels.Set {
 	if shouldAddExtraLabels {
 		// enables filtering resources owned by a team
 		lbls["team"] = c.Postgresql.Spec.TeamID
+
+		// allow to inherit certain labels from the 'postgres' object
+		if spec, err := c.GetSpec(); err == nil {
+			for k, v := range spec.ObjectMeta.Labels {
+				for _, match := range c.OpConfig.InheritedLabels {
+					if k == match {
+						lbls[k] = v
+					}
+				}
+			}
+		} else {
+			c.logger.Warningf("could not get the list of InheritedLabels for cluster %q: %v", c.Name, err)
+		}
 	}
 
 	return labels.Set(lbls)
@@ -398,8 +411,8 @@ func (c *Cluster) labelsSelector() *metav1.LabelSelector {
 	return &metav1.LabelSelector{MatchLabels: c.labelsSet(false), MatchExpressions: nil}
 }
 
-func (c *Cluster) roleLabelsSet(role PostgresRole) labels.Set {
-	lbls := c.labelsSet(false)
+func (c *Cluster) roleLabelsSet(shouldAddExtraLabels bool, role PostgresRole) labels.Set {
+	lbls := c.labelsSet(shouldAddExtraLabels)
 	lbls[c.OpConfig.PodRoleLabel] = string(role)
 	return lbls
 }
@@ -460,6 +473,7 @@ func (c *Cluster) setSpec(newSpec *acidv1.Postgresql) {
 	c.specMu.Unlock()
 }
 
+// GetSpec returns a copy of the operator-side spec of a Postgres cluster in a thread-safe manner
 func (c *Cluster) GetSpec() (*acidv1.Postgresql, error) {
 	c.specMu.RLock()
 	defer c.specMu.RUnlock()