Merge branch 'master' into feature/actions-for-secret-syncing

pkg/cluster/cluster.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"k8s.io/api/apps/v1beta1"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	policybeta1 "k8s.io/api/policy/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -367,7 +367,9 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *v1beta1.StatefulSet) *comp
 		needsRollUpdate = true
 		reasons = append(reasons, "new statefulset's container specification doesn't match the current one")
 	} else {
-		needsRollUpdate, reasons = c.compareContainers(c.Statefulset, statefulSet)
+		var containerReasons []string
+		needsRollUpdate, containerReasons = c.compareContainers(c.Statefulset, statefulSet)
+		reasons = append(reasons, containerReasons...)
 	}
 	if len(c.Statefulset.Spec.Template.Spec.Containers) == 0 {
 		c.logger.Warningf("statefulset %q has no container", util.NameFromMeta(c.Statefulset.ObjectMeta))
@@ -375,7 +377,6 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *v1beta1.StatefulSet) *comp
 	}
 	// In the comparisons below, the needsReplace and needsRollUpdate flags are never reset, since checks fall through
 	// and the combined effect of all the changes should be applied.
-	// TODO: log all reasons for changing the statefulset, not just the last one.
 	// TODO: make sure this is in sync with generatePodTemplate, ideally by using the same list of fields to generate
 	// the template and the diff
 	if c.Statefulset.Spec.Template.Spec.ServiceAccountName != statefulSet.Spec.Template.Spec.ServiceAccountName {
@@ -386,7 +387,7 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *v1beta1.StatefulSet) *comp
 	if *c.Statefulset.Spec.Template.Spec.TerminationGracePeriodSeconds != *statefulSet.Spec.Template.Spec.TerminationGracePeriodSeconds {
 		needsReplace = true
 		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's terminationGracePeriodSeconds  doesn't match the current one")
+		reasons = append(reasons, "new statefulset's terminationGracePeriodSeconds doesn't match the current one")
 	}
 	if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Affinity, statefulSet.Spec.Template.Spec.Affinity) {
 		needsReplace = true
@@ -462,23 +463,23 @@ func newCheck(msg string, cond containerCondition) containerCheck {
 
 // compareContainers: compare containers from two stateful sets
 // and return:
-// * whether or not roll update is needed
+// * whether or not a rolling update is needed
 // * a list of reasons in a human readable format
 func (c *Cluster) compareContainers(setA, setB *v1beta1.StatefulSet) (bool, []string) {
 	reasons := make([]string, 0)
 	needsRollUpdate := false
 	checks := []containerCheck{
-		newCheck("new statefulset's container %d name doesn't match the current one",
+		newCheck("new statefulset's container %s (index %d) name doesn't match the current one",
 			func(a, b v1.Container) bool { return a.Name != b.Name }),
-		newCheck("new statefulset's container %d image doesn't match the current one",
+		newCheck("new statefulset's container %s (index %d) image doesn't match the current one",
 			func(a, b v1.Container) bool { return a.Image != b.Image }),
-		newCheck("new statefulset's container %d ports don't match the current one",
+		newCheck("new statefulset's container %s (index %d) ports don't match the current one",
 			func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Ports, b.Ports) }),
-		newCheck("new statefulset's container %d resources don't match the current ones",
+		newCheck("new statefulset's container %s (index %d) resources don't match the current ones",
 			func(a, b v1.Container) bool { return !compareResources(&a.Resources, &b.Resources) }),
-		newCheck("new statefulset's container %d environment doesn't match the current one",
+		newCheck("new statefulset's container %s (index %d) environment doesn't match the current one",
 			func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Env, b.Env) }),
-		newCheck("new statefulset's container %d environment sources don't match the current one",
+		newCheck("new statefulset's container %s (index %d) environment sources don't match the current one",
 			func(a, b v1.Container) bool { return !reflect.DeepEqual(a.EnvFrom, b.EnvFrom) }),
 	}
 
@@ -487,7 +488,7 @@ func (c *Cluster) compareContainers(setA, setB *v1beta1.StatefulSet) (bool, []st
 		for _, check := range checks {
 			if check.condition(containerA, containerB) {
 				needsRollUpdate = true
-				reasons = append(reasons, fmt.Sprintf(check.reason, index))
+				reasons = append(reasons, fmt.Sprintf(check.reason, containerA.Name, index))
 			}
 		}
 	}
@@ -754,11 +755,16 @@ func (c *Cluster) initRobotUsers() error {
 		if err != nil {
 			return fmt.Errorf("invalid flags for user %q: %v", username, err)
 		}
+		adminRole := ""
+		if c.OpConfig.EnableAdminRoleForUsers {
+			adminRole = c.OpConfig.TeamAdminRole
+		}
 		newRole := spec.PgUser{
-			Origin:   spec.RoleOriginManifest,
-			Name:     username,
-			Password: util.RandomPassword(constants.PasswordLength),
-			Flags:    flags,
+			Origin:    spec.RoleOriginManifest,
+			Name:      username,
+			Password:  util.RandomPassword(constants.PasswordLength),
+			Flags:     flags,
+			AdminRole: adminRole,
 		}
 		if currentRole, present := c.pgUsers[username]; present {
 			c.pgUsers[username] = c.resolveNameConflict(&currentRole, &newRole)
@@ -917,7 +923,7 @@ func (c *Cluster) GetStatus() *ClusterStatus {
 func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) error {
 
 	var err error
-	c.logger.Debugf("failing over from %q to %q", curMaster.Name, candidate)
+	c.logger.Debugf("switching over from %q to %q", curMaster.Name, candidate)
 
 	var wg sync.WaitGroup
 
@@ -943,12 +949,12 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e
 	}()
 
 	if err = c.patroni.Switchover(curMaster, candidate.Name); err == nil {
-		c.logger.Debugf("successfully failed over from %q to %q", curMaster.Name, candidate)
+		c.logger.Debugf("successfully switched over from %q to %q", curMaster.Name, candidate)
 		if err = <-podLabelErr; err != nil {
 			err = fmt.Errorf("could not get master pod label: %v", err)
 		}
 	} else {
-		err = fmt.Errorf("could not failover: %v", err)
+		err = fmt.Errorf("could not switch over: %v", err)
 	}
 
 	// signal the role label waiting goroutine to close the shop and go home

pkg/cluster/k8sres.go

@@ -18,6 +18,7 @@ import (
 	acidv1 "github.com/zalando-incubator/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando-incubator/postgres-operator/pkg/spec"
 	"github.com/zalando-incubator/postgres-operator/pkg/util"
+	"github.com/zalando-incubator/postgres-operator/pkg/util/config"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
 	"k8s.io/apimachinery/pkg/labels"
 )
@@ -36,11 +37,12 @@ type pgUser struct {
 }
 
 type patroniDCS struct {
-	TTL                      uint32                 `json:"ttl,omitempty"`
-	LoopWait                 uint32                 `json:"loop_wait,omitempty"`
-	RetryTimeout             uint32                 `json:"retry_timeout,omitempty"`
-	MaximumLagOnFailover     float32                `json:"maximum_lag_on_failover,omitempty"`
-	PGBootstrapConfiguration map[string]interface{} `json:"postgresql,omitempty"`
+	TTL                      uint32                       `json:"ttl,omitempty"`
+	LoopWait                 uint32                       `json:"loop_wait,omitempty"`
+	RetryTimeout             uint32                       `json:"retry_timeout,omitempty"`
+	MaximumLagOnFailover     float32                      `json:"maximum_lag_on_failover,omitempty"`
+	PGBootstrapConfiguration map[string]interface{}       `json:"postgresql,omitempty"`
+	Slots                    map[string]map[string]string `json:"slots,omitempty"`
 }
 
 type pgBootstrap struct {
@@ -91,18 +93,18 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources {
 	defaultRequests := acidv1.ResourceDescription{CPU: config.DefaultCPURequest, Memory: config.DefaultMemoryRequest}
 	defaultLimits := acidv1.ResourceDescription{CPU: config.DefaultCPULimit, Memory: config.DefaultMemoryLimit}
 
-	return acidv1.Resources{ResourceRequest: defaultRequests, ResourceLimits: defaultLimits}
+	return acidv1.Resources{ResourceRequests: defaultRequests, ResourceLimits: defaultLimits}
 }
 
 func generateResourceRequirements(resources acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) {
 	var err error
 
-	specRequests := resources.ResourceRequest
+	specRequests := resources.ResourceRequests
 	specLimits := resources.ResourceLimits
 
 	result := v1.ResourceRequirements{}
 
-	result.Requests, err = fillResourceList(specRequests, defaultResources.ResourceRequest)
+	result.Requests, err = fillResourceList(specRequests, defaultResources.ResourceRequests)
 	if err != nil {
 		return nil, fmt.Errorf("could not fill resource requests: %v", err)
 	}
@@ -215,6 +217,9 @@ PatroniInitDBParams:
 	if patroni.TTL != 0 {
 		config.Bootstrap.DCS.TTL = patroni.TTL
 	}
+	if patroni.Slots != nil {
+		config.Bootstrap.DCS.Slots = patroni.Slots
+	}
 
 	config.PgLocalConfiguration = make(map[string]interface{})
 	config.PgLocalConfiguration[patroniPGBinariesParameterName] = fmt.Sprintf(pgBinariesLocationTemplate, pg.PgVersion)
@@ -334,7 +339,6 @@ func generateSpiloContainer(
 	envVars []v1.EnvVar,
 	volumeMounts []v1.VolumeMount,
 ) *v1.Container {
-
 	privilegedMode := true
 	return &v1.Container{
 		Name:            name,
@@ -373,8 +377,8 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar,
 
 			resources, err := generateResourceRequirements(
 				makeResources(
-					sidecar.Resources.ResourceRequest.CPU,
-					sidecar.Resources.ResourceRequest.Memory,
+					sidecar.Resources.ResourceRequests.CPU,
+					sidecar.Resources.ResourceRequests.Memory,
 					sidecar.Resources.ResourceLimits.CPU,
 					sidecar.Resources.ResourceLimits.Memory,
 				),
@@ -392,6 +396,16 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar,
 	return nil, nil
 }
 
+// Check whether or not we're requested to mount an shm volume,
+// taking into account that PostgreSQL manifest has precedence.
+func mountShmVolumeNeeded(opConfig config.Config, pgSpec *acidv1.PostgresSpec) bool {
+	if pgSpec.ShmVolume != nil {
+		return *pgSpec.ShmVolume
+	}
+
+	return opConfig.ShmVolume
+}
+
 func generatePodTemplate(
 	namespace string,
 	labels labels.Set,
@@ -403,6 +417,7 @@ func generatePodTemplate(
 	podServiceAccountName string,
 	kubeIAMRole string,
 	priorityClassName string,
+	shmVolume bool,
 ) (*v1.PodTemplateSpec, error) {
 
 	terminateGracePeriodSeconds := terminateGracePeriod
@@ -416,6 +431,10 @@ func generatePodTemplate(
 		Tolerations:                   *tolerationsSpec,
 	}
 
+	if shmVolume {
+		addShmVolume(&podSpec)
+	}
+
 	if nodeAffinity != nil {
 		podSpec.Affinity = nodeAffinity
 	}
@@ -471,6 +490,18 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri
 			Name:  "PGUSER_SUPERUSER",
 			Value: c.OpConfig.SuperUsername,
 		},
+		{
+			Name:  "KUBERNETES_SCOPE_LABEL",
+			Value: c.OpConfig.ClusterNameLabel,
+		},
+		{
+			Name:  "KUBERNETES_ROLE_LABEL",
+			Value: c.OpConfig.PodRoleLabel,
+		},
+		{
+			Name:  "KUBERNETES_LABELS",
+			Value: labels.Set(c.OpConfig.ClusterLabels).String(),
+		},
 		{
 			Name: "PGPASSWORD_SUPERUSER",
 			ValueFrom: &v1.EnvVarSource{
@@ -621,7 +652,7 @@ func getBucketScopeSuffix(uid string) string {
 
 func makeResources(cpuRequest, memoryRequest, cpuLimit, memoryLimit string) acidv1.Resources {
 	return acidv1.Resources{
-		ResourceRequest: acidv1.ResourceDescription{
+		ResourceRequests: acidv1.ResourceDescription{
 			CPU:    cpuRequest,
 			Memory: memoryRequest,
 		},
@@ -640,6 +671,61 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State
 		podTemplate         *v1.PodTemplateSpec
 		volumeClaimTemplate *v1.PersistentVolumeClaim
 	)
+
+	// Improve me. Please.
+	if c.OpConfig.SetMemoryRequestToLimit {
+
+		// controller adjusts the default memory request at operator startup
+
+		request := spec.Resources.ResourceRequests.Memory
+		if request == "" {
+			request = c.OpConfig.DefaultMemoryRequest
+		}
+
+		limit := spec.Resources.ResourceLimits.Memory
+		if limit == "" {
+			limit = c.OpConfig.DefaultMemoryLimit
+		}
+
+		isSmaller, err := util.RequestIsSmallerThanLimit(request, limit)
+		if err != nil {
+			return nil, err
+		}
+		if isSmaller {
+			c.logger.Warningf("The memory request of %v for the Postgres container is increased to match the memory limit of %v.", request, limit)
+			spec.Resources.ResourceRequests.Memory = limit
+
+		}
+
+		// controller adjusts the Scalyr sidecar request at operator startup
+		// as this sidecar is managed separately
+
+		// adjust sidecar containers defined for that particular cluster
+		for _, sidecar := range spec.Sidecars {
+
+			// TODO #413
+			sidecarRequest := sidecar.Resources.ResourceRequests.Memory
+			if request == "" {
+				request = c.OpConfig.DefaultMemoryRequest
+			}
+
+			sidecarLimit := sidecar.Resources.ResourceLimits.Memory
+			if limit == "" {
+				limit = c.OpConfig.DefaultMemoryLimit
+			}
+
+			isSmaller, err := util.RequestIsSmallerThanLimit(sidecarRequest, sidecarLimit)
+			if err != nil {
+				return nil, err
+			}
+			if isSmaller {
+				c.logger.Warningf("The memory request of %v for the %v sidecar container is increased to match the memory limit of %v.", sidecar.Resources.ResourceRequests.Memory, sidecar.Name, sidecar.Resources.ResourceLimits.Memory)
+				sidecar.Resources.ResourceRequests.Memory = sidecar.Resources.ResourceLimits.Memory
+			}
+		}
+
+	}
+
 	defaultResources := c.makeDefaultResources()
 
 	resourceRequirements, err := generateResourceRequirements(spec.Resources, defaultResources)
@@ -666,8 +752,8 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State
 
 	// generate environment variables for the spilo container
 	spiloEnvVars := deduplicateEnvVars(
-		c.generateSpiloPodEnvVars(c.Postgresql.GetUID(), spiloConfiguration, &spec.Clone, customPodEnvVarsList),
-		c.containerName(), c.logger)
+		c.generateSpiloPodEnvVars(c.Postgresql.GetUID(), spiloConfiguration, &spec.Clone,
+			customPodEnvVarsList), c.containerName(), c.logger)
 
 	// pickup the docker image for the spilo container
 	effectiveDockerImage := util.Coalesce(spec.DockerImage, c.OpConfig.DockerImage)
@@ -675,9 +761,15 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State
 	volumeMounts := generateVolumeMounts()
 
 	// generate the spilo container
-	spiloContainer := generateSpiloContainer(c.containerName(), &effectiveDockerImage, resourceRequirements, spiloEnvVars, volumeMounts)
+	c.logger.Debugf("Generating Spilo container, environment variables: %v", spiloEnvVars)
+	spiloContainer := generateSpiloContainer(c.containerName(),
+		&effectiveDockerImage,
+		resourceRequirements,
+		spiloEnvVars,
+		volumeMounts,
+	)
 
-	// resolve conflicts between operator-global and per-cluster sidecards
+	// resolve conflicts between operator-global and per-cluster sidecars
 	sideCars := c.mergeSidecars(spec.Sidecars)
 
 	resourceRequirementsScalyrSidecar := makeResources(
@@ -706,7 +798,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State
 	tolerationSpec := tolerations(&spec.Tolerations, c.OpConfig.PodToleration)
 	effectivePodPriorityClassName := util.Coalesce(spec.PodPriorityClassName, c.OpConfig.PodPriorityClassName)
 
-	// generate pod template for the statefulset, based on the spilo container and sidecards
+	// generate pod template for the statefulset, based on the spilo container and sidecars
 	if podTemplate, err = generatePodTemplate(
 		c.Namespace,
 		c.labelsSet(true),
@@ -717,7 +809,8 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State
 		int64(c.OpConfig.PodTerminateGracePeriod.Seconds()),
 		c.OpConfig.PodServiceAccountName,
 		c.OpConfig.KubeIAMRole,
-		effectivePodPriorityClassName); err != nil {
+		effectivePodPriorityClassName,
+		mountShmVolumeNeeded(c.OpConfig, spec)); err != nil {
 		return nil, fmt.Errorf("could not generate pod template: %v", err)
 	}
 
@@ -824,6 +917,32 @@ func (c *Cluster) getNumberOfInstances(spec *acidv1.PostgresSpec) int32 {
 	return newcur
 }
 
+// To avoid issues with limited /dev/shm inside docker environment, when
+// PostgreSQL can't allocate enough of dsa segments from it, we can
+// mount an extra memory volume
+//
+// see https://docs.okd.io/latest/dev_guide/shared_memory.html
+func addShmVolume(podSpec *v1.PodSpec) {
+	volumes := append(podSpec.Volumes, v1.Volume{
+		Name: constants.ShmVolumeName,
+		VolumeSource: v1.VolumeSource{
+			EmptyDir: &v1.EmptyDirVolumeSource{
+				Medium: "Memory",
+			},
+		},
+	})
+
+	pgIdx := constants.PostgresContainerIdx
+	mounts := append(podSpec.Containers[pgIdx].VolumeMounts,
+		v1.VolumeMount{
+			Name:      constants.ShmVolumeName,
+			MountPath: constants.ShmVolumePath,
+		})
+
+	podSpec.Containers[0].VolumeMounts = mounts
+	podSpec.Volumes = volumes
+}
+
 func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string) (*v1.PersistentVolumeClaim, error) {
 
 	var storageClassName *string
@@ -958,16 +1077,17 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec)
 
 	if c.shouldCreateLoadBalancerForService(role, spec) {
 
-		// safe default value: lock load balancer to only local address unless overridden explicitly.
-		sourceRanges := []string{localHost}
-
-		allowedSourceRanges := spec.AllowedSourceRanges
-		if len(allowedSourceRanges) >= 0 {
-			sourceRanges = allowedSourceRanges
+		// spec.AllowedSourceRanges evaluates to the empty slice of zero length
+		// when omitted or set to 'null'/empty sequence in the PG manifest
+		if len(spec.AllowedSourceRanges) > 0 {
+			serviceSpec.LoadBalancerSourceRanges = spec.AllowedSourceRanges
+		} else {
+			// safe default value: lock a load balancer only to the local address unless overridden explicitly
+			serviceSpec.LoadBalancerSourceRanges = []string{localHost}
 		}
 
+		c.logger.Debugf("final load balancer source ranges as seen in a service spec (not necessarily applied): %q", serviceSpec.LoadBalancerSourceRanges)
 		serviceSpec.Type = v1.ServiceTypeLoadBalancer
-		serviceSpec.LoadBalancerSourceRanges = sourceRanges
 
 		annotations = map[string]string{
 			constants.ZalandoDNSNameAnnotation: dnsName,

pkg/cluster/k8sres_test.go

@@ -1,8 +1,11 @@
 package cluster
 
 import (
+	"k8s.io/api/core/v1"
+
 	acidv1 "github.com/zalando-incubator/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/config"
+	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil"
 	"testing"
 )
@@ -75,3 +78,54 @@ func TestCreateLoadBalancerLogic(t *testing.T) {
 		}
 	}
 }
+
+func TestShmVolume(t *testing.T) {
+	testName := "TestShmVolume"
+	tests := []struct {
+		subTest string
+		podSpec *v1.PodSpec
+		shmPos  int
+	}{
+		{
+			subTest: "empty PodSpec",
+			podSpec: &v1.PodSpec{
+				Volumes: []v1.Volume{},
+				Containers: []v1.Container{
+					v1.Container{
+						VolumeMounts: []v1.VolumeMount{},
+					},
+				},
+			},
+			shmPos: 0,
+		},
+		{
+			subTest: "non empty PodSpec",
+			podSpec: &v1.PodSpec{
+				Volumes: []v1.Volume{v1.Volume{}},
+				Containers: []v1.Container{
+					v1.Container{
+						VolumeMounts: []v1.VolumeMount{
+							v1.VolumeMount{},
+						},
+					},
+				},
+			},
+			shmPos: 1,
+		},
+	}
+	for _, tt := range tests {
+		addShmVolume(tt.podSpec)
+
+		volumeName := tt.podSpec.Volumes[tt.shmPos].Name
+		volumeMountName := tt.podSpec.Containers[0].VolumeMounts[tt.shmPos].Name
+
+		if volumeName != constants.ShmVolumeName {
+			t.Errorf("%s %s: Expected volume %s was not created, have %s instead",
+				testName, tt.subTest, constants.ShmVolumeName, volumeName)
+		}
+		if volumeMountName != constants.ShmVolumeName {
+			t.Errorf("%s %s: Expected mount %s was not created, have %s instead",
+				testName, tt.subTest, constants.ShmVolumeName, volumeMountName)
+		}
+	}
+}

pkg/cluster/pod.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"math/rand"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/zalando-incubator/postgres-operator/pkg/spec"
@@ -77,11 +77,7 @@ func (c *Cluster) deletePod(podName spec.NamespacedName) error {
 		return err
 	}
 
-	if err := c.waitForPodDeletion(ch); err != nil {
-		return err
-	}
-
-	return nil
+	return c.waitForPodDeletion(ch)
 }
 
 func (c *Cluster) unregisterPodSubscriber(podName spec.NamespacedName) {
@@ -122,7 +118,7 @@ func (c *Cluster) movePodFromEndOfLifeNode(pod *v1.Pod) (*v1.Pod, error) {
 	if eol, err = c.podIsEndOfLife(pod); err != nil {
 		return nil, fmt.Errorf("could not get node %q: %v", pod.Spec.NodeName, err)
 	} else if !eol {
-		c.logger.Infof("pod %q is already on a live node", podName)
+		c.logger.Infof("check failed: pod %q is already on a live node", podName)
 		return pod, nil
 	}
 
@@ -162,7 +158,7 @@ func (c *Cluster) masterCandidate(oldNodeName string) (*v1.Pod, error) {
 	}
 
 	if len(replicas) == 0 {
-		c.logger.Warningf("no available master candidates, migration will cause longer downtime of the master instance")
+		c.logger.Warningf("no available master candidates, migration will cause longer downtime of Postgres cluster")
 		return nil, nil
 	}
 
@@ -193,18 +189,18 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
 		return fmt.Errorf("could not get pod: %v", err)
 	}
 
-	c.logger.Infof("migrating master pod %q", podName)
+	c.logger.Infof("starting process to migrate master pod %q", podName)
 
 	if eol, err = c.podIsEndOfLife(oldMaster); err != nil {
 		return fmt.Errorf("could not get node %q: %v", oldMaster.Spec.NodeName, err)
 	}
 	if !eol {
-		c.logger.Debugf("pod is already on a live node")
+		c.logger.Debugf("no action needed: master pod is already on a live node")
 		return nil
 	}
 
 	if role := PostgresRole(oldMaster.Labels[c.OpConfig.PodRoleLabel]); role != Master {
-		c.logger.Warningf("pod %q is not a master", podName)
+		c.logger.Warningf("no action needed: pod %q is not the master (anymore)", podName)
 		return nil
 	}
 	// we must have a statefulset in the cluster for the migration to work
@@ -219,10 +215,10 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
 	// We may not have a cached statefulset if the initial cluster sync has aborted, revert to the spec in that case.
 	if *c.Statefulset.Spec.Replicas > 1 {
 		if masterCandidatePod, err = c.masterCandidate(oldMaster.Spec.NodeName); err != nil {
-			return fmt.Errorf("could not get new master candidate: %v", err)
+			return fmt.Errorf("could not find suitable replica pod as candidate for failover: %v", err)
 		}
 	} else {
-		c.logger.Warningf("single master pod for cluster %q, migration will cause longer downtime of the master instance", c.clusterName())
+		c.logger.Warningf("migrating single pod cluster %q, this will cause downtime of the Postgres cluster until pod is back", c.clusterName())
 	}
 
 	// there are two cases for each postgres cluster that has its master pod on the node to migrate from:
@@ -256,15 +252,15 @@ func (c *Cluster) MigrateReplicaPod(podName spec.NamespacedName, fromNodeName st
 		return fmt.Errorf("could not get pod: %v", err)
 	}
 
-	c.logger.Infof("migrating replica pod %q", podName)
+	c.logger.Infof("migrating replica pod %q to live node", podName)
 
 	if replicaPod.Spec.NodeName != fromNodeName {
-		c.logger.Infof("pod %q has already migrated to node %q", podName, replicaPod.Spec.NodeName)
+		c.logger.Infof("check failed: pod %q has already migrated to node %q", podName, replicaPod.Spec.NodeName)
 		return nil
 	}
 
 	if role := PostgresRole(replicaPod.Labels[c.OpConfig.PodRoleLabel]); role != Replica {
-		return fmt.Errorf("pod %q is not a replica", podName)
+		return fmt.Errorf("check failed: pod %q is not a replica", podName)
 	}
 
 	_, err = c.movePodFromEndOfLifeNode(replicaPod)
@@ -296,7 +292,7 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) {
 }
 
 func (c *Cluster) recreatePods() error {
-	c.setProcessName("recreating pods")
+	c.setProcessName("starting to recreate pods")
 	ls := c.labelsSet(false)
 	namespace := c.Namespace
 
@@ -337,10 +333,10 @@ func (c *Cluster) recreatePods() error {
 		// failover if we have not observed a master pod when re-creating former replicas.
 		if newMasterPod == nil && len(replicas) > 0 {
 			if err := c.Switchover(masterPod, masterCandidate(replicas)); err != nil {
-				c.logger.Warningf("could not perform failover: %v", err)
+				c.logger.Warningf("could not perform switch over: %v", err)
 			}
 		} else if newMasterPod == nil && len(replicas) == 0 {
-			c.logger.Warningf("cannot switch master role before re-creating the pod: no replicas")
+			c.logger.Warningf("cannot perform switch over before re-creating the pod: no replicas")
 		}
 		c.logger.Infof("recreating old master pod %q", util.NameFromMeta(masterPod.ObjectMeta))
 

pkg/cluster/resources.go

@@ -132,16 +132,17 @@ func (c *Cluster) preScaleDown(newStatefulSet *v1beta1.StatefulSet) error {
 	return nil
 }
 
-// setRollingUpdateFlagForStatefulSet sets the indicator or the rolling upgrade requirement
+// setRollingUpdateFlagForStatefulSet sets the indicator or the rolling update requirement
 // in the StatefulSet annotation.
 func (c *Cluster) setRollingUpdateFlagForStatefulSet(sset *v1beta1.StatefulSet, val bool) {
 	anno := sset.GetAnnotations()
-	c.logger.Debugf("rolling upgrade flag has been set to %t", val)
 	if anno == nil {
 		anno = make(map[string]string)
 	}
+
 	anno[rollingUpdateStatefulsetAnnotationKey] = strconv.FormatBool(val)
 	sset.SetAnnotations(anno)
+	c.logger.Debugf("statefulset's rolling update annotation has been set to %t", val)
 }
 
 // applyRollingUpdateFlagforStatefulSet sets the rolling update flag for the cluster's StatefulSet
@@ -176,9 +177,9 @@ func (c *Cluster) getRollingUpdateFlagFromStatefulSet(sset *v1beta1.StatefulSet,
 	return flag
 }
 
-// mergeRollingUpdateFlagUsingCache return the value of the rollingUpdate flag from the passed
+// mergeRollingUpdateFlagUsingCache returns the value of the rollingUpdate flag from the passed
 // statefulset, however, the value can be cleared if there is a cached flag in the cluster that
-// is set to false (the disrepancy could be a result of a failed StatefulSet update).s
+// is set to false (the discrepancy could be a result of a failed StatefulSet update)
 func (c *Cluster) mergeRollingUpdateFlagUsingCache(runningStatefulSet *v1beta1.StatefulSet) bool {
 	var (
 		cachedStatefulsetExists, clearRollingUpdateFromCache, podsRollingUpdateRequired bool
@@ -198,7 +199,7 @@ func (c *Cluster) mergeRollingUpdateFlagUsingCache(runningStatefulSet *v1beta1.S
 			c.logger.Infof("clearing the rolling update flag based on the cached information")
 			podsRollingUpdateRequired = false
 		} else {
-			c.logger.Infof("found a statefulset with an unfinished pods rolling update")
+			c.logger.Infof("found a statefulset with an unfinished rolling update of the pods")
 
 		}
 	}
@@ -436,7 +437,11 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error
 func (c *Cluster) deleteService(role PostgresRole) error {
 	c.logger.Debugf("deleting service %s", role)
 
-	service := c.Services[role]
+	service, ok := c.Services[role]
+	if !ok {
+		c.logger.Debugf("No service for %s role was found, nothing to delete", role)
+		return nil
+	}
 
 	if err := c.KubeClient.Services(service.Namespace).Delete(service.Name, c.deleteOptions); err != nil {
 		return err

pkg/cluster/sync.go

@@ -2,6 +2,7 @@ package cluster
 
 import (
 	"fmt"
+
 	"k8s.io/api/core/v1"
 	policybeta1 "k8s.io/api/policy/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -280,6 +281,7 @@ func (c *Cluster) syncStatefulSet() error {
 				podsRollingUpdateRequired = true
 				c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired)
 			}
+
 			c.logStatefulSetChanges(c.Statefulset, desiredSS, false, cmp.reasons)
 
 			if !cmp.replace {

pkg/cluster/types.go

@@ -1,12 +1,13 @@
 package cluster
 
 import (
+	"time"
+
 	acidv1 "github.com/zalando-incubator/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"k8s.io/api/apps/v1beta1"
 	"k8s.io/api/core/v1"
 	policybeta1 "k8s.io/api/policy/v1beta1"
 	"k8s.io/apimachinery/pkg/types"
-	"time"
 )
 
 // PostgresRole describes role of the node
@@ -20,6 +21,7 @@ const (
 	Replica PostgresRole = "replica"
 )
 
+// PodEventType represents the type of a pod-related event
 type PodEventType string
 
 // Possible values for the EventType

pkg/cluster/util.go

@@ -179,7 +179,7 @@ func (c *Cluster) logStatefulSetChanges(old, new *v1beta1.StatefulSet, isUpdate
 	if !reflect.DeepEqual(old.Annotations, new.Annotations) {
 		c.logger.Debugf("metadata.annotation diff\n%s\n", util.PrettyDiff(old.Annotations, new.Annotations))
 	}
-	c.logger.Debugf("spec diff\n%s\n", util.PrettyDiff(old.Spec, new.Spec))
+	c.logger.Debugf("spec diff between old and new statefulsets: \n%s\n", util.PrettyDiff(old.Spec, new.Spec))
 
 	if len(reasons) > 0 {
 		for _, reason := range reasons {
@@ -460,6 +460,7 @@ func (c *Cluster) setSpec(newSpec *acidv1.Postgresql) {
 	c.specMu.Unlock()
 }
 
+// GetSpec returns a copy of the operator-side spec of a Postgres cluster in a thread-safe manner
 func (c *Cluster) GetSpec() (*acidv1.Postgresql, error) {
 	c.specMu.RLock()
 	defer c.specMu.RUnlock()