Sync clusters states

* move statefulset creation from cluster spec to the separate function
* sync cluster state with desired state;
* move out from arrays for cluster resources;
* recreate pods instead of deleting them in case of statefulset change
* check for master while creating cluster/updating pods
* simplify retryutil
* list pvc while listing resources
* name kubernetes resources with capital letter
* do rolling update in case of env variables change

pkg/cluster/util.go

@@ -9,9 +9,11 @@ import (
 
 	etcdclient "github.com/coreos/etcd/client"
 	"k8s.io/client-go/pkg/api/v1"
+	"k8s.io/client-go/pkg/apis/apps/v1beta1"
 	"k8s.io/client-go/pkg/labels"
 
 	"github.bus.zalan.do/acid/postgres-operator/pkg/spec"
+	"github.bus.zalan.do/acid/postgres-operator/pkg/util"
 	"github.bus.zalan.do/acid/postgres-operator/pkg/util/constants"
 	"github.bus.zalan.do/acid/postgres-operator/pkg/util/retryutil"
 )
@@ -43,6 +45,83 @@ func normalizeUserFlags(userFlags []string) (flags []string, err error) {
 	return
 }
 
+func statefulsetsEqual(ss1, ss2 *v1beta1.StatefulSet) (equal bool, needsRollUpdate bool) {
+	equal = true
+	needsRollUpdate = false
+	//TODO: improve me
+	if *ss1.Spec.Replicas != *ss2.Spec.Replicas {
+		equal = false
+	}
+	if len(ss1.Spec.Template.Spec.Containers) != len(ss1.Spec.Template.Spec.Containers) {
+		equal = false
+		needsRollUpdate = true
+		return
+	}
+	if len(ss1.Spec.Template.Spec.Containers) == 0 {
+		return
+	}
+
+	container1 := ss1.Spec.Template.Spec.Containers[0]
+	container2 := ss2.Spec.Template.Spec.Containers[0]
+	if container1.Image != container2.Image {
+		equal = false
+		needsRollUpdate = true
+		return
+	}
+
+	if !reflect.DeepEqual(container1.Ports, container2.Ports) {
+		equal = false
+		needsRollUpdate = true
+		return
+	}
+
+	if !reflect.DeepEqual(container1.Resources, container2.Resources) {
+		equal = false
+		needsRollUpdate = true
+		return
+	}
+	if !reflect.DeepEqual(container1.Env, container2.Env) {
+		equal = false
+		needsRollUpdate = true
+	}
+
+	return
+}
+
+func servicesEqual(svc1, svc2 *v1.Service) bool {
+	//TODO: check of Ports
+	//TODO: improve me
+	if reflect.DeepEqual(svc1.Spec.LoadBalancerSourceRanges, svc2.Spec.LoadBalancerSourceRanges) {
+		return true
+	}
+
+	return false
+}
+
+func podMatchesTemplate(pod *v1.Pod, ss *v1beta1.StatefulSet) bool {
+	//TODO: improve me
+	if len(pod.Spec.Containers) != 1 {
+		return false
+	}
+	container := pod.Spec.Containers[0]
+	ssContainer := ss.Spec.Template.Spec.Containers[0]
+
+	if container.Image != ssContainer.Image {
+		return false
+	}
+	if !reflect.DeepEqual(container.Env, ssContainer.Env) {
+		return false
+	}
+	if !reflect.DeepEqual(container.Ports, ssContainer.Ports) {
+		return false
+	}
+	if !reflect.DeepEqual(container.Resources, ssContainer.Resources) {
+		return false
+	}
+
+	return true
+}
+
 func (c *Cluster) getTeamMembers() ([]string, error) {
 	teamInfo, err := c.config.TeamsAPIClient.TeamInfo(c.Spec.TeamId)
 	if err != nil {
@@ -56,10 +135,8 @@ func (c *Cluster) waitForPodLabel(podEvents chan spec.PodEvent, spiloRole string
 	for {
 		select {
 		case podEvent := <-podEvents:
-			podLabels := podEvent.CurPod.Labels
-			c.logger.Debugf("Pod has following labels: %+v", podLabels)
-			val, ok := podLabels["spilo-role"]
-			if ok && val == spiloRole {
+			role := util.PodSpiloRole(podEvent.CurPod)
+			if role == spiloRole {
 				return nil
 			}
 		case <-time.After(constants.PodLabelWaitTimeout):
@@ -82,7 +159,7 @@ func (c *Cluster) waitForPodDeletion(podEvents chan spec.PodEvent) error {
 }
 
 func (c *Cluster) waitStatefulsetReady() error {
-	return retryutil.Retry(constants.ResourceCheckInterval, int(constants.ResourceCheckTimeout/constants.ResourceCheckInterval),
+	return retryutil.Retry(constants.ResourceCheckInterval, constants.ResourceCheckTimeout,
 		func() (bool, error) {
 			listOptions := v1.ListOptions{
 				LabelSelector: c.labelsSet().String(),
@@ -119,8 +196,7 @@ func (c *Cluster) waitPodLabelsReady() error {
 	}
 	podsNumber := len(pods.Items)
 
-	return retryutil.Retry(
-		constants.ResourceCheckInterval, int(constants.ResourceCheckTimeout/constants.ResourceCheckInterval),
+	return retryutil.Retry(constants.ResourceCheckInterval, constants.ResourceCheckTimeout,
 		func() (bool, error) {
 			masterPods, err := c.config.KubeClient.Pods(namespace).List(masterListOption)
 			if err != nil {
@@ -133,6 +209,9 @@ func (c *Cluster) waitPodLabelsReady() error {
 			if len(masterPods.Items) > 1 {
 				return false, fmt.Errorf("Too many masters")
 			}
+			if len(replicaPods.Items) == podsNumber {
+				return false, fmt.Errorf("Cluster has no master")
+			}
 
 			return len(masterPods.Items)+len(replicaPods.Items) == podsNumber, nil
 		})
@@ -185,12 +264,3 @@ func (c *Cluster) deleteEtcdKey() error {
 
 	return nil
 }
-
-func servicesEqual(svc1, svc2 *v1.Service) bool {
-	//TODO: improve me
-	if reflect.DeepEqual(svc1.Spec.LoadBalancerSourceRanges, svc2.Spec.LoadBalancerSourceRanges) {
-		return true
-	}
-
-	return false
-}