Pod Rolling update

introduce Pod events channel;
add parsing of the MaintenanceWindows section;
skip deleting Etcd key on cluster delete;
use external etcd host;
watch for tpr/pods in the namespace of the operator pod only;

pkg/cluster/pod.go

@@ -0,0 +1,169 @@
+package cluster
+
+import (
+	"fmt"
+
+	"k8s.io/client-go/pkg/api/v1"
+
+	"github.bus.zalan.do/acid/postgres-operator/pkg/spec"
+	"github.bus.zalan.do/acid/postgres-operator/pkg/util"
+)
+
+func (c *Cluster) clusterPods() ([]v1.Pod, error) {
+	ns := c.Metadata.Namespace
+	listOptions := v1.ListOptions{
+		LabelSelector: c.labelsSet().String(),
+	}
+
+	pods, err := c.config.KubeClient.Pods(ns).List(listOptions)
+	if err != nil {
+		return nil, fmt.Errorf("Can't get list of pods: %s", err)
+	}
+
+	return pods.Items, nil
+}
+
+func (c *Cluster) deletePods() error {
+	pods, err := c.clusterPods()
+	if err != nil {
+		return err
+	}
+
+	for _, obj := range pods {
+		err := c.deletePod(&obj)
+		if err != nil {
+			c.logger.Errorf("Can't delete pod: %s", err)
+		} else {
+			c.logger.Infof("Pod '%s' has been deleted", util.NameFromMeta(obj.ObjectMeta))
+		}
+	}
+
+	return nil
+}
+
+func (c *Cluster) deletePod(pod *v1.Pod) error {
+	podName := spec.PodName{
+		Namespace: pod.Namespace,
+		Name:      pod.Name,
+	}
+
+	ch := make(chan spec.PodEvent)
+	if _, ok := c.podSubscribers[podName]; ok {
+		panic("Pod '" + podName.String() + "' is already subscribed")
+	}
+	c.podSubscribers[podName] = ch
+	defer func() {
+		close(ch)
+		delete(c.podSubscribers, podName)
+	}()
+
+	err := c.config.KubeClient.Pods(pod.Namespace).Delete(pod.Name, deleteOptions)
+	if err != nil {
+		return err
+	}
+
+	err = c.waitForPodDeletion(ch)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Cluster) recreatePod(pod v1.Pod, spiloRole string) error {
+	podName := spec.PodName{
+		Namespace: pod.Namespace,
+		Name:      pod.Name,
+	}
+
+	orphanDependents := false
+	deleteOptions := &v1.DeleteOptions{
+		OrphanDependents: &orphanDependents,
+	}
+
+	ch := make(chan spec.PodEvent)
+	if _, ok := c.podSubscribers[podName]; ok {
+		panic("Pod '" + podName.String() + "' is already subscribed")
+	}
+	c.podSubscribers[podName] = ch
+	defer func() {
+		close(ch)
+		delete(c.podSubscribers, podName)
+	}()
+
+	err := c.config.KubeClient.Pods(pod.Namespace).Delete(pod.Name, deleteOptions)
+	if err != nil {
+		return fmt.Errorf("Can't delete pod: %s", err)
+	}
+	err = c.waitForPodDeletion(ch)
+	if err != nil {
+		return err
+	}
+	err = c.waitForPodLabel(ch, spiloRole)
+	if err != nil {
+		return err
+	}
+	c.logger.Infof("Pod '%s' is ready", podName)
+
+	return nil
+}
+
+func (c *Cluster) podEventsDispatcher(stopCh <-chan struct{}) {
+	c.logger.Infof("Watching '%s' cluster", c.ClusterName())
+	for {
+		select {
+		case event := <-c.podEvents:
+			if subscriber, ok := c.podSubscribers[event.PodName]; ok {
+				c.logger.Debugf("New event for '%s' pod", event.PodName)
+				go func() { subscriber <- event }() //TODO: is it a right way to do nonblocking send to the channel?
+			} else {
+				c.logger.Debugf("Skipping event for an unwatched pod '%s'", event.PodName)
+			}
+		case <-stopCh:
+			return
+		}
+	}
+}
+
+func (c *Cluster) recreatePods() error {
+	ls := c.labelsSet()
+	namespace := c.Metadata.Namespace
+
+	listOptions := v1.ListOptions{
+		LabelSelector: ls.String(),
+	}
+	pods, err := c.config.KubeClient.Pods(namespace).List(listOptions)
+	if err != nil {
+		return fmt.Errorf("Can't get list of pods: %s", err)
+	} else {
+		c.logger.Infof("There are %d pods in the cluster to recreate", len(pods.Items))
+	}
+
+	var masterPod v1.Pod
+	for _, pod := range pods.Items {
+		role, ok := pod.Labels["spilo-role"]
+		if !ok {
+			continue
+		}
+
+		if role == "master" {
+			masterPod = pod
+			continue
+		}
+
+		err = c.recreatePod(pod, "replica")
+		if err != nil {
+			return fmt.Errorf("Can't recreate replica pod '%s': %s", util.NameFromMeta(pod.ObjectMeta), err)
+		}
+	}
+
+	//TODO: do manual failover
+	//TODO: specify master, leave new master empty
+	c.logger.Infof("Recreating master pod '%s'", util.NameFromMeta(masterPod.ObjectMeta))
+	err = c.recreatePod(masterPod, "replica")
+	if err != nil {
+		return fmt.Errorf("Can't recreate master pod '%s': %s", util.NameFromMeta(masterPod.ObjectMeta), err)
+	}
+
+	return nil
+}