From da0de8cff7ef2a57bc2d906947e21fbd3c2a5be2 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Thu, 21 Dec 2017 15:20:43 +0100
Subject: [PATCH] Make sure the statefulset that is deleted manually gets
 re-created. (#191)

* Make sure the statefulset that is deleted manually gets re-created.

Per report and analysis by Manuel Gomez.

* Move the existence checks for other objects out of the Create functions.

create{Object} for services, endpoints and PDBs refused to continue if
there is a cached definition in the cluster, however, the only place
where it makes sense is when creating a new cluster. Note that contrary
to the statefulset this doesn't fix any issues, since those definitions
were nullified correspondingly when the sync code detected there is no
object present in the Kubernetes cluster.
---
 pkg/cluster/cluster.go   | 12 ++++++++++++
 pkg/cluster/resources.go | 13 -------------
 pkg/cluster/sync.go      |  1 +
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index 1275c4cb0..4f131e4f7 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -216,12 +216,18 @@ func (c *Cluster) Create() error {
 		if role == Replica && !c.Spec.ReplicaLoadBalancer {
 			continue
 		}
+		if c.Endpoints[role] != nil {
+			return fmt.Errorf("%s endpoint already exists in the cluster", role)
+		}
 		ep, err = c.createEndpoint(role)
 		if err != nil {
 			return fmt.Errorf("could not create %s endpoint: %v", role, err)
 		}
 		c.logger.Infof("endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta))
 
+		if c.Services[role] != nil {
+			return fmt.Errorf("service already exists in the cluster")
+		}
 		service, err = c.createService(role)
 		if err != nil {
 			return fmt.Errorf("could not create %s service: %v", role, err)
@@ -239,12 +245,18 @@ func (c *Cluster) Create() error {
 	}
 	c.logger.Infof("secrets have been successfully created")
 
+	if c.PodDisruptionBudget != nil {
+		return fmt.Errorf("pod disruption budget already exists in the cluster")
+	}
 	pdb, err := c.createPodDisruptionBudget()
 	if err != nil {
 		return fmt.Errorf("could not create pod disruption budget: %v", err)
 	}
 	c.logger.Infof("pod disruption budget %q has been successfully created", util.NameFromMeta(pdb.ObjectMeta))
 
+	if c.Statefulset != nil {
+		return fmt.Errorf("statefulset already exists in the cluster")
+	}
 	ss, err = c.createStatefulSet()
 	if err != nil {
 		return fmt.Errorf("could not create statefulset: %v", err)
diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go
index 03776b940..1928143b1 100644
--- a/pkg/cluster/resources.go
+++ b/pkg/cluster/resources.go
@@ -61,9 +61,6 @@ func (c *Cluster) listResources() error {
 
 func (c *Cluster) createStatefulSet() (*v1beta1.StatefulSet, error) {
 	c.setProcessName("creating statefulset")
-	if c.Statefulset != nil {
-		return nil, fmt.Errorf("statefulset already exists in the cluster")
-	}
 	statefulSetSpec, err := c.generateStatefulSet(&c.Spec)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate statefulset: %v", err)
@@ -236,11 +233,7 @@ func (c *Cluster) deleteStatefulSet() error {
 func (c *Cluster) createService(role PostgresRole) (*v1.Service, error) {
 	c.setProcessName("creating %v service", role)
 
-	if c.Services[role] != nil {
-		return nil, fmt.Errorf("service already exists in the cluster")
-	}
 	serviceSpec := c.generateService(role, &c.Spec)
-
 	service, err := c.KubeClient.Services(serviceSpec.Namespace).Create(serviceSpec)
 	if err != nil {
 		return nil, err
@@ -349,9 +342,6 @@ func (c *Cluster) deleteService(role PostgresRole) error {
 
 func (c *Cluster) createEndpoint(role PostgresRole) (*v1.Endpoints, error) {
 	c.setProcessName("creating endpoint")
-	if c.Endpoints[role] != nil {
-		return nil, fmt.Errorf("%s endpoint already exists in the cluster", role)
-	}
 	subsets := make([]v1.EndpointSubset, 0)
 	if role == Master {
 		//TODO: set subsets to the master
@@ -369,9 +359,6 @@ func (c *Cluster) createEndpoint(role PostgresRole) (*v1.Endpoints, error) {
 }
 
 func (c *Cluster) createPodDisruptionBudget() (*policybeta1.PodDisruptionBudget, error) {
-	if c.PodDisruptionBudget != nil {
-		return nil, fmt.Errorf("pod disruption budget already exists in the cluster")
-	}
 	podDisruptionBudgetSpec := c.generatePodDisruptionBudget()
 	podDisruptionBudget, err := c.KubeClient.
 		PodDisruptionBudgets(podDisruptionBudgetSpec.Namespace).
diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go
index 9b8baa97b..714809391 100644
--- a/pkg/cluster/sync.go
+++ b/pkg/cluster/sync.go
@@ -245,6 +245,7 @@ func (c *Cluster) syncStatefulSet() error {
 			return fmt.Errorf("could not get statefulset: %v", err)
 		}
 		// statefulset does not exist, try to re-create it
+		c.Statefulset = nil
 		c.logger.Infof("could not find the cluster's statefulset")
 		pods, err := c.listPods()
 		if err != nil {