Make sure the statefulset that is deleted manually gets re-created. (#191)

* Make sure the statefulset that is deleted manually gets re-created.

Per report and analysis by Manuel Gomez.

* Move the existence checks for other objects out of the Create functions.

create{Object} for services, endpoints and PDBs refused to continue if
there is a cached definition in the cluster, however, the only place
where it makes sense is when creating a new cluster. Note that contrary
to the statefulset this doesn't fix any issues, since those definitions
were nullified correspondingly when the sync code detected there is no
object present in the Kubernetes cluster.

pkg/cluster/cluster.go

@@ -216,12 +216,18 @@ func (c *Cluster) Create() error {
 		if role == Replica && !c.Spec.ReplicaLoadBalancer {
 			continue
 		}
+		if c.Endpoints[role] != nil {
+			return fmt.Errorf("%s endpoint already exists in the cluster", role)
+		}
 		ep, err = c.createEndpoint(role)
 		if err != nil {
 			return fmt.Errorf("could not create %s endpoint: %v", role, err)
 		}
 		c.logger.Infof("endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta))
 
+		if c.Services[role] != nil {
+			return fmt.Errorf("service already exists in the cluster")
+		}
 		service, err = c.createService(role)
 		if err != nil {
 			return fmt.Errorf("could not create %s service: %v", role, err)
@@ -239,12 +245,18 @@ func (c *Cluster) Create() error {
 	}
 	c.logger.Infof("secrets have been successfully created")
 
+	if c.PodDisruptionBudget != nil {
+		return fmt.Errorf("pod disruption budget already exists in the cluster")
+	}
 	pdb, err := c.createPodDisruptionBudget()
 	if err != nil {
 		return fmt.Errorf("could not create pod disruption budget: %v", err)
 	}
 	c.logger.Infof("pod disruption budget %q has been successfully created", util.NameFromMeta(pdb.ObjectMeta))
 
+	if c.Statefulset != nil {
+		return fmt.Errorf("statefulset already exists in the cluster")
+	}
 	ss, err = c.createStatefulSet()
 	if err != nil {
 		return fmt.Errorf("could not create statefulset: %v", err)

pkg/cluster/resources.go

@@ -61,9 +61,6 @@ func (c *Cluster) listResources() error {
 
 func (c *Cluster) createStatefulSet() (*v1beta1.StatefulSet, error) {
 	c.setProcessName("creating statefulset")
-	if c.Statefulset != nil {
-		return nil, fmt.Errorf("statefulset already exists in the cluster")
-	}
 	statefulSetSpec, err := c.generateStatefulSet(&c.Spec)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate statefulset: %v", err)
@@ -236,11 +233,7 @@ func (c *Cluster) deleteStatefulSet() error {
 func (c *Cluster) createService(role PostgresRole) (*v1.Service, error) {
 	c.setProcessName("creating %v service", role)
 
-	if c.Services[role] != nil {
-		return nil, fmt.Errorf("service already exists in the cluster")
-	}
 	serviceSpec := c.generateService(role, &c.Spec)
-
 	service, err := c.KubeClient.Services(serviceSpec.Namespace).Create(serviceSpec)
 	if err != nil {
 		return nil, err
@@ -349,9 +342,6 @@ func (c *Cluster) deleteService(role PostgresRole) error {
 
 func (c *Cluster) createEndpoint(role PostgresRole) (*v1.Endpoints, error) {
 	c.setProcessName("creating endpoint")
-	if c.Endpoints[role] != nil {
-		return nil, fmt.Errorf("%s endpoint already exists in the cluster", role)
-	}
 	subsets := make([]v1.EndpointSubset, 0)
 	if role == Master {
 		//TODO: set subsets to the master
@@ -369,9 +359,6 @@ func (c *Cluster) createEndpoint(role PostgresRole) (*v1.Endpoints, error) {
 }
 
 func (c *Cluster) createPodDisruptionBudget() (*policybeta1.PodDisruptionBudget, error) {
-	if c.PodDisruptionBudget != nil {
-		return nil, fmt.Errorf("pod disruption budget already exists in the cluster")
-	}
 	podDisruptionBudgetSpec := c.generatePodDisruptionBudget()
 	podDisruptionBudget, err := c.KubeClient.
 		PodDisruptionBudgets(podDisruptionBudgetSpec.Namespace).

pkg/cluster/sync.go

@@ -245,6 +245,7 @@ func (c *Cluster) syncStatefulSet() error {
 			return fmt.Errorf("could not get statefulset: %v", err)
 		}
 		// statefulset does not exist, try to re-create it
+		c.Statefulset = nil
 		c.logger.Infof("could not find the cluster's statefulset")
 		pods, err := c.listPods()
 		if err != nil {