Fix deletion of the failed cluster; more debug messages

pkg/cluster/cluster.go

@@ -98,6 +98,7 @@ func (c *Cluster) Run(stopCh <-chan struct{}) {
 }
 
 func (c *Cluster) SetStatus(status spec.PostgresStatus) {
+	c.Status = status
 	b, err := json.Marshal(status)
 	if err != nil {
 		c.logger.Fatalf("Can't marshal status: %s", err)
@@ -136,7 +137,11 @@ func (c *Cluster) etcdKeyExists(keyName string) (bool, error) {
 	options := etcdclient.GetOptions{}
 	resp, err := c.EtcdClient.Get(context.Background(), keyName, &options)
 	if err != nil {
-		if err.(etcdclient.Error).Code == etcdclient.ErrorCodeKeyNotFound {
+		etcdErr, ok := err.(etcdclient.Error)
+		if !ok {
+			return false, err
+		}
+		if etcdErr.Code == etcdclient.ErrorCodeKeyNotFound {
 			return false, nil
 		}
 	}
@@ -329,41 +334,56 @@ func (c *Cluster) Update(newSpec *spec.Postgresql) error {
 }
 
 func (c *Cluster) Delete() error {
-	epName := util.NameFromMeta(c.Endpoint.ObjectMeta)
+	if c.Endpoint != nil {
-	if err := c.deleteEndpoint(); err != nil {
+		c.logger.Debugln("Deleting Endpoints")
-		c.logger.Errorf("Can't delete Endpoint: %s", err)
+		epName := util.NameFromMeta(c.Endpoint.ObjectMeta)
-	} else {
+		if err := c.deleteEndpoint(); err != nil {
-		c.logger.Infof("Endpoint '%s' has been deleted", epName)
+			c.logger.Errorf("Can't delete Endpoint: %s", err)
-	}
-
-	svcName := util.NameFromMeta(c.Service.ObjectMeta)
-	if err := c.deleteService(); err != nil {
-		c.logger.Errorf("Can't delete Service: %s", err)
-	} else {
-		c.logger.Infof("Service '%s' has been deleted", svcName)
-	}
-
-	ssName := util.NameFromMeta(c.Statefulset.ObjectMeta)
-	if err := c.deleteStatefulSet(); err != nil {
-		c.logger.Errorf("Can't delete StatefulSet: %s", err)
-	} else {
-		c.logger.Infof("StatefulSet '%s' has been deleted", ssName)
-	}
-
-	for _, obj := range c.Secrets {
-		if err := c.deleteSecret(obj); err != nil {
-			c.logger.Errorf("Can't delete Secret: %s", err)
 		} else {
-			c.logger.Infof("Secret '%s' has been deleted", util.NameFromMeta(obj.ObjectMeta))
+			c.logger.Infof("Endpoint '%s' has been deleted", epName)
 		}
 	}
 
+	if c.Service != nil {
+		c.logger.Debugln("Deleting Service")
+		svcName := util.NameFromMeta(c.Service.ObjectMeta)
+		if err := c.deleteService(); err != nil {
+			c.logger.Errorf("Can't delete Service: %s", err)
+		} else {
+			c.logger.Infof("Service '%s' has been deleted", svcName)
+		}
+	}
+
+	if c.Statefulset != nil {
+		c.logger.Debugln("Deleting StatefulSet")
+		ssName := util.NameFromMeta(c.Statefulset.ObjectMeta)
+		if err := c.deleteStatefulSet(); err != nil {
+			c.logger.Errorf("Can't delete StatefulSet: %s", err)
+		} else {
+			c.logger.Infof("StatefulSet '%s' has been deleted", ssName)
+		}
+	}
+
+	if c.Secrets != nil {
+		c.logger.Debugln("Deleting Secrets")
+		for _, obj := range c.Secrets {
+			c.logger.Debugf("Deleting Secret '%s'", util.NameFromMeta(obj.ObjectMeta))
+			if err := c.deleteSecret(obj); err != nil {
+				c.logger.Errorf("Can't delete Secret: %s", err)
+			} else {
+				c.logger.Infof("Secret '%s' has been deleted", util.NameFromMeta(obj.ObjectMeta))
+			}
+		}
+	}
+
+	c.logger.Debugln("Deleting Pods")
 	if err := c.deletePods(); err != nil {
 		c.logger.Errorf("Can't delete Pods: %s", err)
 	} else {
 		c.logger.Infof("Pods have been deleted")
 	}
 
+	c.logger.Debugln("Deleting PVCs")
 	if err := c.deletePersistenVolumeClaims(); err != nil {
 		return fmt.Errorf("Can't delete PersistentVolumeClaims: %s", err)
 	}

pkg/cluster/pod.go

@@ -44,6 +44,7 @@ func (c *Cluster) deletePods() error {
 	}
 
 	for _, obj := range pods {
+		c.logger.Debugf("Deleting Pod '%s'", util.NameFromMeta(obj.ObjectMeta))
 		if err := c.deletePod(&obj); err != nil {
 			c.logger.Errorf("Can't delete Pod: %s", err)
 		} else {
@@ -61,6 +62,7 @@ func (c *Cluster) deletePersistenVolumeClaims() error {
 		return err
 	}
 	for _, pvc := range pvcs {
+		c.logger.Debugf("Deleting PVC '%s'", util.NameFromMeta(pvc.ObjectMeta))
 		if err := c.KubeClient.PersistentVolumeClaims(ns).Delete(pvc.Name, deleteOptions); err != nil {
 			c.logger.Warningf("Can't delete PersistentVolumeClaim: %s", err)
 		}
@@ -148,7 +150,6 @@ func (c *Cluster) podEventsDispatcher(stopCh <-chan struct{}) {
 		select {
 		case event := <-c.podEvents:
 			if subscriber, ok := c.podSubscribers[event.PodName]; ok {
-				c.logger.Debugf("New event for '%s' Pod", event.PodName)
 				go func() { subscriber <- event }() //TODO: is it a right way to do nonblocking send to the channel?
 			} else {
 				c.logger.Debugf("Skipping event for an unwatched Pod '%s'", event.PodName)

pkg/cluster/resources.go

@@ -117,7 +117,7 @@ func (c *Cluster) createStatefulSet() (*v1beta1.StatefulSet, error) {
 		return nil, err
 	}
 	c.Statefulset = statefulSet
-	c.logger.Debugf("Created new StatefulSet, uid: %s", statefulSet.UID)
+	c.logger.Debugf("Created new StatefulSet '%s', uid: %s", util.NameFromMeta(statefulSet.ObjectMeta), statefulSet.UID)
 
 	return statefulSet, nil
 }
@@ -244,13 +244,13 @@ func (c *Cluster) applySecrets() error {
 	for secretUsername, secretSpec := range secrets {
 		secret, err := c.KubeClient.Secrets(secretSpec.Namespace).Create(secretSpec)
 		if k8sutil.ResourceAlreadyExists(err) {
-			curSecrets, err := c.KubeClient.Secrets(secretSpec.Namespace).Get(secretSpec.Name)
+			curSecret, err := c.KubeClient.Secrets(secretSpec.Namespace).Get(secretSpec.Name)
 			if err != nil {
 				return fmt.Errorf("Can't get current Secret: %s", err)
 			}
-			c.logger.Debugf("Secret '%s' already exists, fetching it's password", util.NameFromMeta(secret.ObjectMeta))
+			c.logger.Debugf("Secret '%s' already exists, fetching it's password", util.NameFromMeta(curSecret.ObjectMeta))
 			pwdUser := c.pgUsers[secretUsername]
-			pwdUser.Password = string(curSecrets.Data["password"])
+			pwdUser.Password = string(curSecret.Data["password"])
 			c.pgUsers[secretUsername] = pwdUser
 
 			continue
@@ -259,7 +259,7 @@ func (c *Cluster) applySecrets() error {
 				return fmt.Errorf("Can't create Secret for user '%s': %s", secretUsername, err)
 			}
 			c.Secrets[secret.UID] = secret
-			c.logger.Debugf("Created new Secret, uid: %s", secret.UID)
+			c.logger.Debugf("Created new Secret '%s', uid: %s", util.NameFromMeta(secret.ObjectMeta), secret.UID)
 		}
 	}
 

pkg/controller/pod.go

@@ -125,7 +125,7 @@ func (c *Controller) podEventsDispatcher(stopCh <-chan struct{}) {
 				c.logger.Debugf("Sending %s event of Pod '%s' to the '%s' cluster channel", event.EventType, event.PodName, event.ClusterName)
 				go subscriber.ReceivePodEvent(event)
 			} else {
-				c.logger.Debugf("Skipping Pods unrelated to clusters: %s", event.PodName)
+				c.logger.Debugf("Skipping %s event of the '%s' pod", event.EventType, event.PodName)
 			}
 		case <-stopCh:
 			return

pkg/controller/postgresql.go

@@ -98,6 +98,8 @@ func (c *Controller) postgresqlAdd(obj interface{}) {
 	cl := cluster.New(c.makeClusterConfig(), *pg, c.logger.Logger)
 
 	c.clusters[clusterName] = cl
+	stopCh := make(chan struct{})
+	c.stopChMap[clusterName] = stopCh
 
 	cl.SetStatus(spec.ClusterStatusCreating)
 	if err := cl.Create(); err != nil {
@@ -106,8 +108,6 @@ func (c *Controller) postgresqlAdd(obj interface{}) {
 		return
 	}
 	cl.SetStatus(spec.ClusterStatusRunning) //TODO: are you sure it's running?
-	stopCh := make(chan struct{})
-	c.stopChMap[clusterName] = stopCh
 	go cl.Run(stopCh)
 
 	c.logger.Infof("Postgresql cluster '%s' has been created", clusterName)
@@ -165,7 +165,7 @@ func (c *Controller) postgresqlDelete(obj interface{}) {
 		return
 	}
 
-	c.logger.Infof("Cluster delete: %s", util.NameFromMeta(pgCur.Metadata))
+	c.logger.Infof("Starting deletion of the '%s' cluster", util.NameFromMeta(pgCur.Metadata))
 	if err := pgCluster.Delete(); err != nil {
 		c.logger.Errorf("Can't delete cluster '%s': %s", clusterName, err)
 		return