From bb4fec25ae8a2b8fe5496bbf4e03257664b6ea92 Mon Sep 17 00:00:00 2001 From: Murat Kabilov Date: Mon, 10 Apr 2017 15:11:53 +0200 Subject: [PATCH] Fix deletion of the failed cluster; more debug messages --- pkg/cluster/cluster.go | 72 +++++++++++++++++++++++------------- pkg/cluster/pod.go | 3 +- pkg/cluster/resources.go | 10 ++--- pkg/controller/pod.go | 2 +- pkg/controller/postgresql.go | 6 +-- 5 files changed, 57 insertions(+), 36 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 1188b6276..05df7ca67 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -98,6 +98,7 @@ func (c *Cluster) Run(stopCh <-chan struct{}) { } func (c *Cluster) SetStatus(status spec.PostgresStatus) { + c.Status = status b, err := json.Marshal(status) if err != nil { c.logger.Fatalf("Can't marshal status: %s", err) @@ -136,7 +137,11 @@ func (c *Cluster) etcdKeyExists(keyName string) (bool, error) { options := etcdclient.GetOptions{} resp, err := c.EtcdClient.Get(context.Background(), keyName, &options) if err != nil { - if err.(etcdclient.Error).Code == etcdclient.ErrorCodeKeyNotFound { + etcdErr, ok := err.(etcdclient.Error) + if !ok { + return false, err + } + if etcdErr.Code == etcdclient.ErrorCodeKeyNotFound { return false, nil } } @@ -329,41 +334,56 @@ func (c *Cluster) Update(newSpec *spec.Postgresql) error { } func (c *Cluster) Delete() error { - epName := util.NameFromMeta(c.Endpoint.ObjectMeta) - if err := c.deleteEndpoint(); err != nil { - c.logger.Errorf("Can't delete Endpoint: %s", err) - } else { - c.logger.Infof("Endpoint '%s' has been deleted", epName) - } - - svcName := util.NameFromMeta(c.Service.ObjectMeta) - if err := c.deleteService(); err != nil { - c.logger.Errorf("Can't delete Service: %s", err) - } else { - c.logger.Infof("Service '%s' has been deleted", svcName) - } - - ssName := util.NameFromMeta(c.Statefulset.ObjectMeta) - if err := c.deleteStatefulSet(); err != nil { - c.logger.Errorf("Can't delete StatefulSet: %s", err) - } else { - c.logger.Infof("StatefulSet '%s' has been deleted", ssName) - } - - for _, obj := range c.Secrets { - if err := c.deleteSecret(obj); err != nil { - c.logger.Errorf("Can't delete Secret: %s", err) + if c.Endpoint != nil { + c.logger.Debugln("Deleting Endpoints") + epName := util.NameFromMeta(c.Endpoint.ObjectMeta) + if err := c.deleteEndpoint(); err != nil { + c.logger.Errorf("Can't delete Endpoint: %s", err) } else { - c.logger.Infof("Secret '%s' has been deleted", util.NameFromMeta(obj.ObjectMeta)) + c.logger.Infof("Endpoint '%s' has been deleted", epName) } } + if c.Service != nil { + c.logger.Debugln("Deleting Service") + svcName := util.NameFromMeta(c.Service.ObjectMeta) + if err := c.deleteService(); err != nil { + c.logger.Errorf("Can't delete Service: %s", err) + } else { + c.logger.Infof("Service '%s' has been deleted", svcName) + } + } + + if c.Statefulset != nil { + c.logger.Debugln("Deleting StatefulSet") + ssName := util.NameFromMeta(c.Statefulset.ObjectMeta) + if err := c.deleteStatefulSet(); err != nil { + c.logger.Errorf("Can't delete StatefulSet: %s", err) + } else { + c.logger.Infof("StatefulSet '%s' has been deleted", ssName) + } + } + + if c.Secrets != nil { + c.logger.Debugln("Deleting Secrets") + for _, obj := range c.Secrets { + c.logger.Debugf("Deleting Secret '%s'", util.NameFromMeta(obj.ObjectMeta)) + if err := c.deleteSecret(obj); err != nil { + c.logger.Errorf("Can't delete Secret: %s", err) + } else { + c.logger.Infof("Secret '%s' has been deleted", util.NameFromMeta(obj.ObjectMeta)) + } + } + } + + c.logger.Debugln("Deleting Pods") if err := c.deletePods(); err != nil { c.logger.Errorf("Can't delete Pods: %s", err) } else { c.logger.Infof("Pods have been deleted") } + c.logger.Debugln("Deleting PVCs") if err := c.deletePersistenVolumeClaims(); err != nil { return fmt.Errorf("Can't delete PersistentVolumeClaims: %s", err) } diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index 9716e2a1c..09164f31f 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -44,6 +44,7 @@ func (c *Cluster) deletePods() error { } for _, obj := range pods { + c.logger.Debugf("Deleting Pod '%s'", util.NameFromMeta(obj.ObjectMeta)) if err := c.deletePod(&obj); err != nil { c.logger.Errorf("Can't delete Pod: %s", err) } else { @@ -61,6 +62,7 @@ func (c *Cluster) deletePersistenVolumeClaims() error { return err } for _, pvc := range pvcs { + c.logger.Debugf("Deleting PVC '%s'", util.NameFromMeta(pvc.ObjectMeta)) if err := c.KubeClient.PersistentVolumeClaims(ns).Delete(pvc.Name, deleteOptions); err != nil { c.logger.Warningf("Can't delete PersistentVolumeClaim: %s", err) } @@ -148,7 +150,6 @@ func (c *Cluster) podEventsDispatcher(stopCh <-chan struct{}) { select { case event := <-c.podEvents: if subscriber, ok := c.podSubscribers[event.PodName]; ok { - c.logger.Debugf("New event for '%s' Pod", event.PodName) go func() { subscriber <- event }() //TODO: is it a right way to do nonblocking send to the channel? } else { c.logger.Debugf("Skipping event for an unwatched Pod '%s'", event.PodName) diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 5132cbc79..bc99a1d1b 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -117,7 +117,7 @@ func (c *Cluster) createStatefulSet() (*v1beta1.StatefulSet, error) { return nil, err } c.Statefulset = statefulSet - c.logger.Debugf("Created new StatefulSet, uid: %s", statefulSet.UID) + c.logger.Debugf("Created new StatefulSet '%s', uid: %s", util.NameFromMeta(statefulSet.ObjectMeta), statefulSet.UID) return statefulSet, nil } @@ -244,13 +244,13 @@ func (c *Cluster) applySecrets() error { for secretUsername, secretSpec := range secrets { secret, err := c.KubeClient.Secrets(secretSpec.Namespace).Create(secretSpec) if k8sutil.ResourceAlreadyExists(err) { - curSecrets, err := c.KubeClient.Secrets(secretSpec.Namespace).Get(secretSpec.Name) + curSecret, err := c.KubeClient.Secrets(secretSpec.Namespace).Get(secretSpec.Name) if err != nil { return fmt.Errorf("Can't get current Secret: %s", err) } - c.logger.Debugf("Secret '%s' already exists, fetching it's password", util.NameFromMeta(secret.ObjectMeta)) + c.logger.Debugf("Secret '%s' already exists, fetching it's password", util.NameFromMeta(curSecret.ObjectMeta)) pwdUser := c.pgUsers[secretUsername] - pwdUser.Password = string(curSecrets.Data["password"]) + pwdUser.Password = string(curSecret.Data["password"]) c.pgUsers[secretUsername] = pwdUser continue @@ -259,7 +259,7 @@ func (c *Cluster) applySecrets() error { return fmt.Errorf("Can't create Secret for user '%s': %s", secretUsername, err) } c.Secrets[secret.UID] = secret - c.logger.Debugf("Created new Secret, uid: %s", secret.UID) + c.logger.Debugf("Created new Secret '%s', uid: %s", util.NameFromMeta(secret.ObjectMeta), secret.UID) } } diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index db252fee6..6abea2afe 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -125,7 +125,7 @@ func (c *Controller) podEventsDispatcher(stopCh <-chan struct{}) { c.logger.Debugf("Sending %s event of Pod '%s' to the '%s' cluster channel", event.EventType, event.PodName, event.ClusterName) go subscriber.ReceivePodEvent(event) } else { - c.logger.Debugf("Skipping Pods unrelated to clusters: %s", event.PodName) + c.logger.Debugf("Skipping %s event of the '%s' pod", event.EventType, event.PodName) } case <-stopCh: return diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index 1e06f7f3b..0a9f7d47e 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -98,6 +98,8 @@ func (c *Controller) postgresqlAdd(obj interface{}) { cl := cluster.New(c.makeClusterConfig(), *pg, c.logger.Logger) c.clusters[clusterName] = cl + stopCh := make(chan struct{}) + c.stopChMap[clusterName] = stopCh cl.SetStatus(spec.ClusterStatusCreating) if err := cl.Create(); err != nil { @@ -106,8 +108,6 @@ func (c *Controller) postgresqlAdd(obj interface{}) { return } cl.SetStatus(spec.ClusterStatusRunning) //TODO: are you sure it's running? - stopCh := make(chan struct{}) - c.stopChMap[clusterName] = stopCh go cl.Run(stopCh) c.logger.Infof("Postgresql cluster '%s' has been created", clusterName) @@ -165,7 +165,7 @@ func (c *Controller) postgresqlDelete(obj interface{}) { return } - c.logger.Infof("Cluster delete: %s", util.NameFromMeta(pgCur.Metadata)) + c.logger.Infof("Starting deletion of the '%s' cluster", util.NameFromMeta(pgCur.Metadata)) if err := pgCluster.Delete(); err != nil { c.logger.Errorf("Can't delete cluster '%s': %s", clusterName, err) return