use sync methods while updating the cluster

pkg/cluster/sync.go

@@ -4,10 +4,12 @@ import (
 	"fmt"
 	"reflect"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	policybeta1 "k8s.io/client-go/pkg/apis/policy/v1beta1"
 
 	"github.com/zalando-incubator/postgres-operator/pkg/spec"
 	"github.com/zalando-incubator/postgres-operator/pkg/util"
+	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/volumes"
 )
@@ -20,11 +22,6 @@ func (c *Cluster) Sync(newSpec *spec.Postgresql) (err error) {
 
 	c.Postgresql = *newSpec
 
-	err = c.loadResources()
-	if err != nil {
-		c.logger.Errorf("could not load resources: %v", err)
-	}
-
 	defer func() {
 		if err != nil {
 			c.setStatus(spec.ClusterStatusSyncFailed)
@@ -41,39 +38,15 @@ func (c *Cluster) Sync(newSpec *spec.Postgresql) (err error) {
 	c.logger.Debugf("syncing secrets")
 
 	//TODO: mind the secrets of the deleted/new users
-	if err = c.applySecrets(); err != nil {
-		if !k8sutil.ResourceAlreadyExists(err) {
-			err = fmt.Errorf("could not sync secrets: %v", err)
-			return
-		}
-	}
-
-	c.logger.Debugf("syncing endpoints")
-	if err = c.syncEndpoint(); err != nil {
-		if !k8sutil.ResourceAlreadyExists(err) {
-			err = fmt.Errorf("could not sync endpoints: %v", err)
-			return
-		}
+	if err = c.syncSecrets(); err != nil {
+		err = fmt.Errorf("could not sync secrets: %v", err)
+		return
 	}
 
 	c.logger.Debugf("syncing services")
-	for _, role := range []PostgresRole{Master, Replica} {
-		if role == Replica && !c.Spec.ReplicaLoadBalancer {
-			if c.Services[role] != nil {
-				// delete the left over replica service
-				if err = c.deleteService(role); err != nil {
-					err = fmt.Errorf("could not delete obsolete %s service: %v", role, err)
-					return
-				}
-			}
-			continue
-		}
-		if err = c.syncService(role); err != nil {
-			if !k8sutil.ResourceAlreadyExists(err) {
-				err = fmt.Errorf("coud not sync %s service: %v", role, err)
-				return
-			}
-		}
+	if err = c.syncServices(); err != nil {
+		err = fmt.Errorf("could not sync services: %v", err)
+		return
 	}
 
 	c.logger.Debugf("syncing statefulsets")
@@ -112,73 +85,170 @@ func (c *Cluster) Sync(newSpec *spec.Postgresql) (err error) {
 	return
 }
 
-func (c *Cluster) syncService(role PostgresRole) error {
-	cSpec := c.Spec
-	if c.Services[role] == nil {
-		c.logger.Infof("could not find the cluster's %s service", role)
-		svc, err := c.createService(role)
-		if err != nil {
-			return fmt.Errorf("could not create missing %s service: %v", role, err)
+func (c *Cluster) syncServices() error {
+	for _, role := range []PostgresRole{Master, Replica} {
+		c.logger.Debugf("syncing %s service", role)
+
+		if err := c.syncEndpoint(role); err != nil {
+			return fmt.Errorf("could not sync %s endpont: %v", role, err)
 		}
-		c.logger.Infof("created missing %s service %q", role, util.NameFromMeta(svc.ObjectMeta))
 
-		return nil
+		if err := c.syncService(role); err != nil {
+			return fmt.Errorf("could not sync %s service: %v", role, err)
+		}
 	}
 
-	desiredSvc := c.generateService(role, &cSpec)
-	match, reason := c.sameServiceWith(role, desiredSvc)
-	if match {
-		return nil
-	}
-	c.logServiceChanges(role, c.Services[role], desiredSvc, false, reason)
-
-	if err := c.updateService(role, desiredSvc); err != nil {
-		return fmt.Errorf("could not update %s service to match desired state: %v", role, err)
-	}
-	c.logger.Infof("%s service %q is in the desired state now", role, util.NameFromMeta(desiredSvc.ObjectMeta))
-
 	return nil
 }
 
-func (c *Cluster) syncEndpoint() error {
-	if c.Endpoint == nil {
-		c.logger.Infof("could not find the cluster's endpoint")
-		ep, err := c.createEndpoint()
-		if err != nil {
-			return fmt.Errorf("could not create missing endpoint: %v", err)
+func (c *Cluster) syncService(role PostgresRole) error {
+	c.setProcessName("syncing %s service", role)
+
+	svc, err := c.KubeClient.Services(c.Namespace).Get(c.serviceName(role), metav1.GetOptions{})
+	if err == nil {
+		if role == Replica && !c.Spec.ReplicaLoadBalancer {
+			if err := c.deleteService(role); err != nil {
+				return fmt.Errorf("could not delete %s service", role)
+			}
 		}
-		c.logger.Infof("created missing endpoint %q", util.NameFromMeta(ep.ObjectMeta))
+
+		desiredSvc := c.generateService(role, &c.Spec)
+		match, reason := k8sutil.SameService(svc, desiredSvc)
+		if match {
+			c.Services[role] = svc
+			return nil
+		}
+		c.logServiceChanges(role, svc, desiredSvc, false, reason)
+
+		if err := c.updateService(role, desiredSvc); err != nil {
+			return fmt.Errorf("could not update %s service to match desired state: %v", role, err)
+		}
+		c.logger.Infof("%s service %q is in the desired state now", role, util.NameFromMeta(desiredSvc.ObjectMeta))
+
 		return nil
+	} else if !k8sutil.ResourceNotFound(err) {
+		return fmt.Errorf("could not get %s service: %v", role, err)
+	}
+	c.Services[role] = nil
+
+	// Service does not exist
+	if role == Replica && !c.Spec.ReplicaLoadBalancer {
+		return nil
+	}
+
+	c.logger.Infof("could not find the cluster's %s service", role)
+
+	if svc, err := c.createService(role); err != nil {
+		if k8sutil.ResourceAlreadyExists(err) {
+			c.logger.Infof("%s service %q already exists", role, util.NameFromMeta(svc.ObjectMeta))
+			svc, err := c.KubeClient.Services(c.Namespace).Get(c.serviceName(role), metav1.GetOptions{})
+			if err == nil {
+				c.Services[role] = svc
+			} else {
+				c.logger.Infof("could not fetch existing %s service: %v", role, err)
+			}
+		} else {
+			return fmt.Errorf("could not create missing %s service: %v", role, err)
+		}
+	} else {
+		c.logger.Infof("created missing %s service %q", role, util.NameFromMeta(svc.ObjectMeta))
+		c.Services[role] = svc
+	}
+
+	return nil
+}
+
+func (c *Cluster) syncEndpoint(role PostgresRole) error {
+	c.setProcessName("syncing %s endpoint", role)
+
+	ep, err := c.KubeClient.Endpoints(c.Namespace).Get(c.endpointName(role), metav1.GetOptions{})
+	if err == nil {
+		if role == Replica && !c.Spec.ReplicaLoadBalancer {
+			if err := c.deleteEndpoint(role); err != nil {
+				return fmt.Errorf("could not delete %s endpoint", role)
+			}
+		}
+
+		c.Endpoints[role] = ep
+		return nil
+	} else if !k8sutil.ResourceNotFound(err) {
+		return fmt.Errorf("could not get %s endpoint: %v", role, err)
+	}
+	c.Endpoints[role] = nil
+
+	if role == Replica && !c.Spec.ReplicaLoadBalancer {
+		return nil
+	}
+
+	c.logger.Infof("could not find the cluster's %s endpoint", role)
+
+	if ep, err := c.createEndpoint(role); err != nil {
+		if k8sutil.ResourceAlreadyExists(err) {
+			c.logger.Infof("%s endpoint %q already exists", role, util.NameFromMeta(ep.ObjectMeta))
+			ep, err := c.KubeClient.Endpoints(c.Namespace).Get(c.endpointName(role), metav1.GetOptions{})
+			if err == nil {
+				c.Endpoints[role] = ep
+			} else {
+				c.logger.Infof("could not fetch existing %s endpoint: %v", role, err)
+			}
+		} else {
+			return fmt.Errorf("could not create missing %s endpoint: %v", role, err)
+		}
+	} else {
+		c.logger.Infof("created missing %s endpoint %q", role, util.NameFromMeta(ep.ObjectMeta))
+		c.Endpoints[role] = ep
 	}
 
 	return nil
 }
 
 func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error {
-	if c.PodDisruptionBudget == nil {
-		c.logger.Infof("could not find the cluster's pod disruption budget")
-		pdb, err := c.createPodDisruptionBudget()
-		if err != nil {
-			return fmt.Errorf("could not create pod disruption budget: %v", err)
-		}
-		c.logger.Infof("created missing pod disruption budget %q", util.NameFromMeta(pdb.ObjectMeta))
-		return nil
-	} else {
+	pdb, err := c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(c.podDisruptionBudgetName(), metav1.GetOptions{})
+	if err == nil {
+		c.PodDisruptionBudget = pdb
 		newPDB := c.generatePodDisruptionBudget()
-		if match, reason := c.samePDBWith(newPDB); !match {
-			c.logPDBChanges(c.PodDisruptionBudget, newPDB, isUpdate, reason)
+		if match, reason := k8sutil.SamePDB(pdb, newPDB); !match {
+			c.logPDBChanges(pdb, newPDB, isUpdate, reason)
 			if err := c.updatePodDisruptionBudget(newPDB); err != nil {
 				return err
 			}
+		} else {
+			c.PodDisruptionBudget = pdb
 		}
+
+		return nil
+	} else if !k8sutil.ResourceNotFound(err) {
+		return fmt.Errorf("could not get pod disruption budget: %v", err)
 	}
+	c.PodDisruptionBudget = nil
+
+	c.logger.Infof("could not find the cluster's pod disruption budget")
+	if pdb, err = c.createPodDisruptionBudget(); err != nil {
+		if k8sutil.ResourceAlreadyExists(err) {
+			c.logger.Infof("pod disruption budget %q already exists", util.NameFromMeta(pdb.ObjectMeta))
+		} else {
+			return fmt.Errorf("could not create pod disruption budget: %v", err)
+		}
+	} else {
+		c.logger.Infof("created missing pod disruption budget %q", util.NameFromMeta(pdb.ObjectMeta))
+		c.PodDisruptionBudget = pdb
+	}
+
 	return nil
 }
 
 func (c *Cluster) syncStatefulSet() error {
-	cSpec := c.Spec
-	var rollUpdate bool
-	if c.Statefulset == nil {
+	var (
+		err        error
+		rollUpdate bool
+	)
+	c.Statefulset, err = c.KubeClient.StatefulSets(c.Namespace).Get(c.statefulSetName(), metav1.GetOptions{})
+
+	if err != nil && !k8sutil.ResourceNotFound(err) {
+		return fmt.Errorf("could not get statefulset: %v", err)
+	}
+
+	if err != nil && k8sutil.ResourceNotFound(err) {
 		c.logger.Infof("could not find the cluster's statefulset")
 		pods, err := c.listPods()
 		if err != nil {
@@ -189,22 +259,25 @@ func (c *Cluster) syncStatefulSet() error {
 			c.logger.Infof("found pods without the statefulset: trigger rolling update")
 			rollUpdate = true
 		}
+
 		ss, err := c.createStatefulSet()
 		if err != nil {
 			return fmt.Errorf("could not create missing statefulset: %v", err)
 		}
-		err = c.waitStatefulsetPodsReady()
-		if err != nil {
+
+		if err = c.waitStatefulsetPodsReady(); err != nil {
 			return fmt.Errorf("cluster is not ready: %v", err)
 		}
+
 		c.logger.Infof("created missing statefulset %q", util.NameFromMeta(ss.ObjectMeta))
 		if !rollUpdate {
 			return nil
 		}
 	}
+
 	/* TODO: should check that we need to replace the statefulset */
 	if !rollUpdate {
-		desiredSS, err := c.generateStatefulSet(cSpec)
+		desiredSS, err := c.generateStatefulSet(&c.Spec)
 		if err != nil {
 			return fmt.Errorf("could not generate statefulset: %v", err)
 		}
@@ -239,6 +312,45 @@ func (c *Cluster) syncStatefulSet() error {
 	return nil
 }
 
+func (c *Cluster) syncSecrets() error {
+	c.setProcessName("syncing secrets")
+	secrets := c.generateUserSecrets()
+
+	for secretUsername, secretSpec := range secrets {
+		secret, err := c.KubeClient.Secrets(secretSpec.Namespace).Create(secretSpec)
+		if k8sutil.ResourceAlreadyExists(err) {
+			var userMap map[string]spec.PgUser
+			curSecret, err2 := c.KubeClient.Secrets(secretSpec.Namespace).Get(secretSpec.Name, metav1.GetOptions{})
+			if err2 != nil {
+				return fmt.Errorf("could not get current secret: %v", err2)
+			}
+			c.logger.Debugf("secret %q already exists, fetching it's password", util.NameFromMeta(curSecret.ObjectMeta))
+			if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name {
+				secretUsername = constants.SuperuserKeyName
+				userMap = c.systemUsers
+			} else if secretUsername == c.systemUsers[constants.ReplicationUserKeyName].Name {
+				secretUsername = constants.ReplicationUserKeyName
+				userMap = c.systemUsers
+			} else {
+				userMap = c.pgUsers
+			}
+			pwdUser := userMap[secretUsername]
+			pwdUser.Password = string(curSecret.Data["password"])
+			userMap[secretUsername] = pwdUser
+
+			continue
+		} else {
+			if err != nil {
+				return fmt.Errorf("could not create secret for user %q: %v", secretUsername, err)
+			}
+			c.Secrets[secret.UID] = secret
+			c.logger.Debugf("created new secret %q, uid: %q", util.NameFromMeta(secret.ObjectMeta), secret.UID)
+		}
+	}
+
+	return nil
+}
+
 func (c *Cluster) syncRoles(readFromDatabase bool) error {
 	c.setProcessName("syncing roles")
 
@@ -268,11 +380,14 @@ func (c *Cluster) syncRoles(readFromDatabase bool) error {
 	if err = c.userSyncStrategy.ExecuteSyncRequests(pgSyncRequests, c.pgDb); err != nil {
 		return fmt.Errorf("error executing sync statements: %v", err)
 	}
+
 	return nil
 }
 
 // syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset.
 func (c *Cluster) syncVolumes() error {
+	c.setProcessName("syncing volumes")
+
 	act, err := c.volumesNeedResizing(c.Spec.Volume)
 	if err != nil {
 		return fmt.Errorf("could not compare size of the volumes: %v", err)