Parallel cluster processing

Run operations concerning multiple clusters in parallel. Each cluster gets its
own worker in order to create, update, sync or delete clusters.  Each worker
acquires the lock on a cluster.  Subsequent operations on the same cluster
have to wait until the current one finishes.  There is a pool of parallel
workers, configurable with the `workers` parameter in the configmap and set by
default to 4. The cluster-related tasks  are assigned to the workers based on
a cluster name: the tasks for the same cluster will be always assigned to the
same worker. There is no blocking between workers, although there is a chance
that a single worker will become a bottleneck if too many clusters are
assigned to it; therefore, for large-scale deployments it might be necessary
to bump up workers from the default value.

pkg/cluster/cluster.go

@@ -39,7 +39,7 @@ type Config struct {
 	RestClient          *rest.RESTClient
 	EtcdClient          etcdclient.KeysAPI
 	TeamsAPIClient      *teams.TeamsAPI
-	OpConfig            *config.Config
+	OpConfig            config.Config
 	InfrastructureRoles map[string]spec.PgUser // inherited from the controller
 }
 
@@ -60,15 +60,18 @@ type Cluster struct {
 	pgUsers              map[string]spec.PgUser
 	podEvents            chan spec.PodEvent
 	podSubscribers       map[spec.NamespacedName]chan spec.PodEvent
+	podSubscribersMu     sync.RWMutex
 	pgDb                 *sql.DB
 	mu                   sync.Mutex
 	masterLess           bool
 	podDispatcherRunning bool
+	deleteOptions        *v1.DeleteOptions
 }
 
-func New(cfg Config, pgSpec spec.Postgresql, logger *logrus.Logger) *Cluster {
+func New(cfg Config, pgSpec spec.Postgresql, logger *logrus.Entry) *Cluster {
 	lg := logger.WithField("pkg", "cluster").WithField("cluster-name", pgSpec.Metadata.Name)
 	kubeResources := kubeResources{Secrets: make(map[types.UID]*v1.Secret)}
+	orphanDependents := true
 
 	cluster := &Cluster{
 		Config:               cfg,
@@ -80,6 +83,7 @@ func New(cfg Config, pgSpec spec.Postgresql, logger *logrus.Logger) *Cluster {
 		kubeResources:        kubeResources,
 		masterLess:           false,
 		podDispatcherRunning: false,
+		deleteOptions:        &v1.DeleteOptions{OrphanDependents: &orphanDependents},
 	}
 
 	return cluster
@@ -89,12 +93,12 @@ func (c *Cluster) ClusterName() spec.NamespacedName {
 	return util.NameFromMeta(c.Metadata)
 }
 
-func (c *Cluster) TeamName() string {
+func (c *Cluster) teamName() string {
 	// TODO: check Teams API for the actual name (in case the user passes an integer Id).
 	return c.Spec.TeamId
 }
 
-func (c *Cluster) SetStatus(status spec.PostgresStatus) {
+func (c *Cluster) setStatus(status spec.PostgresStatus) {
 	c.Status = status
 	b, err := json.Marshal(status)
 	if err != nil {
@@ -154,11 +158,25 @@ func (c *Cluster) etcdKeyExists(keyName string) (bool, error) {
 }
 
 func (c *Cluster) Create(stopCh <-chan struct{}) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	var err error
+
 	if !c.podDispatcherRunning {
 		go c.podEventsDispatcher(stopCh)
 		c.podDispatcherRunning = true
 	}
 
+	defer func() {
+		if err == nil {
+			c.setStatus(spec.ClusterStatusRunning) //TODO: are you sure it's running?
+		} else {
+			c.setStatus(spec.ClusterStatusAddFailed)
+		}
+	}()
+
+	c.setStatus(spec.ClusterStatusCreating)
+
 	keyExist, err := c.etcdKeyExists(fmt.Sprintf("/%s/%s", c.OpConfig.EtcdScope, c.Metadata.Name))
 	if err != nil {
 		c.logger.Warnf("Can't check etcd key: %s", err)
@@ -180,38 +198,36 @@ func (c *Cluster) Create(stopCh <-chan struct{}) error {
 		c.logger.Infof("Service '%s' has been successfully created", util.NameFromMeta(service.ObjectMeta))
 	}
 
-	if err := c.initUsers(); err != nil {
+	if err = c.initUsers(); err != nil {
 		return err
-	} else {
-		c.logger.Infof("User secrets have been initialized")
 	}
+	c.logger.Infof("User secrets have been initialized")
 
-	if err := c.applySecrets(); err != nil {
+	if err = c.applySecrets(); err != nil {
 		return fmt.Errorf("Can't create Secrets: %s", err)
-	} else {
-		c.logger.Infof("Secrets have been successfully created")
 	}
+	c.logger.Infof("Secrets have been successfully created")
 
 	ss, err := c.createStatefulSet()
 	if err != nil {
 		return fmt.Errorf("Can't create StatefulSet: %s", err)
-	} else {
-		c.logger.Infof("StatefulSet '%s' has been successfully created", util.NameFromMeta(ss.ObjectMeta))
 	}
+	c.logger.Infof("StatefulSet '%s' has been successfully created", util.NameFromMeta(ss.ObjectMeta))
 
 	c.logger.Info("Waiting for cluster being ready")
 
-	if err := c.waitStatefulsetPodsReady(); err != nil {
+	if err = c.waitStatefulsetPodsReady(); err != nil {
 		c.logger.Errorf("Failed to create cluster: %s", err)
 		return err
 	}
+	c.logger.Infof("Pods are ready")
 
 	if !c.masterLess {
-		if err := c.initDbConn(); err != nil {
+		if err = c.initDbConn(); err != nil {
 			return fmt.Errorf("Can't init db connection: %s", err)
 		}
 
-		if err := c.createUsers(); err != nil {
+		if err = c.createUsers(); err != nil {
 			return fmt.Errorf("Can't create users: %s", err)
 		} else {
 			c.logger.Infof("Users have been successfully created")
@@ -323,13 +339,18 @@ func compareResoucesAssumeFirstNotNil(a *v1.ResourceRequirements, b *v1.Resource
 }
 
 func (c *Cluster) Update(newSpec *spec.Postgresql) error {
-	c.logger.Infof("Cluster update from version %s to %s",
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.setStatus(spec.ClusterStatusUpdating)
+	c.logger.Debugf("Cluster update from version %s to %s",
 		c.Metadata.ResourceVersion, newSpec.Metadata.ResourceVersion)
 
 	newService := c.genService(newSpec.Spec.AllowedSourceRanges)
 	if match, reason := c.sameServiceWith(newService); !match {
 		c.logServiceChanges(c.Service, newService, true, reason)
 		if err := c.updateService(newService); err != nil {
+			c.setStatus(spec.ClusterStatusUpdateFailed)
 			return fmt.Errorf("Can't update Service: %s", err)
 		} else {
 			c.logger.Infof("Service '%s' has been updated", util.NameFromMeta(c.Service.ObjectMeta))
@@ -348,8 +369,10 @@ func (c *Cluster) Update(newSpec *spec.Postgresql) error {
 		c.logStatefulSetChanges(c.Statefulset, newStatefulSet, true, reason)
 		//TODO: mind the case of updating allowedSourceRanges
 		if err := c.updateStatefulSet(newStatefulSet); err != nil {
+			c.setStatus(spec.ClusterStatusUpdateFailed)
 			return fmt.Errorf("Can't upate StatefulSet: %s", err)
 		}
+		//TODO: if there is a change in numberOfInstances, make sure Pods have been created/deleted
 		c.logger.Infof("StatefulSet '%s' has been updated", util.NameFromMeta(c.Statefulset.ObjectMeta))
 	}
 
@@ -363,15 +386,20 @@ func (c *Cluster) Update(newSpec *spec.Postgresql) error {
 		c.logger.Infof("Rolling update is needed")
 		// TODO: wait for actual streaming to the replica
 		if err := c.recreatePods(); err != nil {
+			c.setStatus(spec.ClusterStatusUpdateFailed)
 			return fmt.Errorf("Can't recreate Pods: %s", err)
 		}
 		c.logger.Infof("Rolling update has been finished")
 	}
+	c.setStatus(spec.ClusterStatusRunning)
 
 	return nil
 }
 
 func (c *Cluster) Delete() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
 	if err := c.deleteEndpoint(); err != nil {
 		c.logger.Errorf("Can't delete Endpoint: %s", err)
 	}

pkg/cluster/pg.go

@@ -26,8 +26,6 @@ func (c *Cluster) pgConnectionString() string {
 func (c *Cluster) initDbConn() error {
 	//TODO: concurrent safe?
 	if c.pgDb == nil {
-		c.mu.Lock()
-		defer c.mu.Unlock()
 		if c.pgDb == nil {
 			conn, err := sql.Open("postgres", c.pgConnectionString())
 			if err != nil {

pkg/cluster/pod.go

@@ -72,7 +72,7 @@ func (c *Cluster) deletePersistenVolumeClaims() error {
 	}
 	for _, pvc := range pvcs {
 		c.logger.Debugf("Deleting PVC '%s'", util.NameFromMeta(pvc.ObjectMeta))
-		if err := c.KubeClient.PersistentVolumeClaims(ns).Delete(pvc.Name, deleteOptions); err != nil {
+		if err := c.KubeClient.PersistentVolumeClaims(ns).Delete(pvc.Name, c.deleteOptions); err != nil {
 			c.logger.Warningf("Can't delete PersistentVolumeClaim: %s", err)
 		}
 	}
@@ -86,17 +86,10 @@ func (c *Cluster) deletePersistenVolumeClaims() error {
 }
 
 func (c *Cluster) deletePod(podName spec.NamespacedName) error {
-	ch := make(chan spec.PodEvent)
-	if _, ok := c.podSubscribers[podName]; ok {
-		panic("Pod '" + podName.String() + "' is already subscribed")
-	}
-	c.podSubscribers[podName] = ch
-	defer func() {
-		close(ch)
-		delete(c.podSubscribers, podName)
-	}()
+	ch := c.registerPodSubscriber(podName)
+	defer c.unregisterPodSubscriber(podName)
 
-	if err := c.KubeClient.Pods(podName.Namespace).Delete(podName.Name, deleteOptions); err != nil {
+	if err := c.KubeClient.Pods(podName.Namespace).Delete(podName.Name, c.deleteOptions); err != nil {
 		return err
 	}
 
@@ -108,6 +101,9 @@ func (c *Cluster) deletePod(podName spec.NamespacedName) error {
 }
 
 func (c *Cluster) unregisterPodSubscriber(podName spec.NamespacedName) {
+	c.podSubscribersMu.Lock()
+	defer c.podSubscribersMu.Unlock()
+
 	if _, ok := c.podSubscribers[podName]; !ok {
 		panic("Subscriber for Pod '" + podName.String() + "' is not found")
 	}
@@ -117,26 +113,25 @@ func (c *Cluster) unregisterPodSubscriber(podName spec.NamespacedName) {
 }
 
 func (c *Cluster) registerPodSubscriber(podName spec.NamespacedName) chan spec.PodEvent {
+	c.podSubscribersMu.Lock()
+	defer c.podSubscribersMu.Unlock()
+
 	ch := make(chan spec.PodEvent)
 	if _, ok := c.podSubscribers[podName]; ok {
 		panic("Pod '" + podName.String() + "' is already subscribed")
 	}
 	c.podSubscribers[podName] = ch
+
 	return ch
 }
 
 func (c *Cluster) recreatePod(pod v1.Pod) error {
 	podName := util.NameFromMeta(pod.ObjectMeta)
 
-	orphanDependents := false
-	deleteOptions := &v1.DeleteOptions{
-		OrphanDependents: &orphanDependents,
-	}
-
 	ch := c.registerPodSubscriber(podName)
 	defer c.unregisterPodSubscriber(podName)
 
-	if err := c.KubeClient.Pods(pod.Namespace).Delete(pod.Name, deleteOptions); err != nil {
+	if err := c.KubeClient.Pods(pod.Namespace).Delete(pod.Name, c.deleteOptions); err != nil {
 		return fmt.Errorf("Can't delete Pod: %s", err)
 	}
 
@@ -156,10 +151,11 @@ func (c *Cluster) podEventsDispatcher(stopCh <-chan struct{}) {
 	for {
 		select {
 		case event := <-c.podEvents:
-			if subscriber, ok := c.podSubscribers[event.PodName]; ok {
+			c.podSubscribersMu.RLock()
+			subscriber, ok := c.podSubscribers[event.PodName]
+			c.podSubscribersMu.RUnlock()
+			if ok {
 				go func() { subscriber <- event }() //TODO: is it a right way to do nonblocking send to the channel?
-			} else {
-				c.logger.Debugf("Skipping event for an unwatched Pod '%s'", event.PodName)
 			}
 		case <-stopCh:
 			return
@@ -183,7 +179,7 @@ func (c *Cluster) recreatePods() error {
 
 	var masterPod v1.Pod
 	for _, pod := range pods.Items {
-		role := c.PodSpiloRole(&pod)
+		role := c.podSpiloRole(&pod)
 
 		if role == constants.PodRoleMaster {
 			masterPod = pod

pkg/cluster/resources.go

@@ -11,12 +11,7 @@ import (
 	"github.bus.zalan.do/acid/postgres-operator/pkg/util/k8sutil"
 )
 
-var (
-	deleteOptions    = &v1.DeleteOptions{OrphanDependents: &orphanDependents}
-	orphanDependents = false
-)
-
-func (c *Cluster) LoadResources() error {
+func (c *Cluster) loadResources() error {
 	ns := c.Metadata.Namespace
 	listOptions := v1.ListOptions{
 		LabelSelector: c.labelsSet().String(),
@@ -152,7 +147,7 @@ func (c *Cluster) deleteStatefulSet() error {
 		return fmt.Errorf("There is no StatefulSet in the cluster")
 	}
 
-	err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Delete(c.Statefulset.Name, deleteOptions)
+	err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Delete(c.Statefulset.Name, c.deleteOptions)
 	if err != nil {
 		return err
 	}
@@ -217,7 +212,7 @@ func (c *Cluster) deleteService() error {
 	if c.Service == nil {
 		return fmt.Errorf("There is no Service in the cluster")
 	}
-	err := c.KubeClient.Services(c.Service.Namespace).Delete(c.Service.Name, deleteOptions)
+	err := c.KubeClient.Services(c.Service.Namespace).Delete(c.Service.Name, c.deleteOptions)
 	if err != nil {
 		return err
 	}
@@ -256,7 +251,7 @@ func (c *Cluster) deleteEndpoint() error {
 	if c.Endpoint == nil {
 		return fmt.Errorf("There is no Endpoint in the cluster")
 	}
-	err := c.KubeClient.Endpoints(c.Endpoint.Namespace).Delete(c.Endpoint.Name, deleteOptions)
+	err := c.KubeClient.Endpoints(c.Endpoint.Namespace).Delete(c.Endpoint.Name, c.deleteOptions)
 	if err != nil {
 		return err
 	}
@@ -300,7 +295,7 @@ func (c *Cluster) applySecrets() error {
 
 func (c *Cluster) deleteSecret(secret *v1.Secret) error {
 	c.logger.Debugf("Deleting Secret '%s'", util.NameFromMeta(secret.ObjectMeta))
-	err := c.KubeClient.Secrets(secret.Namespace).Delete(secret.Name, deleteOptions)
+	err := c.KubeClient.Secrets(secret.Namespace).Delete(secret.Name, c.deleteOptions)
 	if err != nil {
 		return err
 	}

pkg/cluster/sync.go

@@ -7,8 +7,14 @@ import (
 )
 
 func (c *Cluster) SyncCluster(stopCh <-chan struct{}) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.loadResources()
+
 	if !c.podDispatcherRunning {
 		go c.podEventsDispatcher(stopCh)
+		c.podDispatcherRunning = true
 	}
 
 	c.logger.Debugf("Syncing Secrets")

pkg/cluster/util.go

@@ -110,7 +110,7 @@ func (c *Cluster) waitForPodLabel(podEvents chan spec.PodEvent) error {
 	for {
 		select {
 		case podEvent := <-podEvents:
-			role := c.PodSpiloRole(podEvent.CurPod)
+			role := c.podSpiloRole(podEvent.CurPod)
 			// We cannot assume any role of the newly created pod. Normally, for a multi-pod cluster
 			// we should observe the 'replica' value, but it could be that some pods are not allowed
 			// to promote, therefore, the new pod could be a master as well.
@@ -127,7 +127,7 @@ func (c *Cluster) waitForPodDeletion(podEvents chan spec.PodEvent) error {
 	for {
 		select {
 		case podEvent := <-podEvents:
-			if podEvent.EventType == spec.PodEventDelete {
+			if podEvent.EventType == spec.EventDelete {
 				return nil
 			}
 		case <-time.After(c.OpConfig.PodDeletionWaitTimeout):
@@ -233,7 +233,7 @@ func (c *Cluster) dnsName() string {
 	return strings.ToLower(fmt.Sprintf(
 		c.OpConfig.DNSNameFormat,
 		c.Spec.ClusterName,
-		c.TeamName(),
+		c.teamName(),
 		c.OpConfig.DbHostedZone))
 }
 
@@ -266,6 +266,6 @@ func (c *Cluster) deleteEtcdKey() error {
 	return nil
 }
 
-func (c *Cluster) PodSpiloRole(pod *v1.Pod) string {
+func (c *Cluster) podSpiloRole(pod *v1.Pod) string {
 	return pod.Labels[c.OpConfig.PodRoleLabel]
 }