Parallel cluster processing

Run operations concerning multiple clusters in parallel. Each cluster gets its
own worker in order to create, update, sync or delete clusters.  Each worker
acquires the lock on a cluster.  Subsequent operations on the same cluster
have to wait until the current one finishes.  There is a pool of parallel
workers, configurable with the `workers` parameter in the configmap and set by
default to 4. The cluster-related tasks  are assigned to the workers based on
a cluster name: the tasks for the same cluster will be always assigned to the
same worker. There is no blocking between workers, although there is a chance
that a single worker will become a bottleneck if too many clusters are
assigned to it; therefore, for large-scale deployments it might be necessary
to bump up workers from the default value.

pkg/controller/controller.go

@@ -1,6 +1,7 @@
 package controller
 
 import (
+	"fmt"
 	"sync"
 
 	"github.com/Sirupsen/logrus"
@@ -28,14 +29,17 @@ type Controller struct {
 	Config
 	opConfig    *config.Config
 	logger      *logrus.Entry
-	clusters    map[spec.NamespacedName]*cluster.Cluster
-	stopChMap   map[spec.NamespacedName]chan struct{}
 	waitCluster sync.WaitGroup
 
+	clustersMu sync.RWMutex
+	clusters   map[spec.NamespacedName]*cluster.Cluster
+	stopChs    map[spec.NamespacedName]chan struct{}
+
 	postgresqlInformer cache.SharedIndexInformer
 	podInformer        cache.SharedIndexInformer
+	podCh              chan spec.PodEvent
 
-	podCh chan spec.PodEvent
+	clusterEventQueues []*cache.FIFO
 }
 
 func New(controllerConfig *Config, operatorConfig *config.Config) *Controller {
@@ -47,12 +51,12 @@ func New(controllerConfig *Config, operatorConfig *config.Config) *Controller {
 
 	controllerConfig.TeamsAPIClient = teams.NewTeamsAPI(operatorConfig.TeamsAPIUrl, logger)
 	return &Controller{
-		Config:    *controllerConfig,
-		opConfig:  operatorConfig,
-		logger:    logger.WithField("pkg", "controller"),
-		clusters:  make(map[spec.NamespacedName]*cluster.Cluster),
-		stopChMap: make(map[spec.NamespacedName]chan struct{}),
-		podCh:     make(chan spec.PodEvent),
+		Config:   *controllerConfig,
+		opConfig: operatorConfig,
+		logger:   logger.WithField("pkg", "controller"),
+		clusters: make(map[spec.NamespacedName]*cluster.Cluster),
+		stopChs:  make(map[spec.NamespacedName]chan struct{}),
+		podCh:    make(chan spec.PodEvent),
 	}
 }
 
@@ -64,6 +68,10 @@ func (c *Controller) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) {
 
 	go c.runInformers(stopCh)
 
+	for i := range c.clusterEventQueues {
+		go c.processClusterEventsQueue(i)
+	}
+
 	c.logger.Info("Started working in background")
 }
 
@@ -117,6 +125,18 @@ func (c *Controller) initController() {
 	if err := c.initEtcdClient(c.opConfig.EtcdHost); err != nil {
 		c.logger.Fatalf("Can't get etcd client: %s", err)
 	}
+
+	c.clusterEventQueues = make([]*cache.FIFO, c.opConfig.Workers)
+	for i := range c.clusterEventQueues {
+		c.clusterEventQueues[i] = cache.NewFIFO(func(obj interface{}) (string, error) {
+			e, ok := obj.(spec.ClusterEvent)
+			if !ok {
+				return "", fmt.Errorf("Can't cast to ClusterEvent")
+			}
+
+			return fmt.Sprintf("%s-%s", e.EventType, e.UID), nil
+		})
+	}
 }
 
 func (c *Controller) runInformers(stopCh <-chan struct{}) {