Use queues for the pod events (#30)

pkg/cluster/cluster.go

@@ -17,6 +17,7 @@ import (
 	"k8s.io/client-go/pkg/apis/apps/v1beta1"
 	"k8s.io/client-go/pkg/types"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
 
 	"github.com/zalando-incubator/postgres-operator/pkg/spec"
 	"github.com/zalando-incubator/postgres-operator/pkg/util"
@@ -54,18 +55,17 @@ type Cluster struct {
 	kubeResources
 	spec.Postgresql
 	Config
-	logger               *logrus.Entry
-	pgUsers              map[string]spec.PgUser
-	systemUsers          map[string]spec.PgUser
-	podEvents            chan spec.PodEvent
-	podSubscribers       map[spec.NamespacedName]chan spec.PodEvent
-	podSubscribersMu     sync.RWMutex
-	pgDb                 *sql.DB
-	mu                   sync.Mutex
-	masterLess           bool
-	podDispatcherRunning bool
-	userSyncStrategy     spec.UserSyncer
-	deleteOptions        *v1.DeleteOptions
+	logger           *logrus.Entry
+	pgUsers          map[string]spec.PgUser
+	systemUsers      map[string]spec.PgUser
+	podSubscribers   map[spec.NamespacedName]chan spec.PodEvent
+	podSubscribersMu sync.RWMutex
+	pgDb             *sql.DB
+	mu               sync.Mutex
+	masterLess       bool
+	userSyncStrategy spec.UserSyncer
+	deleteOptions    *v1.DeleteOptions
+	podEventsQueue   *cache.FIFO
 }
 
 func New(cfg Config, pgSpec spec.Postgresql, logger *logrus.Entry) *Cluster {
@@ -73,19 +73,27 @@ func New(cfg Config, pgSpec spec.Postgresql, logger *logrus.Entry) *Cluster {
 	kubeResources := kubeResources{Secrets: make(map[types.UID]*v1.Secret)}
 	orphanDependents := true
 
+	podEventsQueue := cache.NewFIFO(func(obj interface{}) (string, error) {
+		e, ok := obj.(spec.PodEvent)
+		if !ok {
+			return "", fmt.Errorf("could not cast to PodEvent")
+		}
+
+		return fmt.Sprintf("%s-%s", e.PodName, e.ResourceVersion), nil
+	})
+
 	cluster := &Cluster{
-		Config:               cfg,
-		Postgresql:           pgSpec,
-		logger:               lg,
-		pgUsers:              make(map[string]spec.PgUser),
-		systemUsers:          make(map[string]spec.PgUser),
-		podEvents:            make(chan spec.PodEvent),
-		podSubscribers:       make(map[spec.NamespacedName]chan spec.PodEvent),
-		kubeResources:        kubeResources,
-		masterLess:           false,
-		podDispatcherRunning: false,
-		userSyncStrategy:     users.DefaultUserSyncStrategy{},
-		deleteOptions:        &v1.DeleteOptions{OrphanDependents: &orphanDependents},
+		Config:           cfg,
+		Postgresql:       pgSpec,
+		logger:           lg,
+		pgUsers:          make(map[string]spec.PgUser),
+		systemUsers:      make(map[string]spec.PgUser),
+		podSubscribers:   make(map[spec.NamespacedName]chan spec.PodEvent),
+		kubeResources:    kubeResources,
+		masterLess:       false,
+		userSyncStrategy: users.DefaultUserSyncStrategy{},
+		deleteOptions:    &v1.DeleteOptions{OrphanDependents: &orphanDependents},
+		podEventsQueue:   podEventsQueue,
 	}
 
 	return cluster
@@ -143,16 +151,11 @@ func (c *Cluster) initUsers() error {
 	return nil
 }
 
-func (c *Cluster) Create(stopCh <-chan struct{}) error {
+func (c *Cluster) Create() error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	var err error
 
-	if !c.podDispatcherRunning {
-		go c.podEventsDispatcher(stopCh)
-		c.podDispatcherRunning = true
-	}
-
 	defer func() {
 		if err == nil {
 			c.setStatus(spec.ClusterStatusRunning) //TODO: are you sure it's running?
@@ -460,7 +463,38 @@ func (c *Cluster) Delete() error {
 }
 
 func (c *Cluster) ReceivePodEvent(event spec.PodEvent) {
-	c.podEvents <- event
+	c.podEventsQueue.Add(event)
+}
+
+func (c *Cluster) processPodEvent(obj interface{}) error {
+	event, ok := obj.(spec.PodEvent)
+	if !ok {
+		return fmt.Errorf("could not cast to PodEvent")
+	}
+
+	c.podSubscribersMu.RLock()
+	subscriber, ok := c.podSubscribers[event.PodName]
+	c.podSubscribersMu.RUnlock()
+	if ok {
+		subscriber <- event
+	}
+
+	return nil
+}
+
+func (c *Cluster) Run(stopCh <-chan struct{}) {
+	go c.processPodEventQueue(stopCh)
+}
+
+func (c *Cluster) processPodEventQueue(stopCh <-chan struct{}) {
+	for {
+		select {
+		case <-stopCh:
+			return
+		default:
+			c.podEventsQueue.Pop(cache.PopProcessFunc(c.processPodEvent))
+		}
+	}
 }
 
 func (c *Cluster) initSystemUsers() {

pkg/cluster/pod.go

@@ -146,23 +146,6 @@ func (c *Cluster) recreatePod(pod v1.Pod) error {
 	return nil
 }
 
-func (c *Cluster) podEventsDispatcher(stopCh <-chan struct{}) {
-	c.logger.Infof("Watching '%s' cluster", c.ClusterName())
-	for {
-		select {
-		case event := <-c.podEvents:
-			c.podSubscribersMu.RLock()
-			subscriber, ok := c.podSubscribers[event.PodName]
-			c.podSubscribersMu.RUnlock()
-			if ok {
-				go func() { subscriber <- event }() //TODO: is it a right way to do nonblocking send to the channel?
-			}
-		case <-stopCh:
-			return
-		}
-	}
-}
-
 func (c *Cluster) recreatePods() error {
 	ls := c.labelsSet()
 	namespace := c.Metadata.Namespace

pkg/cluster/sync.go

@@ -7,7 +7,7 @@ import (
 	"github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil"
 )
 
-func (c *Cluster) Sync(stopCh <-chan struct{}) error {
+func (c *Cluster) Sync() error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -16,11 +16,6 @@ func (c *Cluster) Sync(stopCh <-chan struct{}) error {
 		c.logger.Errorf("could not load resources: %v", err)
 	}
 
-	if !c.podDispatcherRunning {
-		go c.podEventsDispatcher(stopCh)
-		c.podDispatcherRunning = true
-	}
-
 	c.logger.Debugf("Syncing secrets")
 	if err := c.syncSecrets(); err != nil {
 		if !k8sutil.ResourceAlreadyExists(err) {