public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-10-03 16:30:11 +02:00
Code Issues Projects Releases Wiki Activity

Parallel cluster processing

Browse Source 
Run operations concerning multiple clusters in parallel. Each cluster gets its
own worker in order to create, update, sync or delete clusters.  Each worker
acquires the lock on a cluster.  Subsequent operations on the same cluster
have to wait until the current one finishes.  There is a pool of parallel
workers, configurable with the `workers` parameter in the configmap and set by
default to 4. The cluster-related tasks  are assigned to the workers based on
a cluster name: the tasks for the same cluster will be always assigned to the
same worker. There is no blocking between workers, although there is a chance
that a single worker will become a bottleneck if too many clusters are
assigned to it; therefore, for large-scale deployments it might be necessary
to bump up workers from the default value.
This commit is contained in:
Murat Kabilov
2017-05-12 11:41:35 +02:00
parent a9c6c4861c
commit 2370659c69
13 changed files with 304 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cluster/pod.go
+17 -21
View File
@@ -72,7 +72,7 @@ func (c *Cluster) deletePersistenVolumeClaims() error {
}
for _, pvc := range pvcs {
c.logger.Debugf("Deleting PVC '%s'", util.NameFromMeta(pvc.ObjectMeta))
if err := c.KubeClient.PersistentVolumeClaims(ns).Delete(pvc.Name, deleteOptions); err != nil {
if err := c.KubeClient.PersistentVolumeClaims(ns).Delete(pvc.Name, c.deleteOptions); err != nil {
c.logger.Warningf("Can't delete PersistentVolumeClaim: %s", err)
}
}
@@ -86,17 +86,10 @@ func (c *Cluster) deletePersistenVolumeClaims() error {
}
func (c *Cluster) deletePod(podName spec.NamespacedName) error {
ch := make(chan spec.PodEvent)
if _, ok := c.podSubscribers[podName]; ok {
panic("Pod '" + podName.String() + "' is already subscribed")
}
c.podSubscribers[podName] = ch
defer func() {
close(ch)
delete(c.podSubscribers, podName)
}()
ch := c.registerPodSubscriber(podName)
defer c.unregisterPodSubscriber(podName)
if err := c.KubeClient.Pods(podName.Namespace).Delete(podName.Name, deleteOptions); err != nil {
if err := c.KubeClient.Pods(podName.Namespace).Delete(podName.Name, c.deleteOptions); err != nil {
return err
}
@@ -108,6 +101,9 @@ func (c *Cluster) deletePod(podName spec.NamespacedName) error {
}
func (c *Cluster) unregisterPodSubscriber(podName spec.NamespacedName) {
c.podSubscribersMu.Lock()
defer c.podSubscribersMu.Unlock()
if _, ok := c.podSubscribers[podName]; !ok {
panic("Subscriber for Pod '" + podName.String() + "' is not found")
}
@@ -117,26 +113,25 @@ func (c *Cluster) unregisterPodSubscriber(podName spec.NamespacedName) {
}
func (c *Cluster) registerPodSubscriber(podName spec.NamespacedName) chan spec.PodEvent {
c.podSubscribersMu.Lock()
defer c.podSubscribersMu.Unlock()
ch := make(chan spec.PodEvent)
if _, ok := c.podSubscribers[podName]; ok {
panic("Pod '" + podName.String() + "' is already subscribed")
}
c.podSubscribers[podName] = ch
return ch
}
func (c *Cluster) recreatePod(pod v1.Pod) error {
podName := util.NameFromMeta(pod.ObjectMeta)
orphanDependents := false
deleteOptions := &v1.DeleteOptions{
OrphanDependents: &orphanDependents,
}
ch := c.registerPodSubscriber(podName)
defer c.unregisterPodSubscriber(podName)
if err := c.KubeClient.Pods(pod.Namespace).Delete(pod.Name, deleteOptions); err != nil {
if err := c.KubeClient.Pods(pod.Namespace).Delete(pod.Name, c.deleteOptions); err != nil {
return fmt.Errorf("Can't delete Pod: %s", err)
}
@@ -156,10 +151,11 @@ func (c *Cluster) podEventsDispatcher(stopCh <-chan struct{}) {
for {
select {
case event := <-c.podEvents:
if subscriber, ok := c.podSubscribers[event.PodName]; ok {
c.podSubscribersMu.RLock()
subscriber, ok := c.podSubscribers[event.PodName]
c.podSubscribersMu.RUnlock()
if ok {
go func() { subscriber <- event }() //TODO: is it a right way to do nonblocking send to the channel?
} else {
c.logger.Debugf("Skipping event for an unwatched Pod '%s'", event.PodName)
}
case <-stopCh:
return
@@ -183,7 +179,7 @@ func (c *Cluster) recreatePods() error {
var masterPod v1.Pod
for _, pod := range pods.Items {
role := c.PodSpiloRole(&pod)
role := c.podSpiloRole(&pod)
if role == constants.PodRoleMaster {
masterPod = pod