WIP: Hold the proper locks while examining the list of databases.

Introduce a new lock called specMu lock to protect the cluster spec.
This lock is held on update and sync, and when retrieving the spec in
the API code. There is no need to acquire it for cluster creation and
deletion: creation assigns the spec to the cluster before linking it to
the controller, and deletion just removes the cluster from the list in
the controller, both holding the global clustersMu Lock.

pkg/cluster/cluster.go

@@ -75,7 +75,8 @@ type Cluster struct {
 	oauthTokenGetter OAuthTokenGetter
 	KubeClient       k8sutil.KubernetesClient //TODO: move clients to the better place?
 	currentProcess   spec.Process
-	processMu        sync.RWMutex
+	processMu        sync.RWMutex // protects the current operation for reporting, no need to hold the master mutex
+	specMu           sync.RWMutex // protects the spec for reporting, no need to hold the master mutex
 }
 
 type compareStatefulsetResult struct {
@@ -437,7 +438,7 @@ func (c *Cluster) Update(oldSpec, newSpec *spec.Postgresql) error {
 	defer c.mu.Unlock()
 
 	c.setStatus(spec.ClusterStatusUpdating)
-	c.Postgresql = *newSpec
+	c.setSpec(newSpec)
 
 	defer func() {
 		if updateFailed {

pkg/cluster/sync.go

@@ -20,7 +20,7 @@ func (c *Cluster) Sync(newSpec *spec.Postgresql) (err error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	c.Postgresql = *newSpec
+	c.setSpec(newSpec)
 
 	defer func() {
 		if err != nil {

pkg/cluster/util.go

@@ -1,9 +1,12 @@
 package cluster
 
 import (
+	"bytes"
+	"encoding/gob"
 	"encoding/json"
 	"fmt"
 	"math/rand"
+	"sort"
 	"strings"
 	"time"
 
@@ -18,7 +21,6 @@ import (
 	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/retryutil"
-	"sort"
 )
 
 // OAuthTokenGetter provides the method for fetching OAuth tokens
@@ -386,3 +388,32 @@ func (c *Cluster) credentialSecretNameForCluster(username string, clusterName st
 func masterCandidate(replicas []spec.NamespacedName) spec.NamespacedName {
 	return replicas[rand.Intn(len(replicas))]
 }
+
+func cloneSpec(from *spec.Postgresql) (*spec.Postgresql, error) {
+	var (
+		buf    bytes.Buffer
+		result *spec.Postgresql
+		err    error
+	)
+	enc := gob.NewEncoder(&buf)
+	if err = enc.Encode(*from); err != nil {
+		return nil, fmt.Errorf("could not encode the spec: %v", err)
+	}
+	dec := gob.NewDecoder(&buf)
+	if err = dec.Decode(&result); err != nil {
+		return nil, fmt.Errorf("could not decode the spec: %v", err)
+	}
+	return result, nil
+}
+
+func (c *Cluster) setSpec(newSpec *spec.Postgresql) {
+	c.specMu.Lock()
+	c.Postgresql = *newSpec
+	c.specMu.Unlock()
+}
+
+func (c *Cluster) GetSpec() (*spec.Postgresql, error) {
+	c.specMu.RLock()
+	defer c.specMu.RUnlock()
+	return cloneSpec(&c.Postgresql)
+}