WIP: Hold the proper locks while examining the list of databases.

Introduce a new lock called specMu lock to protect the cluster spec.
This lock is held on update and sync, and when retrieving the spec in
the API code. There is no need to acquire it for cluster creation and
deletion: creation assigns the spec to the cluster before linking it to
the controller, and deletion just removes the cluster from the list in
the controller, both holding the global clustersMu Lock.

pkg/controller/status.go

@@ -33,13 +33,21 @@ func (c *Controller) ClusterStatus(team, cluster string) (*spec.ClusterStatus, e
 }
 
 // GetClusterDatabasesMap returns for each cluster the list of databases running there
-func (c *Controller) GetClusterDatabasesMap() map[string][]string {
+func (c *Controller) ClusterDatabasesMap() map[string][]string {
 
 	m := make(map[string][]string)
 
+	// avoid modifying the cluster list while we are fetching each one of them.
+	c.clustersMu.RLock()
+	defer c.clustersMu.RUnlock()
 	for _, cluster := range c.clusters {
-		for database := range cluster.Postgresql.Spec.Databases {
-			m[cluster.Name] = append(m[cluster.Name], database)
+		// GetSpec holds the specMu lock of a cluster
+		if spec, err := cluster.GetSpec(); err == nil {
+			for database := range spec.Spec.Databases {
+				m[cluster.Name] = append(m[cluster.Name], database)
+			}
+		} else {
+			c.logger.Warningf("could not get the list of databases for cluster %q: %v", cluster.Name, err)
 		}
 	}