cluster history api endpoint

manifests/configmap.yaml

@@ -31,4 +31,5 @@ data:
   enable_load_balancer: "true"
   api_port: "8080"
   ring_log_lines: "100"
+  cluster_history_entries: "1000"
   pod_terminate_grace_period: 5m

pkg/apiserver/apiserver.go

@@ -32,6 +32,7 @@ type controllerInformer interface {
 	TeamClusterList() map[string][]spec.NamespacedName
 	ClusterStatus(team, cluster string) (*spec.ClusterStatus, error)
 	ClusterLogs(team, cluster string) ([]*spec.LogEntry, error)
+	ClusterHistory(team, cluster string) ([]*spec.Diff, error)
 	WorkerLogs(workerID uint32) ([]*spec.LogEntry, error)
 	ListQueue(workerID uint32) (*spec.QueueDump, error)
 	GetWorkersCnt() uint32
@@ -47,6 +48,7 @@ type Server struct {
 var (
 	clusterStatusURL     = regexp.MustCompile(`^/clusters/(?P<team>[a-zA-Z][a-zA-Z0-9]*)/(?P<cluster>[a-zA-Z][a-zA-Z0-9]*)/?$`)
 	clusterLogsURL       = regexp.MustCompile(`^/clusters/(?P<team>[a-zA-Z][a-zA-Z0-9]*)/(?P<cluster>[a-zA-Z][a-zA-Z0-9]*)/logs/?$`)
+	clusterHistoryURL    = regexp.MustCompile(`^/clusters/(?P<team>[a-zA-Z][a-zA-Z0-9]*)/(?P<cluster>[a-zA-Z][a-zA-Z0-9]*)/history/?$`)
 	teamURL              = regexp.MustCompile(`^/clusters/(?P<team>[a-zA-Z][a-zA-Z0-9]*)/?$`)
 	workerLogsURL        = regexp.MustCompile(`^/workers/(?P<id>\d+)/logs/?$`)
 	workerEventsQueueURL = regexp.MustCompile(`^/workers/(?P<id>\d+)/queue/?$`)
@@ -160,6 +162,8 @@ func (s *Server) clusters(w http.ResponseWriter, req *http.Request) {
 		return
 	} else if matches := util.FindNamedStringSubmatch(clusterLogsURL, req.URL.Path); matches != nil {
 		resp, err = s.controller.ClusterLogs(matches["team"], matches["cluster"])
+	} else if matches := util.FindNamedStringSubmatch(clusterHistoryURL, req.URL.Path); matches != nil {
+		resp, err = s.controller.ClusterHistory(matches["team"], matches["cluster"])
 	} else if req.URL.Path == clustersURL {
 		res := make(map[string][]string)
 		for team, clusters := range s.controller.TeamClusterList() {
@@ -171,8 +175,7 @@ func (s *Server) clusters(w http.ResponseWriter, req *http.Request) {
 		s.respond(res, nil, w)
 		return
 	} else {
-		s.respond(nil, fmt.Errorf("page not found"), w)
+		err = fmt.Errorf("page not found")
-		return
 	}
 
 	s.respond(resp, err, w)

pkg/controller/controller.go

@@ -32,10 +32,11 @@ type Controller struct {
 
 	stopCh chan struct{}
 
-	clustersMu   sync.RWMutex
+	clustersMu     sync.RWMutex
-	clusters     map[spec.NamespacedName]*cluster.Cluster
+	clusters       map[spec.NamespacedName]*cluster.Cluster
-	clusterLogs  map[spec.NamespacedName]ringlog.RingLogger
+	clusterLogs    map[spec.NamespacedName]ringlog.RingLogger
-	teamClusters map[string][]spec.NamespacedName
+	clusterHistory map[spec.NamespacedName]ringlog.RingLogger // history of the cluster changes
+	teamClusters   map[string][]spec.NamespacedName
 
 	postgresqlInformer cache.SharedIndexInformer
 	podInformer        cache.SharedIndexInformer
@@ -52,14 +53,15 @@ func NewController(controllerConfig *spec.ControllerConfig) *Controller {
 	logger := logrus.New()
 
 	c := &Controller{
-		config:       *controllerConfig,
+		config:         *controllerConfig,
-		opConfig:     &config.Config{},
+		opConfig:       &config.Config{},
-		logger:       logger.WithField("pkg", "controller"),
+		logger:         logger.WithField("pkg", "controller"),
-		clusters:     make(map[spec.NamespacedName]*cluster.Cluster),
+		clusters:       make(map[spec.NamespacedName]*cluster.Cluster),
-		clusterLogs:  make(map[spec.NamespacedName]ringlog.RingLogger),
+		clusterLogs:    make(map[spec.NamespacedName]ringlog.RingLogger),
-		teamClusters: make(map[string][]spec.NamespacedName),
+		clusterHistory: make(map[spec.NamespacedName]ringlog.RingLogger),
-		stopCh:       make(chan struct{}),
+		teamClusters:   make(map[string][]spec.NamespacedName),
-		podCh:        make(chan spec.PodEvent),
+		stopCh:         make(chan struct{}),
+		podCh:          make(chan spec.PodEvent),
 	}
 	logger.Hooks.Add(c)
 

pkg/controller/postgresql.go

@@ -138,12 +138,14 @@ func (c *Controller) addCluster(lg *logrus.Entry, clusterName spec.NamespacedNam
 	c.teamClusters[teamName] = append(c.teamClusters[teamName], clusterName)
 	c.clusters[clusterName] = cl
 	c.clusterLogs[clusterName] = ringlog.New(c.opConfig.RingLogLines)
+	c.clusterHistory[clusterName] = ringlog.New(c.opConfig.ClusterHistoryEntries)
 
 	return cl
 }
 
 func (c *Controller) processEvent(event spec.ClusterEvent) {
 	var clusterName spec.NamespacedName
+	var clHistory ringlog.RingLogger
 
 	lg := c.logger.WithField("worker", event.WorkerID)
 
@@ -156,6 +158,9 @@ func (c *Controller) processEvent(event spec.ClusterEvent) {
 
 	c.clustersMu.RLock()
 	cl, clusterFound := c.clusters[clusterName]
+	if clusterFound {
+		clHistory = c.clusterHistory[clusterName]
+	}
 	c.clustersMu.RUnlock()
 
 	switch event.EventType {
@@ -192,6 +197,12 @@ func (c *Controller) processEvent(event spec.ClusterEvent) {
 		}
 		cl.Error = nil
 		lg.Infoln("cluster has been updated")
+
+		clHistory.Insert(&spec.Diff{
+			EventTime:   event.EventTime,
+			ProcessTime: time.Now(),
+			Diff:        util.Diff(event.OldSpec, event.NewSpec),
+		})
 	case spec.EventDelete:
 		if !clusterFound {
 			lg.Errorf("unknown cluster: %q", clusterName)
@@ -211,6 +222,7 @@ func (c *Controller) processEvent(event spec.ClusterEvent) {
 
 			delete(c.clusters, clusterName)
 			delete(c.clusterLogs, clusterName)
+			delete(c.clusterHistory, clusterName)
 			for i, val := range c.teamClusters[teamName] {
 				if val == clusterName {
 					copy(c.teamClusters[teamName][i:], c.teamClusters[teamName][i+1:])

pkg/controller/status.go

@@ -166,3 +166,25 @@ func (c *Controller) ListQueue(workerID uint32) (*spec.QueueDump, error) {
 func (c *Controller) GetWorkersCnt() uint32 {
 	return c.opConfig.Workers
 }
+
+// ClusterHistory dumps history of cluster changes
+func (c *Controller) ClusterHistory(team, name string) ([]*spec.Diff, error) {
+	clusterName := spec.NamespacedName{
+		Namespace: c.opConfig.Namespace,
+		Name:      team + "-" + name,
+	}
+
+	c.clustersMu.RLock()
+	cl, ok := c.clusterHistory[clusterName]
+	c.clustersMu.RUnlock()
+	if !ok {
+		return nil, fmt.Errorf("could not find cluster")
+	}
+
+	res := make([]*spec.Diff, 0)
+	for _, e := range cl.Walk() {
+		res = append(res, e.(*spec.Diff))
+	}
+
+	return res, nil
+}

pkg/spec/types.go

@@ -101,6 +101,13 @@ type ClusterStatus struct {
 	Error  error
 }
 
+// Diff describes diff
+type Diff struct {
+	EventTime   time.Time
+	ProcessTime time.Time
+	Diff        []string
+}
+
 // ControllerStatus describes status of the controller
 type ControllerStatus struct {
 	LastSyncTime    int64

pkg/util/config/config.go

@@ -46,23 +46,24 @@ type Config struct {
 	TPR
 	Resources
 	Auth
-	Namespace            string         `name:"namespace"`
+	Namespace             string         `name:"namespace"`
-	EtcdHost             string         `name:"etcd_host" default:"etcd-client.default.svc.cluster.local:2379"`
+	EtcdHost              string         `name:"etcd_host" default:"etcd-client.default.svc.cluster.local:2379"`
-	DockerImage          string         `name:"docker_image" default:"registry.opensource.zalan.do/acid/spiloprivate-9.6:1.2-p4"`
+	DockerImage           string         `name:"docker_image" default:"registry.opensource.zalan.do/acid/spiloprivate-9.6:1.2-p4"`
-	ServiceAccountName   string         `name:"service_account_name" default:"operator"`
+	ServiceAccountName    string         `name:"service_account_name" default:"operator"`
-	DbHostedZone         string         `name:"db_hosted_zone" default:"db.example.com"`
+	DbHostedZone          string         `name:"db_hosted_zone" default:"db.example.com"`
-	EtcdScope            string         `name:"etcd_scope" default:"service"`
+	EtcdScope             string         `name:"etcd_scope" default:"service"`
-	WALES3Bucket         string         `name:"wal_s3_bucket"`
+	WALES3Bucket          string         `name:"wal_s3_bucket"`
-	KubeIAMRole          string         `name:"kube_iam_role"`
+	KubeIAMRole           string         `name:"kube_iam_role"`
-	DebugLogging         bool           `name:"debug_logging" default:"true"`
+	DebugLogging          bool           `name:"debug_logging" default:"true"`
-	EnableDBAccess       bool           `name:"enable_database_access" default:"true"`
+	EnableDBAccess        bool           `name:"enable_database_access" default:"true"`
-	EnableTeamsAPI       bool           `name:"enable_teams_api" default:"true"`
+	EnableTeamsAPI        bool           `name:"enable_teams_api" default:"true"`
-	EnableLoadBalancer   bool           `name:"enable_load_balancer" default:"true"`
+	EnableLoadBalancer    bool           `name:"enable_load_balancer" default:"true"`
-	MasterDNSNameFormat  stringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"`
+	MasterDNSNameFormat   stringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"`
-	ReplicaDNSNameFormat stringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"`
+	ReplicaDNSNameFormat  stringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"`
-	Workers              uint32         `name:"workers" default:"4"`
+	Workers               uint32         `name:"workers" default:"4"`
-	APIPort              int            `name:"api_port" default:"8080"`
+	APIPort               int            `name:"api_port" default:"8080"`
-	RingLogLines         int            `name:"ring_log_lines" default:"100"`
+	RingLogLines          int            `name:"ring_log_lines" default:"100"`
+	ClusterHistoryEntries int            `name:"cluster_history_entries" default:"1000"`
 
 	PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"`
 }