From 3a0332dfdc56118c09c55b3610daee2ad9706c5d Mon Sep 17 00:00:00 2001 From: Hidetake Iwata Date: Fri, 19 Mar 2021 16:14:02 +0900 Subject: [PATCH] Add metrics of RunnerDeployment and HRA (#408) * Add metrics of RunnerDeployment and HRA * Use kube-state-metrics-style label names --- .../horizontalrunnerautoscaler_controller.go | 5 ++ .../metrics/horizontalrunnerautoscaler.go | 67 +++++++++++++++++++ controllers/metrics/metrics.go | 14 ++++ controllers/metrics/runnerdeployment.go | 37 ++++++++++ controllers/runnerdeployment_controller.go | 3 + 5 files changed, 126 insertions(+) create mode 100644 controllers/metrics/horizontalrunnerautoscaler.go create mode 100644 controllers/metrics/metrics.go create mode 100644 controllers/metrics/runnerdeployment.go diff --git a/controllers/horizontalrunnerautoscaler_controller.go b/controllers/horizontalrunnerautoscaler_controller.go index a64b83b1..e2c2f548 100644 --- a/controllers/horizontalrunnerautoscaler_controller.go +++ b/controllers/horizontalrunnerautoscaler_controller.go @@ -34,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/summerwind/actions-runner-controller/api/v1alpha1" + "github.com/summerwind/actions-runner-controller/controllers/metrics" ) const ( @@ -73,6 +74,8 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl return ctrl.Result{}, nil } + metrics.SetHorizontalRunnerAutoscalerSpec(hra.ObjectMeta, hra.Spec) + var rd v1alpha1.RunnerDeployment if err := r.Get(ctx, types.NamespacedName{ Namespace: req.Namespace, @@ -145,6 +148,8 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl } if updated != nil { + metrics.SetHorizontalRunnerAutoscalerStatus(updated.ObjectMeta, updated.Status) + if err := r.Status().Patch(ctx, updated, client.MergeFrom(&hra)); err != nil { return ctrl.Result{}, fmt.Errorf("patching horizontalrunnerautoscaler status to add cache entry: %w", err) } diff --git a/controllers/metrics/horizontalrunnerautoscaler.go b/controllers/metrics/horizontalrunnerautoscaler.go new file mode 100644 index 00000000..338de194 --- /dev/null +++ b/controllers/metrics/horizontalrunnerautoscaler.go @@ -0,0 +1,67 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/summerwind/actions-runner-controller/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + hraName = "horizontalrunnerautoscaler" + hraNamespace = "namespace" +) + +var ( + horizontalRunnerAutoscalerMetrics = []prometheus.Collector{ + horizontalRunnerAutoscalerMinReplicas, + horizontalRunnerAutoscalerMaxReplicas, + horizontalRunnerAutoscalerDesiredReplicas, + } +) + +var ( + horizontalRunnerAutoscalerMinReplicas = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_spec_min_replicas", + Help: "minReplicas of HorizontalRunnerAutoscaler", + }, + []string{hraName, hraNamespace}, + ) + horizontalRunnerAutoscalerMaxReplicas = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_spec_max_replicas", + Help: "maxReplicas of HorizontalRunnerAutoscaler", + }, + []string{hraName, hraNamespace}, + ) + horizontalRunnerAutoscalerDesiredReplicas = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_status_desired_replicas", + Help: "desiredReplicas of HorizontalRunnerAutoscaler", + }, + []string{hraName, hraNamespace}, + ) +) + +func SetHorizontalRunnerAutoscalerSpec(o metav1.ObjectMeta, spec v1alpha1.HorizontalRunnerAutoscalerSpec) { + labels := prometheus.Labels{ + hraName: o.Name, + hraNamespace: o.Namespace, + } + if spec.MaxReplicas != nil { + horizontalRunnerAutoscalerMaxReplicas.With(labels).Set(float64(*spec.MaxReplicas)) + } + if spec.MinReplicas != nil { + horizontalRunnerAutoscalerMinReplicas.With(labels).Set(float64(*spec.MinReplicas)) + } +} + +func SetHorizontalRunnerAutoscalerStatus(o metav1.ObjectMeta, status v1alpha1.HorizontalRunnerAutoscalerStatus) { + labels := prometheus.Labels{ + hraName: o.Name, + hraNamespace: o.Namespace, + } + if status.DesiredReplicas != nil { + horizontalRunnerAutoscalerDesiredReplicas.With(labels).Set(float64(*status.DesiredReplicas)) + } +} diff --git a/controllers/metrics/metrics.go b/controllers/metrics/metrics.go new file mode 100644 index 00000000..113df6e0 --- /dev/null +++ b/controllers/metrics/metrics.go @@ -0,0 +1,14 @@ +// Package metrics provides the metrics of custom resources such as HRA. +// +// This depends on the metrics exporter of kubebuilder. +// See https://book.kubebuilder.io/reference/metrics.html for details. +package metrics + +import ( + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +func init() { + metrics.Registry.MustRegister(runnerDeploymentMetrics...) + metrics.Registry.MustRegister(horizontalRunnerAutoscalerMetrics...) +} diff --git a/controllers/metrics/runnerdeployment.go b/controllers/metrics/runnerdeployment.go new file mode 100644 index 00000000..c062a1cc --- /dev/null +++ b/controllers/metrics/runnerdeployment.go @@ -0,0 +1,37 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/summerwind/actions-runner-controller/api/v1alpha1" +) + +const ( + rdName = "runnerdeployment" + rdNamespace = "namespace" +) + +var ( + runnerDeploymentMetrics = []prometheus.Collector{ + runnerDeploymentReplicas, + } +) + +var ( + runnerDeploymentReplicas = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "runnerdeployment_spec_replicas", + Help: "replicas of RunnerDeployment", + }, + []string{rdName, rdNamespace}, + ) +) + +func SetRunnerDeployment(rd v1alpha1.RunnerDeployment) { + labels := prometheus.Labels{ + rdName: rd.Name, + rdNamespace: rd.Namespace, + } + if rd.Spec.Replicas != nil { + runnerDeploymentReplicas.With(labels).Set(float64(*rd.Spec.Replicas)) + } +} diff --git a/controllers/runnerdeployment_controller.go b/controllers/runnerdeployment_controller.go index ed5d84e0..04a1afb7 100644 --- a/controllers/runnerdeployment_controller.go +++ b/controllers/runnerdeployment_controller.go @@ -38,6 +38,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/summerwind/actions-runner-controller/api/v1alpha1" + "github.com/summerwind/actions-runner-controller/controllers/metrics" ) const ( @@ -77,6 +78,8 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e return ctrl.Result{}, nil } + metrics.SetRunnerDeployment(rd) + var myRunnerReplicaSetList v1alpha1.RunnerReplicaSetList if err := r.List(ctx, &myRunnerReplicaSetList, client.InNamespace(req.Namespace), client.MatchingFields{runnerSetOwnerKey: req.Name}); err != nil { return ctrl.Result{}, err