Add Additional Autoscaling Metrics to Prometheus (#1720)

* Add prometheus metrics for autoscaling

* Add desc for prometheus-metrics

* FIX: Typo

* Remove replicas_desired_before in metrics

* Remove Num prefix in metricws

controllers/autoscaling.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	prometheus_metrics "github.com/actions-runner-controller/actions-runner-controller/controllers/metrics"
 	arcgithub "github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/google/go-github/v45/github"
 	corev1 "k8s.io/api/core/v1"
@@ -211,6 +212,20 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 
 	necessaryReplicas := queued + inProgress
 
+	prometheus_metrics.SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns(
+		hra.ObjectMeta,
+		st.enterprise,
+		st.org,
+		st.repo,
+		st.kind,
+		st.st,
+		necessaryReplicas,
+		completed,
+		inProgress,
+		queued,
+		unknown,
+	)
+
 	r.Log.V(1).Info(
 		fmt.Sprintf("Suggested desired replicas of %d by TotalNumberOfQueuedAndInProgressWorkflowRuns", necessaryReplicas),
 		"workflow_runs_completed", completed,
@@ -382,6 +397,19 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 	//
 	// - num_runners can be as twice as large as replicas_desired_before while
 	//   the runnerdeployment controller is replacing RunnerReplicaSet for runner update.
+	prometheus_metrics.SetHorizontalRunnerAutoscalerPercentageRunnersBusy(
+		hra.ObjectMeta,
+		st.enterprise,
+		st.org,
+		st.repo,
+		st.kind,
+		st.st,
+		desiredReplicas,
+		numRunners,
+		numRunnersRegistered,
+		numRunnersBusy,
+		numTerminatingBusy,
+	)
 
 	r.Log.V(1).Info(
 		fmt.Sprintf("Suggested desired replicas of %d by PercentageRunnersBusy", desiredReplicas),

controllers/metrics/horizontalrunnerautoscaler.go

@@ -7,8 +7,13 @@ import (
 )
 
 const (
-	hraName      = "horizontalrunnerautoscaler"
+	hraName        = "horizontalrunnerautoscaler"
-	hraNamespace = "namespace"
+	hraNamespace   = "namespace"
+	stEnterprise   = "enterprise"
+	stOrganization = "organization"
+	stRepository   = "repository"
+	stKind         = "kind"
+	stName         = "name"
 )
 
 var (
@@ -16,6 +21,16 @@ var (
 		horizontalRunnerAutoscalerMinReplicas,
 		horizontalRunnerAutoscalerMaxReplicas,
 		horizontalRunnerAutoscalerDesiredReplicas,
+		horizontalRunnerAutoscalerReplicasDesired,
+		horizontalRunnerAutoscalerRunners,
+		horizontalRunnerAutoscalerRunnersRegistered,
+		horizontalRunnerAutoscalerRunnersBusy,
+		horizontalRunnerAutoscalerTerminatingBusy,
+		horizontalRunnerAutoscalerNecessaryReplicas,
+		horizontalRunnerAutoscalerWorkflowRunsCompleted,
+		horizontalRunnerAutoscalerWorkflowRunsInProgress,
+		horizontalRunnerAutoscalerWorkflowRunsQueued,
+		horizontalRunnerAutoscalerWorkflowRunsUnknown,
 	}
 )
 
@@ -41,6 +56,78 @@ var (
 		},
 		[]string{hraName, hraNamespace},
 	)
+	// PercentageRunnersBusy
+	horizontalRunnerAutoscalerReplicasDesired = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_replicas_desired",
+			Help: "replicas_desired of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerRunners = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_runners",
+			Help: "num_runners of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerRunnersRegistered = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_runners_registered",
+			Help: "num_runners_registered of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerRunnersBusy = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_runners_busy",
+			Help: "num_runners_busy of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerTerminatingBusy = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_terminating_busy",
+			Help: "num_terminating_busy of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	// QueuedAndInProgressWorkflowRuns
+	horizontalRunnerAutoscalerNecessaryReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_necessary_replicas",
+			Help: "necessary_replicas of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsCompleted = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_completed",
+			Help: "workflow_runs_completed of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsInProgress = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_in_progress",
+			Help: "workflow_runs_in_progress of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsQueued = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_queued",
+			Help: "workflow_runs_queued of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsUnknown = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_unknown",
+			Help: "workflow_runs_unknown of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
 )
 
 func SetHorizontalRunnerAutoscalerSpec(o metav1.ObjectMeta, spec v1alpha1.HorizontalRunnerAutoscalerSpec) {
@@ -65,3 +152,61 @@ func SetHorizontalRunnerAutoscalerStatus(o metav1.ObjectMeta, status v1alpha1.Ho
 		horizontalRunnerAutoscalerDesiredReplicas.With(labels).Set(float64(*status.DesiredReplicas))
 	}
 }
+
+func SetHorizontalRunnerAutoscalerPercentageRunnersBusy(
+	o metav1.ObjectMeta,
+	enterprise string,
+	organization string,
+	repository string,
+	kind string,
+	name string,
+	desiredReplicas int,
+	numRunners int,
+	numRunnersRegistered int,
+	numRunnersBusy int,
+	numTerminatingBusy int,
+) {
+	labels := prometheus.Labels{
+		hraName:        o.Name,
+		hraNamespace:   o.Namespace,
+		stEnterprise:   enterprise,
+		stOrganization: organization,
+		stRepository:   repository,
+		stKind:         kind,
+		stName:         name,
+	}
+	horizontalRunnerAutoscalerReplicasDesired.With(labels).Set(float64(desiredReplicas))
+	horizontalRunnerAutoscalerRunners.With(labels).Set(float64(numRunners))
+	horizontalRunnerAutoscalerRunnersRegistered.With(labels).Set(float64(numRunnersRegistered))
+	horizontalRunnerAutoscalerRunnersBusy.With(labels).Set(float64(numRunnersBusy))
+	horizontalRunnerAutoscalerTerminatingBusy.With(labels).Set(float64(numTerminatingBusy))
+}
+
+func SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns(
+	o metav1.ObjectMeta,
+	enterprise string,
+	organization string,
+	repository string,
+	kind string,
+	name string,
+	necessaryReplicas int,
+	workflowRunsCompleted int,
+	workflowRunsInProgress int,
+	workflowRunsQueued int,
+	workflowRunsUnknown int,
+) {
+	labels := prometheus.Labels{
+		hraName:        o.Name,
+		hraNamespace:   o.Namespace,
+		stEnterprise:   enterprise,
+		stOrganization: organization,
+		stRepository:   repository,
+		stKind:         kind,
+		stName:         name,
+	}
+	horizontalRunnerAutoscalerNecessaryReplicas.With(labels).Set(float64(necessaryReplicas))
+	horizontalRunnerAutoscalerWorkflowRunsCompleted.With(labels).Set(float64(workflowRunsCompleted))
+	horizontalRunnerAutoscalerWorkflowRunsInProgress.With(labels).Set(float64(workflowRunsInProgress))
+	horizontalRunnerAutoscalerWorkflowRunsQueued.With(labels).Set(float64(workflowRunsQueued))
+	horizontalRunnerAutoscalerWorkflowRunsUnknown.With(labels).Set(float64(workflowRunsUnknown))
+}