diff --git a/controllers/autoscaling.go b/controllers/autoscaling.go index d7c2fa29..7534625f 100644 --- a/controllers/autoscaling.go +++ b/controllers/autoscaling.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1" + prometheus_metrics "github.com/actions-runner-controller/actions-runner-controller/controllers/metrics" arcgithub "github.com/actions-runner-controller/actions-runner-controller/github" "github.com/google/go-github/v45/github" corev1 "k8s.io/api/core/v1" @@ -211,6 +212,20 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr necessaryReplicas := queued + inProgress + prometheus_metrics.SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns( + hra.ObjectMeta, + st.enterprise, + st.org, + st.repo, + st.kind, + st.st, + necessaryReplicas, + completed, + inProgress, + queued, + unknown, + ) + r.Log.V(1).Info( fmt.Sprintf("Suggested desired replicas of %d by TotalNumberOfQueuedAndInProgressWorkflowRuns", necessaryReplicas), "workflow_runs_completed", completed, @@ -382,6 +397,19 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner // // - num_runners can be as twice as large as replicas_desired_before while // the runnerdeployment controller is replacing RunnerReplicaSet for runner update. + prometheus_metrics.SetHorizontalRunnerAutoscalerPercentageRunnersBusy( + hra.ObjectMeta, + st.enterprise, + st.org, + st.repo, + st.kind, + st.st, + desiredReplicas, + numRunners, + numRunnersRegistered, + numRunnersBusy, + numTerminatingBusy, + ) r.Log.V(1).Info( fmt.Sprintf("Suggested desired replicas of %d by PercentageRunnersBusy", desiredReplicas), diff --git a/controllers/metrics/horizontalrunnerautoscaler.go b/controllers/metrics/horizontalrunnerautoscaler.go index 0815e2a5..c3bfe97c 100644 --- a/controllers/metrics/horizontalrunnerautoscaler.go +++ b/controllers/metrics/horizontalrunnerautoscaler.go @@ -7,8 +7,13 @@ import ( ) const ( - hraName = "horizontalrunnerautoscaler" - hraNamespace = "namespace" + hraName = "horizontalrunnerautoscaler" + hraNamespace = "namespace" + stEnterprise = "enterprise" + stOrganization = "organization" + stRepository = "repository" + stKind = "kind" + stName = "name" ) var ( @@ -16,6 +21,16 @@ var ( horizontalRunnerAutoscalerMinReplicas, horizontalRunnerAutoscalerMaxReplicas, horizontalRunnerAutoscalerDesiredReplicas, + horizontalRunnerAutoscalerReplicasDesired, + horizontalRunnerAutoscalerRunners, + horizontalRunnerAutoscalerRunnersRegistered, + horizontalRunnerAutoscalerRunnersBusy, + horizontalRunnerAutoscalerTerminatingBusy, + horizontalRunnerAutoscalerNecessaryReplicas, + horizontalRunnerAutoscalerWorkflowRunsCompleted, + horizontalRunnerAutoscalerWorkflowRunsInProgress, + horizontalRunnerAutoscalerWorkflowRunsQueued, + horizontalRunnerAutoscalerWorkflowRunsUnknown, } ) @@ -41,6 +56,78 @@ var ( }, []string{hraName, hraNamespace}, ) + // PercentageRunnersBusy + horizontalRunnerAutoscalerReplicasDesired = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_replicas_desired", + Help: "replicas_desired of PercentageRunnersBusy", + }, + []string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName}, + ) + horizontalRunnerAutoscalerRunners = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_runners", + Help: "num_runners of PercentageRunnersBusy", + }, + []string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName}, + ) + horizontalRunnerAutoscalerRunnersRegistered = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_runners_registered", + Help: "num_runners_registered of PercentageRunnersBusy", + }, + []string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName}, + ) + horizontalRunnerAutoscalerRunnersBusy = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_runners_busy", + Help: "num_runners_busy of PercentageRunnersBusy", + }, + []string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName}, + ) + horizontalRunnerAutoscalerTerminatingBusy = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_terminating_busy", + Help: "num_terminating_busy of PercentageRunnersBusy", + }, + []string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName}, + ) + // QueuedAndInProgressWorkflowRuns + horizontalRunnerAutoscalerNecessaryReplicas = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_necessary_replicas", + Help: "necessary_replicas of QueuedAndInProgressWorkflowRuns", + }, + []string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName}, + ) + horizontalRunnerAutoscalerWorkflowRunsCompleted = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_workflow_runs_completed", + Help: "workflow_runs_completed of QueuedAndInProgressWorkflowRuns", + }, + []string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName}, + ) + horizontalRunnerAutoscalerWorkflowRunsInProgress = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_workflow_runs_in_progress", + Help: "workflow_runs_in_progress of QueuedAndInProgressWorkflowRuns", + }, + []string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName}, + ) + horizontalRunnerAutoscalerWorkflowRunsQueued = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_workflow_runs_queued", + Help: "workflow_runs_queued of QueuedAndInProgressWorkflowRuns", + }, + []string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName}, + ) + horizontalRunnerAutoscalerWorkflowRunsUnknown = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontalrunnerautoscaler_workflow_runs_unknown", + Help: "workflow_runs_unknown of QueuedAndInProgressWorkflowRuns", + }, + []string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName}, + ) ) func SetHorizontalRunnerAutoscalerSpec(o metav1.ObjectMeta, spec v1alpha1.HorizontalRunnerAutoscalerSpec) { @@ -65,3 +152,61 @@ func SetHorizontalRunnerAutoscalerStatus(o metav1.ObjectMeta, status v1alpha1.Ho horizontalRunnerAutoscalerDesiredReplicas.With(labels).Set(float64(*status.DesiredReplicas)) } } + +func SetHorizontalRunnerAutoscalerPercentageRunnersBusy( + o metav1.ObjectMeta, + enterprise string, + organization string, + repository string, + kind string, + name string, + desiredReplicas int, + numRunners int, + numRunnersRegistered int, + numRunnersBusy int, + numTerminatingBusy int, +) { + labels := prometheus.Labels{ + hraName: o.Name, + hraNamespace: o.Namespace, + stEnterprise: enterprise, + stOrganization: organization, + stRepository: repository, + stKind: kind, + stName: name, + } + horizontalRunnerAutoscalerReplicasDesired.With(labels).Set(float64(desiredReplicas)) + horizontalRunnerAutoscalerRunners.With(labels).Set(float64(numRunners)) + horizontalRunnerAutoscalerRunnersRegistered.With(labels).Set(float64(numRunnersRegistered)) + horizontalRunnerAutoscalerRunnersBusy.With(labels).Set(float64(numRunnersBusy)) + horizontalRunnerAutoscalerTerminatingBusy.With(labels).Set(float64(numTerminatingBusy)) +} + +func SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns( + o metav1.ObjectMeta, + enterprise string, + organization string, + repository string, + kind string, + name string, + necessaryReplicas int, + workflowRunsCompleted int, + workflowRunsInProgress int, + workflowRunsQueued int, + workflowRunsUnknown int, +) { + labels := prometheus.Labels{ + hraName: o.Name, + hraNamespace: o.Namespace, + stEnterprise: enterprise, + stOrganization: organization, + stRepository: repository, + stKind: kind, + stName: name, + } + horizontalRunnerAutoscalerNecessaryReplicas.With(labels).Set(float64(necessaryReplicas)) + horizontalRunnerAutoscalerWorkflowRunsCompleted.With(labels).Set(float64(workflowRunsCompleted)) + horizontalRunnerAutoscalerWorkflowRunsInProgress.With(labels).Set(float64(workflowRunsInProgress)) + horizontalRunnerAutoscalerWorkflowRunsQueued.With(labels).Set(float64(workflowRunsQueued)) + horizontalRunnerAutoscalerWorkflowRunsUnknown.With(labels).Set(float64(workflowRunsUnknown)) +}