Add Additional Autoscaling Metrics to Prometheus (#1720)
* Add prometheus metrics for autoscaling * Add desc for prometheus-metrics * FIX: Typo * Remove replicas_desired_before in metrics * Remove Num prefix in metricws
This commit is contained in:
Rahul Kumar
GitHub
parent
d439ed5c81
commit
72ca998266
|
|
@ -9,6 +9,7 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
|
||||
prometheus_metrics "github.com/actions-runner-controller/actions-runner-controller/controllers/metrics"
|
||||
arcgithub "github.com/actions-runner-controller/actions-runner-controller/github"
|
||||
"github.com/google/go-github/v45/github"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
|
|
@ -211,6 +212,20 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
|
|||
|
||||
necessaryReplicas := queued + inProgress
|
||||
|
||||
prometheus_metrics.SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns(
|
||||
hra.ObjectMeta,
|
||||
st.enterprise,
|
||||
st.org,
|
||||
st.repo,
|
||||
st.kind,
|
||||
st.st,
|
||||
necessaryReplicas,
|
||||
completed,
|
||||
inProgress,
|
||||
queued,
|
||||
unknown,
|
||||
)
|
||||
|
||||
r.Log.V(1).Info(
|
||||
fmt.Sprintf("Suggested desired replicas of %d by TotalNumberOfQueuedAndInProgressWorkflowRuns", necessaryReplicas),
|
||||
"workflow_runs_completed", completed,
|
||||
|
|
@ -382,6 +397,19 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
|
|||
//
|
||||
// - num_runners can be as twice as large as replicas_desired_before while
|
||||
// the runnerdeployment controller is replacing RunnerReplicaSet for runner update.
|
||||
prometheus_metrics.SetHorizontalRunnerAutoscalerPercentageRunnersBusy(
|
||||
hra.ObjectMeta,
|
||||
st.enterprise,
|
||||
st.org,
|
||||
st.repo,
|
||||
st.kind,
|
||||
st.st,
|
||||
desiredReplicas,
|
||||
numRunners,
|
||||
numRunnersRegistered,
|
||||
numRunnersBusy,
|
||||
numTerminatingBusy,
|
||||
)
|
||||
|
||||
r.Log.V(1).Info(
|
||||
fmt.Sprintf("Suggested desired replicas of %d by PercentageRunnersBusy", desiredReplicas),
|
||||
|
|
|
|||
|
|
@ -7,8 +7,13 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
hraName = "horizontalrunnerautoscaler"
|
||||
hraNamespace = "namespace"
|
||||
hraName = "horizontalrunnerautoscaler"
|
||||
hraNamespace = "namespace"
|
||||
stEnterprise = "enterprise"
|
||||
stOrganization = "organization"
|
||||
stRepository = "repository"
|
||||
stKind = "kind"
|
||||
stName = "name"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -16,6 +21,16 @@ var (
|
|||
horizontalRunnerAutoscalerMinReplicas,
|
||||
horizontalRunnerAutoscalerMaxReplicas,
|
||||
horizontalRunnerAutoscalerDesiredReplicas,
|
||||
horizontalRunnerAutoscalerReplicasDesired,
|
||||
horizontalRunnerAutoscalerRunners,
|
||||
horizontalRunnerAutoscalerRunnersRegistered,
|
||||
horizontalRunnerAutoscalerRunnersBusy,
|
||||
horizontalRunnerAutoscalerTerminatingBusy,
|
||||
horizontalRunnerAutoscalerNecessaryReplicas,
|
||||
horizontalRunnerAutoscalerWorkflowRunsCompleted,
|
||||
horizontalRunnerAutoscalerWorkflowRunsInProgress,
|
||||
horizontalRunnerAutoscalerWorkflowRunsQueued,
|
||||
horizontalRunnerAutoscalerWorkflowRunsUnknown,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -41,6 +56,78 @@ var (
|
|||
},
|
||||
[]string{hraName, hraNamespace},
|
||||
)
|
||||
// PercentageRunnersBusy
|
||||
horizontalRunnerAutoscalerReplicasDesired = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "horizontalrunnerautoscaler_replicas_desired",
|
||||
Help: "replicas_desired of PercentageRunnersBusy",
|
||||
},
|
||||
[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
|
||||
)
|
||||
horizontalRunnerAutoscalerRunners = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "horizontalrunnerautoscaler_runners",
|
||||
Help: "num_runners of PercentageRunnersBusy",
|
||||
},
|
||||
[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
|
||||
)
|
||||
horizontalRunnerAutoscalerRunnersRegistered = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "horizontalrunnerautoscaler_runners_registered",
|
||||
Help: "num_runners_registered of PercentageRunnersBusy",
|
||||
},
|
||||
[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
|
||||
)
|
||||
horizontalRunnerAutoscalerRunnersBusy = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "horizontalrunnerautoscaler_runners_busy",
|
||||
Help: "num_runners_busy of PercentageRunnersBusy",
|
||||
},
|
||||
[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
|
||||
)
|
||||
horizontalRunnerAutoscalerTerminatingBusy = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "horizontalrunnerautoscaler_terminating_busy",
|
||||
Help: "num_terminating_busy of PercentageRunnersBusy",
|
||||
},
|
||||
[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
|
||||
)
|
||||
// QueuedAndInProgressWorkflowRuns
|
||||
horizontalRunnerAutoscalerNecessaryReplicas = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "horizontalrunnerautoscaler_necessary_replicas",
|
||||
Help: "necessary_replicas of QueuedAndInProgressWorkflowRuns",
|
||||
},
|
||||
[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
|
||||
)
|
||||
horizontalRunnerAutoscalerWorkflowRunsCompleted = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "horizontalrunnerautoscaler_workflow_runs_completed",
|
||||
Help: "workflow_runs_completed of QueuedAndInProgressWorkflowRuns",
|
||||
},
|
||||
[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
|
||||
)
|
||||
horizontalRunnerAutoscalerWorkflowRunsInProgress = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "horizontalrunnerautoscaler_workflow_runs_in_progress",
|
||||
Help: "workflow_runs_in_progress of QueuedAndInProgressWorkflowRuns",
|
||||
},
|
||||
[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
|
||||
)
|
||||
horizontalRunnerAutoscalerWorkflowRunsQueued = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "horizontalrunnerautoscaler_workflow_runs_queued",
|
||||
Help: "workflow_runs_queued of QueuedAndInProgressWorkflowRuns",
|
||||
},
|
||||
[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
|
||||
)
|
||||
horizontalRunnerAutoscalerWorkflowRunsUnknown = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "horizontalrunnerautoscaler_workflow_runs_unknown",
|
||||
Help: "workflow_runs_unknown of QueuedAndInProgressWorkflowRuns",
|
||||
},
|
||||
[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
|
||||
)
|
||||
)
|
||||
|
||||
func SetHorizontalRunnerAutoscalerSpec(o metav1.ObjectMeta, spec v1alpha1.HorizontalRunnerAutoscalerSpec) {
|
||||
|
|
@ -65,3 +152,61 @@ func SetHorizontalRunnerAutoscalerStatus(o metav1.ObjectMeta, status v1alpha1.Ho
|
|||
horizontalRunnerAutoscalerDesiredReplicas.With(labels).Set(float64(*status.DesiredReplicas))
|
||||
}
|
||||
}
|
||||
|
||||
func SetHorizontalRunnerAutoscalerPercentageRunnersBusy(
|
||||
o metav1.ObjectMeta,
|
||||
enterprise string,
|
||||
organization string,
|
||||
repository string,
|
||||
kind string,
|
||||
name string,
|
||||
desiredReplicas int,
|
||||
numRunners int,
|
||||
numRunnersRegistered int,
|
||||
numRunnersBusy int,
|
||||
numTerminatingBusy int,
|
||||
) {
|
||||
labels := prometheus.Labels{
|
||||
hraName: o.Name,
|
||||
hraNamespace: o.Namespace,
|
||||
stEnterprise: enterprise,
|
||||
stOrganization: organization,
|
||||
stRepository: repository,
|
||||
stKind: kind,
|
||||
stName: name,
|
||||
}
|
||||
horizontalRunnerAutoscalerReplicasDesired.With(labels).Set(float64(desiredReplicas))
|
||||
horizontalRunnerAutoscalerRunners.With(labels).Set(float64(numRunners))
|
||||
horizontalRunnerAutoscalerRunnersRegistered.With(labels).Set(float64(numRunnersRegistered))
|
||||
horizontalRunnerAutoscalerRunnersBusy.With(labels).Set(float64(numRunnersBusy))
|
||||
horizontalRunnerAutoscalerTerminatingBusy.With(labels).Set(float64(numTerminatingBusy))
|
||||
}
|
||||
|
||||
func SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns(
|
||||
o metav1.ObjectMeta,
|
||||
enterprise string,
|
||||
organization string,
|
||||
repository string,
|
||||
kind string,
|
||||
name string,
|
||||
necessaryReplicas int,
|
||||
workflowRunsCompleted int,
|
||||
workflowRunsInProgress int,
|
||||
workflowRunsQueued int,
|
||||
workflowRunsUnknown int,
|
||||
) {
|
||||
labels := prometheus.Labels{
|
||||
hraName: o.Name,
|
||||
hraNamespace: o.Namespace,
|
||||
stEnterprise: enterprise,
|
||||
stOrganization: organization,
|
||||
stRepository: repository,
|
||||
stKind: kind,
|
||||
stName: name,
|
||||
}
|
||||
horizontalRunnerAutoscalerNecessaryReplicas.With(labels).Set(float64(necessaryReplicas))
|
||||
horizontalRunnerAutoscalerWorkflowRunsCompleted.With(labels).Set(float64(workflowRunsCompleted))
|
||||
horizontalRunnerAutoscalerWorkflowRunsInProgress.With(labels).Set(float64(workflowRunsInProgress))
|
||||
horizontalRunnerAutoscalerWorkflowRunsQueued.With(labels).Set(float64(workflowRunsQueued))
|
||||
horizontalRunnerAutoscalerWorkflowRunsUnknown.With(labels).Set(float64(workflowRunsUnknown))
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue