mirror of
https://github.com/actions-runner-controller/actions-runner-controller.git
synced 2026-10-18 16:06:50 +02:00
Use metrics to display runner statuses instead of status field for EphemeralRunnerSet and AutoscalingRunnerSet
This commit is contained in:
Nikola Jokic
@@ -0,0 +1,115 @@
|
||||
package metrics_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
v1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
|
||||
actionsgithubcom "github.com/actions/actions-runner-controller/controllers/actions.github.com"
|
||||
githubmetrics "github.com/actions/actions-runner-controller/controllers/actions.github.com/metrics"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
controllerMetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
|
||||
)
|
||||
|
||||
var registerMetricsOnce sync.Once
|
||||
|
||||
func TestEphemeralRunnerLifecycleBucketsSetPrometheusMetrics(t *testing.T) {
|
||||
registerMetricsOnce.Do(githubmetrics.RegisterMetrics)
|
||||
|
||||
now := metav1.Now()
|
||||
runners := []v1alpha1.EphemeralRunner{
|
||||
{Status: v1alpha1.EphemeralRunnerStatus{Phase: v1alpha1.EphemeralRunnerPhasePending}},
|
||||
{Status: v1alpha1.EphemeralRunnerStatus{Phase: v1alpha1.EphemeralRunnerPhaseRunning}},
|
||||
{Status: v1alpha1.EphemeralRunnerStatus{Phase: v1alpha1.EphemeralRunnerPhaseRunning}},
|
||||
{Status: v1alpha1.EphemeralRunnerStatus{Phase: v1alpha1.EphemeralRunnerPhaseSucceeded}},
|
||||
{Status: v1alpha1.EphemeralRunnerStatus{Phase: v1alpha1.EphemeralRunnerPhaseFailed}},
|
||||
{Status: v1alpha1.EphemeralRunnerStatus{Phase: v1alpha1.EphemeralRunnerPhaseOutdated}},
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{DeletionTimestamp: &now},
|
||||
Status: v1alpha1.EphemeralRunnerStatus{Phase: v1alpha1.EphemeralRunnerPhaseRunning},
|
||||
},
|
||||
}
|
||||
|
||||
buckets := actionsgithubcom.AggregateEphemeralRunnerLifecycle(runners)
|
||||
labels := githubmetrics.CommonLabels{
|
||||
Name: "lifecycle-metrics-test",
|
||||
Namespace: "default",
|
||||
Repository: "test/repo",
|
||||
Organization: "test-org",
|
||||
Enterprise: "test-enterprise",
|
||||
}
|
||||
|
||||
githubmetrics.SetEphemeralRunnerCountsByLifecycle(
|
||||
labels,
|
||||
buckets.Pending,
|
||||
buckets.Running,
|
||||
buckets.Succeeded,
|
||||
buckets.Failed,
|
||||
buckets.Outdated,
|
||||
buckets.Deleting,
|
||||
)
|
||||
|
||||
metricFamilies, err := controllerMetrics.Registry.Gather()
|
||||
require.NoError(t, err)
|
||||
|
||||
labelSet := commonLabelSet(labels)
|
||||
assertGatheredGauge(t, metricFamilies, "gha_controller_pending_ephemeral_runners", labelSet, 1)
|
||||
assertGatheredGauge(t, metricFamilies, "gha_controller_running_ephemeral_runners", labelSet, 2)
|
||||
assertGatheredGauge(t, metricFamilies, "gha_controller_succeeded_ephemeral_runners", labelSet, 1)
|
||||
assertGatheredGauge(t, metricFamilies, "gha_controller_failed_ephemeral_runners", labelSet, 1)
|
||||
assertGatheredGauge(t, metricFamilies, "gha_controller_outdated_ephemeral_runners", labelSet, 1)
|
||||
assertGatheredGauge(t, metricFamilies, "gha_controller_deleting_ephemeral_runners", labelSet, 1)
|
||||
}
|
||||
|
||||
func assertGatheredGauge(t *testing.T, metricFamilies []*dto.MetricFamily, metricName string, expectedLabels map[string]string, expectedValue float64) {
|
||||
t.Helper()
|
||||
|
||||
for _, family := range metricFamilies {
|
||||
if family.GetName() != metricName {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, metric := range family.GetMetric() {
|
||||
if !metricHasLabels(metric, expectedLabels) {
|
||||
continue
|
||||
}
|
||||
|
||||
require.NotNil(t, metric.Gauge, "metric %q should be a gauge", metricName)
|
||||
assert.Equal(t, expectedValue, metric.Gauge.GetValue(), "metric %q value mismatch", metricName)
|
||||
return
|
||||
}
|
||||
|
||||
require.Fail(t, fmt.Sprintf("metric %q with expected labels was not gathered", metricName))
|
||||
}
|
||||
|
||||
require.Fail(t, fmt.Sprintf("metric family %q was not gathered", metricName))
|
||||
}
|
||||
|
||||
func metricHasLabels(metric *dto.Metric, expectedLabels map[string]string) bool {
|
||||
actualLabels := make(map[string]string, len(metric.GetLabel()))
|
||||
for _, label := range metric.GetLabel() {
|
||||
actualLabels[label.GetName()] = label.GetValue()
|
||||
}
|
||||
|
||||
for name, expectedValue := range expectedLabels {
|
||||
if actualLabels[name] != expectedValue {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func commonLabelSet(labels githubmetrics.CommonLabels) map[string]string {
|
||||
return map[string]string{
|
||||
"name": labels.Name,
|
||||
"namespace": labels.Namespace,
|
||||
"repository": labels.Repository,
|
||||
"organization": labels.Organization,
|
||||
"enterprise": labels.Enterprise,
|
||||
}
|
||||
}
|
||||
@@ -58,6 +58,30 @@ var (
|
||||
},
|
||||
labels,
|
||||
)
|
||||
succeededEphemeralRunners = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Subsystem: githubScaleSetControllerSubsystem,
|
||||
Name: "succeeded_ephemeral_runners",
|
||||
Help: "Number of ephemeral runners in a succeeded state.",
|
||||
},
|
||||
labels,
|
||||
)
|
||||
outdatedEphemeralRunners = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Subsystem: githubScaleSetControllerSubsystem,
|
||||
Name: "outdated_ephemeral_runners",
|
||||
Help: "Number of ephemeral runners in an outdated state.",
|
||||
},
|
||||
labels,
|
||||
)
|
||||
deletingEphemeralRunners = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Subsystem: githubScaleSetControllerSubsystem,
|
||||
Name: "deleting_ephemeral_runners",
|
||||
Help: "Number of ephemeral runners in a deleting state.",
|
||||
},
|
||||
labels,
|
||||
)
|
||||
runningListeners = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Subsystem: githubScaleSetControllerSubsystem,
|
||||
@@ -72,15 +96,31 @@ func RegisterMetrics() {
|
||||
metrics.Registry.MustRegister(
|
||||
pendingEphemeralRunners,
|
||||
runningEphemeralRunners,
|
||||
succeededEphemeralRunners,
|
||||
failedEphemeralRunners,
|
||||
outdatedEphemeralRunners,
|
||||
deletingEphemeralRunners,
|
||||
runningListeners,
|
||||
)
|
||||
}
|
||||
|
||||
func SetEphemeralRunnerCountsByStatus(commonLabels CommonLabels, pending, running, failed int) {
|
||||
pendingEphemeralRunners.With(commonLabels.labels()).Set(float64(pending))
|
||||
runningEphemeralRunners.With(commonLabels.labels()).Set(float64(running))
|
||||
failedEphemeralRunners.With(commonLabels.labels()).Set(float64(failed))
|
||||
// SetEphemeralRunnerCountsByLifecycle sets ephemeral runner counts across all six lifecycle states
|
||||
// using deterministic bucket assignment with explicit precedence:
|
||||
//
|
||||
// Lifecycle Precedence Contract:
|
||||
// 1. deleting: if runner has DeletionTimestamp set
|
||||
// 2. explicit phase buckets: if runner.Status.Phase is one of (Running/Succeeded/Failed/Outdated)
|
||||
// 3. pending (fallback): for empty/unset/other phase values
|
||||
//
|
||||
// This ensures each runner maps to exactly one metric bucket (no double-counting).
|
||||
func SetEphemeralRunnerCountsByLifecycle(commonLabels CommonLabels, pending, running, succeeded, failed, outdated, deleting int) {
|
||||
labels := commonLabels.labels()
|
||||
pendingEphemeralRunners.With(labels).Set(float64(pending))
|
||||
runningEphemeralRunners.With(labels).Set(float64(running))
|
||||
succeededEphemeralRunners.With(labels).Set(float64(succeeded))
|
||||
failedEphemeralRunners.With(labels).Set(float64(failed))
|
||||
outdatedEphemeralRunners.With(labels).Set(float64(outdated))
|
||||
deletingEphemeralRunners.With(labels).Set(float64(deleting))
|
||||
}
|
||||
|
||||
func AddRunningListener(commonLabels CommonLabels) {
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSetEphemeralRunnerCountsByLifecycle(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
labels CommonLabels
|
||||
pending int
|
||||
running int
|
||||
succeeded int
|
||||
failed int
|
||||
outdated int
|
||||
deleting int
|
||||
}{
|
||||
{
|
||||
name: "all zeros",
|
||||
labels: CommonLabels{
|
||||
Name: "test-runner",
|
||||
Namespace: "default",
|
||||
Repository: "org/repo",
|
||||
Organization: "org",
|
||||
Enterprise: "enterprise1",
|
||||
},
|
||||
pending: 0,
|
||||
running: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
outdated: 0,
|
||||
deleting: 0,
|
||||
},
|
||||
{
|
||||
name: "mixed counts",
|
||||
labels: CommonLabels{
|
||||
Name: "test-runner",
|
||||
Namespace: "default",
|
||||
Repository: "org/repo",
|
||||
Organization: "org",
|
||||
Enterprise: "enterprise1",
|
||||
},
|
||||
pending: 5,
|
||||
running: 10,
|
||||
succeeded: 3,
|
||||
failed: 2,
|
||||
outdated: 1,
|
||||
deleting: 1,
|
||||
},
|
||||
{
|
||||
name: "all non-zero",
|
||||
labels: CommonLabels{
|
||||
Name: "test-runner-2",
|
||||
Namespace: "kube-system",
|
||||
Repository: "other/repo",
|
||||
Organization: "otherorg",
|
||||
Enterprise: "enterprise2",
|
||||
},
|
||||
pending: 1,
|
||||
running: 1,
|
||||
succeeded: 1,
|
||||
failed: 1,
|
||||
outdated: 1,
|
||||
deleting: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
SetEphemeralRunnerCountsByLifecycle(
|
||||
tc.labels,
|
||||
tc.pending,
|
||||
tc.running,
|
||||
tc.succeeded,
|
||||
tc.failed,
|
||||
tc.outdated,
|
||||
tc.deleting,
|
||||
)
|
||||
|
||||
labelString := testutil.CollectAndCount(pendingEphemeralRunners)
|
||||
require.Greater(t, labelString, 0, "metrics should be registered")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetEphemeralRunnerCountsByLifecycleValues(t *testing.T) {
|
||||
lbls := CommonLabels{
|
||||
Name: "precedence-test",
|
||||
Namespace: "default",
|
||||
Repository: "test/repo",
|
||||
Organization: "test",
|
||||
Enterprise: "testent",
|
||||
}
|
||||
|
||||
SetEphemeralRunnerCountsByLifecycle(lbls, 1, 2, 3, 4, 5, 6)
|
||||
|
||||
assertGaugeValue(t, pendingEphemeralRunners, lbls, 1)
|
||||
assertGaugeValue(t, runningEphemeralRunners, lbls, 2)
|
||||
assertGaugeValue(t, succeededEphemeralRunners, lbls, 3)
|
||||
assertGaugeValue(t, failedEphemeralRunners, lbls, 4)
|
||||
assertGaugeValue(t, outdatedEphemeralRunners, lbls, 5)
|
||||
assertGaugeValue(t, deletingEphemeralRunners, lbls, 6)
|
||||
}
|
||||
|
||||
func assertGaugeValue(t *testing.T, gauge *prometheus.GaugeVec, labels CommonLabels, expected float64) {
|
||||
t.Helper()
|
||||
assert.Equal(t, expected, testutil.ToFloat64(gauge.With(labels.labels())))
|
||||
}
|
||||
|
||||
func TestCommonLabelsContract(t *testing.T) {
|
||||
lbls := CommonLabels{
|
||||
Name: "test",
|
||||
Namespace: "ns",
|
||||
Repository: "repo",
|
||||
Organization: "org",
|
||||
Enterprise: "ent",
|
||||
}
|
||||
|
||||
promLabels := lbls.labels()
|
||||
|
||||
expected := prometheus.Labels{
|
||||
"name": "test",
|
||||
"namespace": "ns",
|
||||
"repository": "repo",
|
||||
"organization": "org",
|
||||
"enterprise": "ent",
|
||||
}
|
||||
|
||||
assert.Equal(t, expected, promLabels)
|
||||
}
|
||||
Reference in New Issue
Block a user