Merge bf63bb5170 into 631f39bb2a

charts/gha-runner-scale-set-experimental/values.yaml

@@ -340,6 +340,10 @@ controllerServiceAccount:
 #           "job_workflow_target",
 #         ]
 #   gauges:
+#     gha_available_jobs:
+#       labels: ["name", "namespace", "repository", "organization", "enterprise"]
+#     gha_acquired_jobs:
+#       labels: ["name", "namespace", "repository", "organization", "enterprise"]
 #     gha_assigned_jobs:
 #       labels: ["name", "namespace", "repository", "organization", "enterprise"]
 #     gha_running_jobs:

charts/gha-runner-scale-set/values.yaml

@@ -172,6 +172,10 @@ githubConfigSecret:
 #           "job_workflow_target",
 #         ]
 #   gauges:
+#     gha_available_jobs:
+#       labels: ["name", "namespace", "repository", "organization", "enterprise"]
+#     gha_acquired_jobs:
+#       labels: ["name", "namespace", "repository", "organization", "enterprise"]
 #     gha_assigned_jobs:
 #       labels: ["name", "namespace", "repository", "organization", "enterprise"]
 #     gha_running_jobs:

cmd/ghalistener/metrics/metrics.go

@@ -34,6 +34,8 @@ const (
 
 // Names of all metrics available on the listener
 const (
+	MetricAvailableJobs               = "gha_available_jobs"
+	MetricAcquiredJobs                = "gha_acquired_jobs"
 	MetricAssignedJobs                = "gha_assigned_jobs"
 	MetricRunningJobs                 = "gha_running_jobs"
 	MetricRegisteredRunners           = "gha_registered_runners"
@@ -60,6 +62,8 @@ var metricsHelp = metricsHelpRegistry{
 		MetricCompletedJobsTotal: "Total number of jobs completed.",
 	},
 	gauges: map[string]string{
+		MetricAvailableJobs:     "Number of jobs available to be acquired by this scale set.",
+		MetricAcquiredJobs:      "Number of jobs acquired by this scale set but not yet assigned to a runner.",
 		MetricAssignedJobs:      "Number of jobs assigned to this scale set.",
 		MetricRunningJobs:       "Number of jobs running (or about to be run).",
 		MetricRegisteredRunners: "Number of runners registered by the scale set.",
@@ -182,6 +186,24 @@ var defaultMetrics = v1alpha1.MetricsConfig{
 		},
 	},
 	Gauges: map[string]*v1alpha1.GaugeMetric{
+		MetricAvailableJobs: {
+			Labels: []string{
+				labelKeyEnterprise,
+				labelKeyOrganization,
+				labelKeyRepository,
+				labelKeyRunnerScaleSetName,
+				labelKeyRunnerScaleSetNamespace,
+			},
+		},
+		MetricAcquiredJobs: {
+			Labels: []string{
+				labelKeyEnterprise,
+				labelKeyOrganization,
+				labelKeyRepository,
+				labelKeyRunnerScaleSetName,
+				labelKeyRunnerScaleSetNamespace,
+			},
+		},
 		MetricAssignedJobs: {
 			Labels: []string{
 				labelKeyEnterprise,
@@ -477,6 +499,8 @@ func (e *exporter) RecordStatic(min, max int) {
 }
 
 func (e *exporter) RecordStatistics(stats *scaleset.RunnerScaleSetStatistic) {
+	e.setGauge(MetricAvailableJobs, e.scaleSetLabels, float64(stats.TotalAvailableJobs))
+	e.setGauge(MetricAcquiredJobs, e.scaleSetLabels, float64(stats.TotalAcquiredJobs))
 	e.setGauge(MetricAssignedJobs, e.scaleSetLabels, float64(stats.TotalAssignedJobs))
 	e.setGauge(MetricRunningJobs, e.scaleSetLabels, float64(stats.TotalRunningJobs))
 	e.setGauge(MetricRegisteredRunners, e.scaleSetLabels, float64(stats.TotalRegisteredRunners))

cmd/ghalistener/metrics/metrics_integration_test.go

@@ -5,7 +5,9 @@ import (
 
 	"github.com/actions/scaleset"
 	"github.com/prometheus/client_golang/prometheus"
+	io_prometheus_client "github.com/prometheus/client_model/go"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestMetricsWithWorkflowRefParsing(t *testing.T) {
@@ -97,3 +99,56 @@ func TestMetricsWithWorkflowRefParsing(t *testing.T) {
 		})
 	}
 }
+
+func TestRecordStatisticsExposesAllFields(t *testing.T) {
+	reg := prometheus.NewRegistry()
+	m := installMetrics(defaultMetrics, reg, discardLogger)
+
+	exp := &exporter{
+		scaleSetLabels: prometheus.Labels{
+			labelKeyRunnerScaleSetName:      "test-scale-set",
+			labelKeyRunnerScaleSetNamespace: "test-namespace",
+			labelKeyEnterprise:              "",
+			labelKeyOrganization:            "test-org",
+			labelKeyRepository:              "test-repo",
+		},
+		metrics: m,
+	}
+
+	stats := &scaleset.RunnerScaleSetStatistic{
+		TotalAvailableJobs:     5,
+		TotalAcquiredJobs:      3,
+		TotalAssignedJobs:      10,
+		TotalRunningJobs:       7,
+		TotalRegisteredRunners: 12,
+		TotalBusyRunners:       7,
+		TotalIdleRunners:       5,
+	}
+
+	exp.RecordStatistics(stats)
+
+	gathered, err := reg.Gather()
+	require.NoError(t, err)
+
+	metricsByName := make(map[string]*io_prometheus_client.MetricFamily, len(gathered))
+	for _, mf := range gathered {
+		metricsByName[mf.GetName()] = mf
+	}
+
+	wantGauges := map[string]float64{
+		"gha_available_jobs":     5,
+		"gha_acquired_jobs":      3,
+		"gha_assigned_jobs":      10,
+		"gha_running_jobs":       7,
+		"gha_registered_runners": 12,
+		"gha_busy_runners":       7,
+		"gha_idle_runners":       5,
+	}
+
+	for name, wantVal := range wantGauges {
+		mf, ok := metricsByName[name]
+		require.True(t, ok, "metric %q not found in gathered metrics", name)
+		require.NotEmpty(t, mf.GetMetric(), "metric %q has no samples", name)
+		assert.Equal(t, wantVal, mf.GetMetric()[0].GetGauge().GetValue(), "metric %q value mismatch", name)
+	}
+}

docs/gha-runner-scale-set-controller/samples/grafana-dashboard/README.md

@@ -31,7 +31,9 @@ The following metrics are required to be scraped by Prometheus in order to popul
 | controller_runtime_active_workers | controller | ARC Controller
 | controller_runtime_reconcile_time_seconds_sum | namespace | ARC Controller
 | controller_runtime_reconcile_errors_total | namespace | ARC Controller
-| gha_assigned_jobs | actions_github_com_scale_set_name, namespace | ARC Controller
+| gha_available_jobs | actions_github_com_scale_set_name, namespace | ARC Listener
+| gha_acquired_jobs | actions_github_com_scale_set_name, namespace | ARC Listener
+| gha_assigned_jobs | actions_github_com_scale_set_name, namespace | ARC Listener
 | gha_controller_failed_ephemeral_runners | name, namespace | ARC Controller
 | gha_controller_pending_ephemeral_runners | name, namespace | ARC Controller
 | gha_controller_running_ephemeral_runners | name, namespace | ARC Controller

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.28.1
 	github.com/onsi/gomega v1.39.1
 	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
 	github.com/stretchr/testify v1.11.1
 	github.com/teambition/rrule-go v1.8.2
 	go.uber.org/multierr v1.11.0
@@ -169,7 +170,6 @@ require (
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/pquerna/otp v1.5.0 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
 	github.com/prometheus/procfs v0.20.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect