Merge branch 'master' into master

.github/workflows/gha-e2e-tests.yaml

@@ -16,7 +16,7 @@ env:
   TARGET_ORG: actions-runner-controller
   TARGET_REPO: arc_e2e_test_dummy
   IMAGE_NAME: "arc-test-image"
-  IMAGE_VERSION: "0.9.0"
+  IMAGE_VERSION: "0.9.2"
 
 concurrency:
   # This will make sure we only apply the concurrency limits on pull requests

.golangci.yaml

@@ -1,7 +1,9 @@
 run:
   timeout: 3m
 output:
-  format: github-actions
+  formats:
+    - format: github-actions
+      path: stdout
 linters-settings:
   errcheck:
     exclude-functions:

Makefile

@@ -6,7 +6,7 @@ endif
 DOCKER_USER ?= $(shell echo ${DOCKER_IMAGE_NAME} | cut -d / -f1)
 VERSION ?= dev
 COMMIT_SHA = $(shell git rev-parse HEAD)
-RUNNER_VERSION ?= 2.314.1
+RUNNER_VERSION ?= 2.316.1
 TARGETPLATFORM ?= $(shell arch)
 RUNNER_NAME ?= ${DOCKER_USER}/actions-runner
 RUNNER_TAG  ?= ${VERSION}
@@ -68,7 +68,7 @@ endif
 all: manager
 
 lint:
-	docker run --rm -v $(PWD):/app -w /app golangci/golangci-lint:v1.55.2 golangci-lint run
+	docker run --rm -v $(PWD):/app -w /app golangci/golangci-lint:v1.57.2 golangci-lint run
 
 GO_TEST_ARGS ?= -short
 

charts/gha-runner-scale-set-controller/Chart.yaml

@@ -15,13 +15,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.9.0
+version: 0.9.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.9.0"
+appVersion: "0.9.2"
 
 home: https://github.com/actions/actions-runner-controller
 

charts/gha-runner-scale-set-controller/templates/_helpers.tpl

@@ -126,7 +126,3 @@ Create the name of the service account to use
 {{- end }}
 {{- $names | join ","}}
 {{- end }}
-
-{{- define "gha-runner-scale-set-controller.serviceMonitorName" -}}
-{{- include "gha-runner-scale-set-controller.fullname" . }}-service-monitor
-{{- end }}

charts/gha-runner-scale-set-controller/templates/deployment.yaml

@@ -128,6 +128,10 @@ spec:
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}

charts/gha-runner-scale-set-controller/tests/template_test.go

@@ -345,6 +345,7 @@ func TestTemplate_ControllerDeployment_Defaults(t *testing.T) {
 
 	assert.Len(t, deployment.Spec.Template.Spec.NodeSelector, 0)
 	assert.Nil(t, deployment.Spec.Template.Spec.Affinity)
+	assert.Len(t, deployment.Spec.Template.Spec.TopologySpreadConstraints, 0)
 	assert.Len(t, deployment.Spec.Template.Spec.Tolerations, 0)
 
 	managerImage := "ghcr.io/actions/gha-runner-scale-set-controller:dev"
@@ -424,6 +425,9 @@ func TestTemplate_ControllerDeployment_Customize(t *testing.T) {
 			"tolerations[0].key":           "foo",
 			"affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key":      "foo",
 			"affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator": "bar",
+			"topologySpreadConstraints[0].labelSelector.matchLabels.foo":                                                             "bar",
+			"topologySpreadConstraints[0].maxSkew":                                                                                   "1",
+			"topologySpreadConstraints[0].topologyKey":                                                                               "foo",
 			"priorityClassName":         "test-priority-class",
 			"flags.updateStrategy":      "eventual",
 			"flags.logLevel":            "info",
@@ -487,6 +491,11 @@ func TestTemplate_ControllerDeployment_Customize(t *testing.T) {
 	assert.Equal(t, "foo", deployment.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key)
 	assert.Equal(t, "bar", string(deployment.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Operator))
 
+	assert.Len(t, deployment.Spec.Template.Spec.TopologySpreadConstraints, 1)
+	assert.Equal(t, "bar", deployment.Spec.Template.Spec.TopologySpreadConstraints[0].LabelSelector.MatchLabels["foo"])
+	assert.Equal(t, int32(1), deployment.Spec.Template.Spec.TopologySpreadConstraints[0].MaxSkew)
+	assert.Equal(t, "foo", deployment.Spec.Template.Spec.TopologySpreadConstraints[0].TopologyKey)
+
 	assert.Len(t, deployment.Spec.Template.Spec.Tolerations, 1)
 	assert.Equal(t, "foo", deployment.Spec.Template.Spec.Tolerations[0].Key)
 
@@ -745,6 +754,7 @@ func TestTemplate_ControllerDeployment_WatchSingleNamespace(t *testing.T) {
 
 	assert.Len(t, deployment.Spec.Template.Spec.NodeSelector, 0)
 	assert.Nil(t, deployment.Spec.Template.Spec.Affinity)
+	assert.Len(t, deployment.Spec.Template.Spec.TopologySpreadConstraints, 0)
 	assert.Len(t, deployment.Spec.Template.Spec.Tolerations, 0)
 
 	managerImage := "ghcr.io/actions/gha-runner-scale-set-controller:dev"

charts/gha-runner-scale-set-controller/values.yaml

@@ -72,6 +72,8 @@ tolerations: []
 
 affinity: {}
 
+topologySpreadConstraints: []
+
 # Mount volumes in the container.
 volumes: []
 volumeMounts: []
@@ -81,9 +83,9 @@ volumeMounts: []
 # PriorityClass: system-cluster-critical
 priorityClassName: ""
 
-## If `metrics:` object is not provided, or commented out, the following flags 
-## will be applied the controller-manager and listener pods with empty values: 
-## `--metrics-addr`, `--listener-metrics-addr`, `--listener-metrics-endpoint`. 
+## If `metrics:` object is not provided, or commented out, the following flags
+## will be applied the controller-manager and listener pods with empty values:
+## `--metrics-addr`, `--listener-metrics-addr`, `--listener-metrics-endpoint`.
 ## This will disable metrics.
 ##
 ## To enable metrics, uncomment the following lines.

charts/gha-runner-scale-set/Chart.yaml

@@ -15,13 +15,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.9.0
+version: 0.9.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.9.0"
+appVersion: "0.9.2"
 
 home: https://github.com/actions/actions-runner-controller
 

charts/gha-runner-scale-set/values.yaml

@@ -88,7 +88,7 @@ githubConfigSecret:
 #   kubernetesModeServiceAccount:
 #     annotations:
 
-## template is the PodSpec for each listener Pod
+## listenerTemplate is the PodSpec for each listener Pod
 ## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
 # listenerTemplate:
 #   spec:

cmd/ghalistener/app/app.go

@@ -117,15 +117,19 @@ func (app *App) Run(ctx context.Context) error {
 	}
 
 	g, ctx := errgroup.WithContext(ctx)
+	metricsCtx, cancelMetrics := context.WithCancelCause(ctx)
+
 	g.Go(func() error {
 		app.logger.Info("Starting listener")
-		return app.listener.Listen(ctx, app.worker)
+		listnerErr := app.listener.Listen(ctx, app.worker)
+		cancelMetrics(fmt.Errorf("Listener exited: %w", listnerErr))
+		return listnerErr
 	})
 
 	if app.metrics != nil {
 		g.Go(func() error {
 			app.logger.Info("Starting metrics server")
-			return app.metrics.ListenAndServe(ctx)
+			return app.metrics.ListenAndServe(metricsCtx)
 		})
 	}
 

cmd/ghalistener/listener/listener.go

@@ -31,7 +31,7 @@ const (
 type Client interface {
 	GetAcquirableJobs(ctx context.Context, runnerScaleSetId int) (*actions.AcquirableJobList, error)
 	CreateMessageSession(ctx context.Context, runnerScaleSetId int, owner string) (*actions.RunnerScaleSetSession, error)
-	GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64) (*actions.RunnerScaleSetMessage, error)
+	GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64, maxCapacity int) (*actions.RunnerScaleSetMessage, error)
 	DeleteMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, messageId int64) error
 	AcquireJobs(ctx context.Context, runnerScaleSetId int, messageQueueAccessToken string, requestIds []int64) ([]int64, error)
 	RefreshMessageSession(ctx context.Context, runnerScaleSetId int, sessionId *uuid.UUID) (*actions.RunnerScaleSetSession, error)
@@ -80,6 +80,7 @@ type Listener struct {
 
 	// updated fields
 	lastMessageID int64                          // The ID of the last processed message.
+	maxCapacity   int                            // The maximum number of runners that can be created.
 	session       *actions.RunnerScaleSetSession // The session for managing the runner scale set.
 }
 
@@ -89,10 +90,11 @@ func New(config Config) (*Listener, error) {
 	}
 
 	listener := &Listener{
-		scaleSetID: config.ScaleSetID,
-		client:     config.Client,
-		logger:     config.Logger,
-		metrics:    metrics.Discard,
+		scaleSetID:  config.ScaleSetID,
+		client:      config.Client,
+		logger:      config.Logger,
+		metrics:     metrics.Discard,
+		maxCapacity: config.MaxRunners,
 	}
 
 	if config.Metrics != nil {
@@ -164,11 +166,16 @@ func (l *Listener) Listen(ctx context.Context, handler Handler) error {
 		}
 
 		if msg == nil {
+			_, err := handler.HandleDesiredRunnerCount(ctx, 0, 0)
+			if err != nil {
+				return fmt.Errorf("handling nil message failed: %w", err)
+			}
+
 			continue
 		}
 
-		// New context is created to avoid cancelation during message handling.
-		if err := l.handleMessage(context.Background(), handler, msg); err != nil {
+		// Remove cancellation from the context to avoid cancelling the message handling.
+		if err := l.handleMessage(context.WithoutCancel(ctx), handler, msg); err != nil {
 			return fmt.Errorf("failed to handle message: %w", err)
 		}
 	}
@@ -262,7 +269,7 @@ func (l *Listener) createSession(ctx context.Context) error {
 
 func (l *Listener) getMessage(ctx context.Context) (*actions.RunnerScaleSetMessage, error) {
 	l.logger.Info("Getting next message", "lastMessageID", l.lastMessageID)
-	msg, err := l.client.GetMessage(ctx, l.session.MessageQueueUrl, l.session.MessageQueueAccessToken, l.lastMessageID)
+	msg, err := l.client.GetMessage(ctx, l.session.MessageQueueUrl, l.session.MessageQueueAccessToken, l.lastMessageID, l.maxCapacity)
 	if err == nil { // if NO error
 		return msg, nil
 	}
@@ -278,7 +285,7 @@ func (l *Listener) getMessage(ctx context.Context) (*actions.RunnerScaleSetMessa
 
 	l.logger.Info("Getting next message", "lastMessageID", l.lastMessageID)
 
-	msg, err = l.client.GetMessage(ctx, l.session.MessageQueueUrl, l.session.MessageQueueAccessToken, l.lastMessageID)
+	msg, err = l.client.GetMessage(ctx, l.session.MessageQueueUrl, l.session.MessageQueueAccessToken, l.lastMessageID, l.maxCapacity)
 	if err != nil { // if NO error
 		return nil, fmt.Errorf("failed to get next message after message session refresh: %w", err)
 	}
@@ -288,8 +295,23 @@ func (l *Listener) getMessage(ctx context.Context) (*actions.RunnerScaleSetMessa
 
 func (l *Listener) deleteLastMessage(ctx context.Context) error {
 	l.logger.Info("Deleting last message", "lastMessageID", l.lastMessageID)
-	if err := l.client.DeleteMessage(ctx, l.session.MessageQueueUrl, l.session.MessageQueueAccessToken, l.lastMessageID); err != nil {
-		return fmt.Errorf("failed to delete message: %w", err)
+	err := l.client.DeleteMessage(ctx, l.session.MessageQueueUrl, l.session.MessageQueueAccessToken, l.lastMessageID)
+	if err == nil { // if NO error
+		return nil
+	}
+
+	expiredError := &actions.MessageQueueTokenExpiredError{}
+	if !errors.As(err, &expiredError) {
+		return fmt.Errorf("failed to delete last message: %w", err)
+	}
+
+	if err := l.refreshSession(ctx); err != nil {
+		return err
+	}
+
+	err = l.client.DeleteMessage(ctx, l.session.MessageQueueUrl, l.session.MessageQueueAccessToken, l.lastMessageID)
+	if err != nil {
+		return fmt.Errorf("failed to delete last message after message session refresh: %w", err)
 	}
 
 	return nil

cmd/ghalistener/listener/listener_test.go

@@ -123,13 +123,14 @@ func TestListener_getMessage(t *testing.T) {
 		config := Config{
 			ScaleSetID: 1,
 			Metrics:    metrics.Discard,
+			MaxRunners: 10,
 		}
 
 		client := listenermocks.NewClient(t)
 		want := &actions.RunnerScaleSetMessage{
 			MessageId: 1,
 		}
-		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything).Return(want, nil).Once()
+		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything, 10).Return(want, nil).Once()
 		config.Client = client
 
 		l, err := New(config)
@@ -148,10 +149,11 @@ func TestListener_getMessage(t *testing.T) {
 		config := Config{
 			ScaleSetID: 1,
 			Metrics:    metrics.Discard,
+			MaxRunners: 10,
 		}
 
 		client := listenermocks.NewClient(t)
-		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything).Return(nil, &actions.HttpClientSideError{Code: http.StatusNotFound}).Once()
+		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything, 10).Return(nil, &actions.HttpClientSideError{Code: http.StatusNotFound}).Once()
 		config.Client = client
 
 		l, err := New(config)
@@ -170,6 +172,7 @@ func TestListener_getMessage(t *testing.T) {
 		config := Config{
 			ScaleSetID: 1,
 			Metrics:    metrics.Discard,
+			MaxRunners: 10,
 		}
 
 		client := listenermocks.NewClient(t)
@@ -185,12 +188,12 @@ func TestListener_getMessage(t *testing.T) {
 		}
 		client.On("RefreshMessageSession", ctx, mock.Anything, mock.Anything).Return(session, nil).Once()
 
-		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything).Return(nil, &actions.MessageQueueTokenExpiredError{}).Once()
+		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything, 10).Return(nil, &actions.MessageQueueTokenExpiredError{}).Once()
 
 		want := &actions.RunnerScaleSetMessage{
 			MessageId: 1,
 		}
-		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything).Return(want, nil).Once()
+		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything, 10).Return(want, nil).Once()
 
 		config.Client = client
 
@@ -214,6 +217,7 @@ func TestListener_getMessage(t *testing.T) {
 		config := Config{
 			ScaleSetID: 1,
 			Metrics:    metrics.Discard,
+			MaxRunners: 10,
 		}
 
 		client := listenermocks.NewClient(t)
@@ -229,7 +233,7 @@ func TestListener_getMessage(t *testing.T) {
 		}
 		client.On("RefreshMessageSession", ctx, mock.Anything, mock.Anything).Return(session, nil).Once()
 
-		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything).Return(nil, &actions.MessageQueueTokenExpiredError{}).Twice()
+		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything, 10).Return(nil, &actions.MessageQueueTokenExpiredError{}).Twice()
 
 		config.Client = client
 
@@ -373,6 +377,93 @@ func TestListener_deleteLastMessage(t *testing.T) {
 		err = l.deleteLastMessage(ctx)
 		assert.NotNil(t, err)
 	})
+
+	t.Run("RefreshAndSucceeds", func(t *testing.T) {
+		t.Parallel()
+
+		ctx := context.Background()
+		config := Config{
+			ScaleSetID: 1,
+			Metrics:    metrics.Discard,
+		}
+
+		client := listenermocks.NewClient(t)
+
+		newUUID := uuid.New()
+		session := &actions.RunnerScaleSetSession{
+			SessionId:               &newUUID,
+			OwnerName:               "example",
+			RunnerScaleSet:          &actions.RunnerScaleSet{},
+			MessageQueueUrl:         "https://example.com",
+			MessageQueueAccessToken: "1234567890",
+			Statistics:              nil,
+		}
+		client.On("RefreshMessageSession", ctx, mock.Anything, mock.Anything).Return(session, nil).Once()
+
+		client.On("DeleteMessage", ctx, mock.Anything, mock.Anything, mock.Anything).Return(&actions.MessageQueueTokenExpiredError{}).Once()
+
+		client.On("DeleteMessage", ctx, mock.Anything, mock.Anything, mock.MatchedBy(func(lastMessageID any) bool {
+			return lastMessageID.(int64) == int64(5)
+		})).Return(nil).Once()
+		config.Client = client
+
+		l, err := New(config)
+		require.Nil(t, err)
+
+		oldUUID := uuid.New()
+		l.session = &actions.RunnerScaleSetSession{
+			SessionId:      &oldUUID,
+			RunnerScaleSet: &actions.RunnerScaleSet{},
+		}
+		l.lastMessageID = 5
+
+		config.Client = client
+
+		err = l.deleteLastMessage(ctx)
+		assert.NoError(t, err)
+	})
+
+	t.Run("RefreshAndFails", func(t *testing.T) {
+		t.Parallel()
+
+		ctx := context.Background()
+		config := Config{
+			ScaleSetID: 1,
+			Metrics:    metrics.Discard,
+		}
+
+		client := listenermocks.NewClient(t)
+
+		newUUID := uuid.New()
+		session := &actions.RunnerScaleSetSession{
+			SessionId:               &newUUID,
+			OwnerName:               "example",
+			RunnerScaleSet:          &actions.RunnerScaleSet{},
+			MessageQueueUrl:         "https://example.com",
+			MessageQueueAccessToken: "1234567890",
+			Statistics:              nil,
+		}
+		client.On("RefreshMessageSession", ctx, mock.Anything, mock.Anything).Return(session, nil).Once()
+
+		client.On("DeleteMessage", ctx, mock.Anything, mock.Anything, mock.Anything).Return(&actions.MessageQueueTokenExpiredError{}).Twice()
+
+		config.Client = client
+
+		l, err := New(config)
+		require.Nil(t, err)
+
+		oldUUID := uuid.New()
+		l.session = &actions.RunnerScaleSetSession{
+			SessionId:      &oldUUID,
+			RunnerScaleSet: &actions.RunnerScaleSet{},
+		}
+		l.lastMessageID = 5
+
+		config.Client = client
+
+		err = l.deleteLastMessage(ctx)
+		assert.Error(t, err)
+	})
 }
 
 func TestListener_Listen(t *testing.T) {
@@ -450,6 +541,7 @@ func TestListener_Listen(t *testing.T) {
 		config := Config{
 			ScaleSetID: 1,
 			Metrics:    metrics.Discard,
+			MaxRunners: 10,
 		}
 
 		client := listenermocks.NewClient(t)
@@ -470,7 +562,7 @@ func TestListener_Listen(t *testing.T) {
 			MessageType: "RunnerScaleSetJobMessages",
 			Statistics:  &actions.RunnerScaleSetStatistic{},
 		}
-		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything).
+		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything, 10).
 			Return(msg, nil).
 			Run(
 				func(mock.Arguments) {
@@ -479,8 +571,8 @@ func TestListener_Listen(t *testing.T) {
 			).
 			Once()
 
-			// Ensure delete message is called with background context
-		client.On("DeleteMessage", context.Background(), mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
+		// Ensure delete message is called without cancel
+		client.On("DeleteMessage", context.WithoutCancel(ctx), mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
 
 		config.Client = client
 

cmd/ghalistener/listener/mocks/client.go

@@ -123,25 +123,25 @@ func (_m *Client) GetAcquirableJobs(ctx context.Context, runnerScaleSetId int) (
 	return r0, r1
 }
 
-// GetMessage provides a mock function with given fields: ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId
-func (_m *Client) GetMessage(ctx context.Context, messageQueueUrl string, messageQueueAccessToken string, lastMessageId int64) (*actions.RunnerScaleSetMessage, error) {
-	ret := _m.Called(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+// GetMessage provides a mock function with given fields: ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity
+func (_m *Client) GetMessage(ctx context.Context, messageQueueUrl string, messageQueueAccessToken string, lastMessageId int64, maxCapacity int) (*actions.RunnerScaleSetMessage, error) {
+	ret := _m.Called(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 
 	var r0 *actions.RunnerScaleSetMessage
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64) (*actions.RunnerScaleSetMessage, error)); ok {
-		return rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int) (*actions.RunnerScaleSetMessage, error)); ok {
+		return rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64) *actions.RunnerScaleSetMessage); ok {
-		r0 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int) *actions.RunnerScaleSetMessage); ok {
+		r0 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(*actions.RunnerScaleSetMessage)
 		}
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, string, string, int64) error); ok {
-		r1 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+	if rf, ok := ret.Get(1).(func(context.Context, string, string, int64, int) error); ok {
+		r1 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 	} else {
 		r1 = ret.Error(1)
 	}

cmd/ghalistener/metrics/metrics.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net/http"
 	"strconv"
+	"time"
 
 	"github.com/actions/actions-runner-controller/github/actions"
 	"github.com/go-logr/logr"
@@ -338,7 +339,9 @@ func (e *exporter) ListenAndServe(ctx context.Context) error {
 	e.logger.Info("starting metrics server", "addr", e.srv.Addr)
 	go func() {
 		<-ctx.Done()
-		e.logger.Info("stopping metrics server")
+		e.logger.Info("stopping metrics server", "err", ctx.Err())
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
 		e.srv.Shutdown(ctx)
 	}()
 	return e.srv.ListenAndServe()

cmd/ghalistener/worker/worker.go

@@ -38,20 +38,20 @@ type Config struct {
 // The Worker's role is to process the messages it receives from the listener.
 // It then initiates Kubernetes API requests to carry out the necessary actions.
 type Worker struct {
-	clientset   *kubernetes.Clientset
-	config      Config
-	lastPatch   int
-	lastPatchID int
-	logger      *logr.Logger
+	clientset *kubernetes.Clientset
+	config    Config
+	lastPatch int
+	patchSeq  int
+	logger    *logr.Logger
 }
 
 var _ listener.Handler = (*Worker)(nil)
 
 func New(config Config, options ...Option) (*Worker, error) {
 	w := &Worker{
-		config:      config,
-		lastPatch:   -1,
-		lastPatchID: -1,
+		config:    config,
+		lastPatch: -1,
+		patchSeq:  -1,
 	}
 
 	conf, err := rest.InClusterConfig()
@@ -163,27 +163,8 @@ func (w *Worker) HandleJobStarted(ctx context.Context, jobInfo *actions.JobStart
 // The function then scales the ephemeral runner set by applying the merge patch.
 // Finally, it logs the scaled ephemeral runner set details and returns nil if successful.
 // If any error occurs during the process, it returns an error with a descriptive message.
-func (w *Worker) HandleDesiredRunnerCount(ctx context.Context, count int, jobsCompleted int) (int, error) {
-	// Max runners should always be set by the resource builder either to the configured value,
-	// or the maximum int32 (resourcebuilder.newAutoScalingListener()).
-	targetRunnerCount := min(w.config.MinRunners+count, w.config.MaxRunners)
-
-	logValues := []any{
-		"assigned job", count,
-		"decision", targetRunnerCount,
-		"min", w.config.MinRunners,
-		"max", w.config.MaxRunners,
-		"currentRunnerCount", w.lastPatch,
-		"jobsCompleted", jobsCompleted,
-	}
-
-	if w.lastPatch == targetRunnerCount && jobsCompleted == 0 {
-		w.logger.Info("Skipping patch", logValues...)
-		return targetRunnerCount, nil
-	}
-
-	w.lastPatchID++
-	w.lastPatch = targetRunnerCount
+func (w *Worker) HandleDesiredRunnerCount(ctx context.Context, count, jobsCompleted int) (int, error) {
+	patchID := w.setDesiredWorkerState(count, jobsCompleted)
 
 	original, err := json.Marshal(
 		&v1alpha1.EphemeralRunnerSet{
@@ -200,8 +181,8 @@ func (w *Worker) HandleDesiredRunnerCount(ctx context.Context, count int, jobsCo
 	patch, err := json.Marshal(
 		&v1alpha1.EphemeralRunnerSet{
 			Spec: v1alpha1.EphemeralRunnerSetSpec{
-				Replicas: targetRunnerCount,
-				PatchID:  w.lastPatchID,
+				Replicas: w.lastPatch,
+				PatchID:  patchID,
 			},
 		},
 	)
@@ -210,14 +191,13 @@ func (w *Worker) HandleDesiredRunnerCount(ctx context.Context, count int, jobsCo
 		return 0, err
 	}
 
+	w.logger.Info("Compare", "original", string(original), "patch", string(patch))
 	mergePatch, err := jsonpatch.CreateMergePatch(original, patch)
 	if err != nil {
 		return 0, fmt.Errorf("failed to create merge patch json for ephemeral runner set: %w", err)
 	}
 
-	w.logger.Info("Created merge patch json for EphemeralRunnerSet update", "json", string(mergePatch))
-
-	w.logger.Info("Scaling ephemeral runner set", logValues...)
+	w.logger.Info("Preparing EphemeralRunnerSet update", "json", string(mergePatch))
 
 	patchedEphemeralRunnerSet := &v1alpha1.EphemeralRunnerSet{}
 	err = w.clientset.RESTClient().
@@ -238,5 +218,40 @@ func (w *Worker) HandleDesiredRunnerCount(ctx context.Context, count int, jobsCo
 		"name", w.config.EphemeralRunnerSetName,
 		"replicas", patchedEphemeralRunnerSet.Spec.Replicas,
 	)
-	return targetRunnerCount, nil
+	return w.lastPatch, nil
+}
+
+// calculateDesiredState calculates the desired state of the worker based on the desired count and the the number of jobs completed.
+func (w *Worker) setDesiredWorkerState(count, jobsCompleted int) int {
+	// Max runners should always be set by the resource builder either to the configured value,
+	// or the maximum int32 (resourcebuilder.newAutoScalingListener()).
+	targetRunnerCount := min(w.config.MinRunners+count, w.config.MaxRunners)
+	w.patchSeq++
+	desiredPatchID := w.patchSeq
+
+	if count == 0 && jobsCompleted == 0 { // empty batch
+		targetRunnerCount = max(w.lastPatch, targetRunnerCount)
+		if targetRunnerCount == w.config.MinRunners {
+			// We have an empty batch, and the last patch was the min runners.
+			// Since this is an empty batch, and we are at the min runners, they should all be idle.
+			// If controller created few more pods on accident (during scale down events),
+			// this situation allows the controller to scale down to the min runners.
+			// However, it is important to keep the patch sequence increasing so we don't ignore one batch.
+			desiredPatchID = 0
+		}
+	}
+
+	w.lastPatch = targetRunnerCount
+
+	w.logger.Info(
+		"Calculated target runner count",
+		"assigned job", count,
+		"decision", targetRunnerCount,
+		"min", w.config.MinRunners,
+		"max", w.config.MaxRunners,
+		"currentRunnerCount", w.lastPatch,
+		"jobsCompleted", jobsCompleted,
+	)
+
+	return desiredPatchID
 }

cmd/ghalistener/worker/worker_test.go

@@ -0,0 +1,326 @@
+package worker
+
+import (
+	"math"
+	"testing"
+
+	"github.com/go-logr/logr"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSetDesiredWorkerState_MinMaxDefaults(t *testing.T) {
+	logger := logr.Discard()
+	newEmptyWorker := func() *Worker {
+		return &Worker{
+			config: Config{
+				MinRunners: 0,
+				MaxRunners: math.MaxInt32,
+			},
+			lastPatch: -1,
+			patchSeq:  -1,
+			logger:    &logger,
+		}
+	}
+
+	t.Run("init calculate with acquired 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 0, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+		assert.Equal(t, 0, patchID)
+	})
+
+	t.Run("init calculate with acquired 1", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(1, 0)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+		assert.Equal(t, 0, patchID)
+	})
+
+	t.Run("increment patch when job done", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(1, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(0, 1)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 0, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("increment patch when called with same parameters", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(1, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(1, 0)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("calculate desired scale when acquired > 0 and completed > 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(1, 1)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+	})
+
+	t.Run("re-use the last state when acquired == 0 and completed == 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(1, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("adjust when acquired == 0 and completed == 1", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(1, 1)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(0, 1)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 0, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+}
+
+func TestSetDesiredWorkerState_MinSet(t *testing.T) {
+	logger := logr.Discard()
+	newEmptyWorker := func() *Worker {
+		return &Worker{
+			config: Config{
+				MinRunners: 1,
+				MaxRunners: math.MaxInt32,
+			},
+			lastPatch: -1,
+			patchSeq:  -1,
+			logger:    &logger,
+		}
+	}
+
+	t.Run("initial scale when acquired == 0 and completed == 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+	})
+
+	t.Run("re-use the old state on count == 0 and completed == 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(2, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 3, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("request back to 0 on job done", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(2, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(0, 1)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("desired patch is 0 but sequence continues on empty batch and min runners", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(3, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 4, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+
+		patchID = w.setDesiredWorkerState(0, 3)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+
+		// Empty batch on min runners
+		patchID = w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 0, patchID) // forcing the state
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 2, w.patchSeq)
+	})
+
+}
+
+func TestSetDesiredWorkerState_MaxSet(t *testing.T) {
+	logger := logr.Discard()
+	newEmptyWorker := func() *Worker {
+		return &Worker{
+			config: Config{
+				MinRunners: 0,
+				MaxRunners: 5,
+			},
+			lastPatch: -1,
+			patchSeq:  -1,
+			logger:    &logger,
+		}
+	}
+
+	t.Run("initial scale when acquired == 0 and completed == 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 0, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+	})
+
+	t.Run("re-use the old state on count == 0 and completed == 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(2, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 2, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("request back to 0 on job done", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(2, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(0, 1)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 0, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("scale up to max when count > max", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(6, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 5, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+	})
+
+	t.Run("scale to max when count == max", func(t *testing.T) {
+		w := newEmptyWorker()
+		w.setDesiredWorkerState(5, 0)
+		assert.Equal(t, 5, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+	})
+
+	t.Run("scale to max when count > max and completed > 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(1, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(6, 1)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 5, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("scale back to 0 when count was > max", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(6, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(0, 1)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 0, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("force 0 on empty batch and last patch == min runners", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(3, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 3, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+
+		patchID = w.setDesiredWorkerState(0, 3)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 0, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+
+		// Empty batch on min runners
+		patchID = w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 0, patchID) // forcing the state
+		assert.Equal(t, 0, w.lastPatch)
+		assert.Equal(t, 2, w.patchSeq)
+	})
+}
+
+func TestSetDesiredWorkerState_MinMaxSet(t *testing.T) {
+	logger := logr.Discard()
+	newEmptyWorker := func() *Worker {
+		return &Worker{
+			config: Config{
+				MinRunners: 1,
+				MaxRunners: 3,
+			},
+			lastPatch: -1,
+			patchSeq:  -1,
+			logger:    &logger,
+		}
+	}
+
+	t.Run("initial scale when acquired == 0 and completed == 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+	})
+
+	t.Run("re-use the old state on count == 0 and completed == 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(2, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 3, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("scale to min when count == 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(2, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(0, 1)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("scale up to max when count > max", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(4, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 3, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+	})
+
+	t.Run("scale to max when count == max", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(3, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 3, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+	})
+
+	t.Run("force 0 on empty batch and last patch == min runners", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(3, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 3, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+
+		patchID = w.setDesiredWorkerState(0, 3)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+
+		// Empty batch on min runners
+		patchID = w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 0, patchID) // forcing the state
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 2, w.patchSeq)
+	})
+}

cmd/githubrunnerscalesetlistener/autoScalerMessageListener.go

@@ -129,7 +129,7 @@ func (m *AutoScalerClient) Close() error {
 	return m.client.Close()
 }
 
-func (m *AutoScalerClient) GetRunnerScaleSetMessage(ctx context.Context, handler func(msg *actions.RunnerScaleSetMessage) error) error {
+func (m *AutoScalerClient) GetRunnerScaleSetMessage(ctx context.Context, handler func(msg *actions.RunnerScaleSetMessage) error, maxCapacity int) error {
 	if m.initialMessage != nil {
 		err := handler(m.initialMessage)
 		if err != nil {
@@ -141,7 +141,7 @@ func (m *AutoScalerClient) GetRunnerScaleSetMessage(ctx context.Context, handler
 	}
 
 	for {
-		message, err := m.client.GetMessage(ctx, m.lastMessageId)
+		message, err := m.client.GetMessage(ctx, m.lastMessageId, maxCapacity)
 		if err != nil {
 			return fmt.Errorf("get message failed from refreshing client. %w", err)
 		}

cmd/githubrunnerscalesetlistener/autoScalerMessageListener_test.go

@@ -317,7 +317,7 @@ func TestGetRunnerScaleSetMessage(t *testing.T) {
 		Statistics: &actions.RunnerScaleSetStatistic{},
 	}
 	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything).Return(session, nil)
-	mockSessionClient.On("GetMessage", ctx, int64(0)).Return(&actions.RunnerScaleSetMessage{
+	mockSessionClient.On("GetMessage", ctx, int64(0), mock.Anything).Return(&actions.RunnerScaleSetMessage{
 		MessageId:   1,
 		MessageType: "test",
 		Body:        "test",
@@ -332,7 +332,7 @@ func TestGetRunnerScaleSetMessage(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 
 	assert.NoError(t, err, "Error getting message")
 	assert.Equal(t, int64(0), asClient.lastMessageId, "Initial message")
@@ -340,7 +340,7 @@ func TestGetRunnerScaleSetMessage(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 
 	assert.NoError(t, err, "Error getting message")
 	assert.Equal(t, int64(1), asClient.lastMessageId, "Last message id should be updated")
@@ -368,7 +368,7 @@ func TestGetRunnerScaleSetMessage_HandleFailed(t *testing.T) {
 		Statistics: &actions.RunnerScaleSetStatistic{},
 	}
 	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything).Return(session, nil)
-	mockSessionClient.On("GetMessage", ctx, int64(0)).Return(&actions.RunnerScaleSetMessage{
+	mockSessionClient.On("GetMessage", ctx, int64(0), mock.Anything).Return(&actions.RunnerScaleSetMessage{
 		MessageId:   1,
 		MessageType: "test",
 		Body:        "test",
@@ -383,14 +383,14 @@ func TestGetRunnerScaleSetMessage_HandleFailed(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 
 	assert.NoError(t, err, "Error getting message")
 
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return fmt.Errorf("error")
-	})
+	}, 10)
 
 	assert.ErrorContains(t, err, "handle message failed. error", "Error getting message")
 	assert.Equal(t, int64(0), asClient.lastMessageId, "Last message id should not be updated")
@@ -419,7 +419,7 @@ func TestGetRunnerScaleSetMessage_HandleInitialMessage(t *testing.T) {
 			TotalAssignedJobs:  2,
 		},
 	}
-	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything).Return(session, nil)
+	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything, mock.Anything).Return(session, nil)
 	mockActionsClient.On("GetAcquirableJobs", ctx, 1).Return(&actions.AcquirableJobList{
 		Count: 1,
 		Jobs: []actions.AcquirableJob{
@@ -439,7 +439,7 @@ func TestGetRunnerScaleSetMessage_HandleInitialMessage(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 
 	assert.NoError(t, err, "Error getting message")
 	assert.Nil(t, asClient.initialMessage, "Initial message should be nil")
@@ -488,7 +488,7 @@ func TestGetRunnerScaleSetMessage_HandleInitialMessageFailed(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return fmt.Errorf("error")
-	})
+	}, 10)
 
 	assert.ErrorContains(t, err, "fail to process initial message. error", "Error getting message")
 	assert.NotNil(t, asClient.initialMessage, "Initial message should be nil")
@@ -516,8 +516,8 @@ func TestGetRunnerScaleSetMessage_RetryUntilGetMessage(t *testing.T) {
 		Statistics: &actions.RunnerScaleSetStatistic{},
 	}
 	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything).Return(session, nil)
-	mockSessionClient.On("GetMessage", ctx, int64(0)).Return(nil, nil).Times(3)
-	mockSessionClient.On("GetMessage", ctx, int64(0)).Return(&actions.RunnerScaleSetMessage{
+	mockSessionClient.On("GetMessage", ctx, int64(0), mock.Anything).Return(nil, nil).Times(3)
+	mockSessionClient.On("GetMessage", ctx, int64(0), mock.Anything).Return(&actions.RunnerScaleSetMessage{
 		MessageId:   1,
 		MessageType: "test",
 		Body:        "test",
@@ -532,13 +532,13 @@ func TestGetRunnerScaleSetMessage_RetryUntilGetMessage(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 	assert.NoError(t, err, "Error getting initial message")
 
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 
 	assert.NoError(t, err, "Error getting message")
 	assert.Equal(t, int64(1), asClient.lastMessageId, "Last message id should be updated")
@@ -565,7 +565,7 @@ func TestGetRunnerScaleSetMessage_ErrorOnGetMessage(t *testing.T) {
 		Statistics: &actions.RunnerScaleSetStatistic{},
 	}
 	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything).Return(session, nil)
-	mockSessionClient.On("GetMessage", ctx, int64(0)).Return(nil, fmt.Errorf("error"))
+	mockSessionClient.On("GetMessage", ctx, int64(0), mock.Anything).Return(nil, fmt.Errorf("error"))
 
 	asClient, err := NewAutoScalerClient(ctx, mockActionsClient, &logger, 1, func(asc *AutoScalerClient) {
 		asc.client = mockSessionClient
@@ -575,12 +575,12 @@ func TestGetRunnerScaleSetMessage_ErrorOnGetMessage(t *testing.T) {
 	// process initial message
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		return nil
-	})
+	}, 10)
 	assert.NoError(t, err, "Error getting initial message")
 
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		return fmt.Errorf("Should not be called")
-	})
+	}, 10)
 
 	assert.ErrorContains(t, err, "get message failed from refreshing client. error", "Error should be returned")
 	assert.Equal(t, int64(0), asClient.lastMessageId, "Last message id should be updated")
@@ -608,7 +608,7 @@ func TestDeleteRunnerScaleSetMessage_Error(t *testing.T) {
 		Statistics: &actions.RunnerScaleSetStatistic{},
 	}
 	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything).Return(session, nil)
-	mockSessionClient.On("GetMessage", ctx, int64(0)).Return(&actions.RunnerScaleSetMessage{
+	mockSessionClient.On("GetMessage", ctx, int64(0), mock.Anything).Return(&actions.RunnerScaleSetMessage{
 		MessageId:   1,
 		MessageType: "test",
 		Body:        "test",
@@ -623,13 +623,13 @@ func TestDeleteRunnerScaleSetMessage_Error(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 	assert.NoError(t, err, "Error getting initial message")
 
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 
 	assert.ErrorContains(t, err, "delete message failed from refreshing client. error", "Error getting message")
 	assert.Equal(t, int64(1), asClient.lastMessageId, "Last message id should be updated")

cmd/githubrunnerscalesetlistener/autoScalerService.go

@@ -89,7 +89,7 @@ func (s *Service) Start() error {
 			s.logger.Info("service is stopped.")
 			return nil
 		default:
-			err := s.rsClient.GetRunnerScaleSetMessage(s.ctx, s.processMessage)
+			err := s.rsClient.GetRunnerScaleSetMessage(s.ctx, s.processMessage, s.settings.MaxRunners)
 			if err != nil {
 				return fmt.Errorf("could not get and process message. %w", err)
 			}

cmd/githubrunnerscalesetlistener/autoScalerService_test.go

@@ -64,7 +64,7 @@ func TestStart(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything).Run(func(args mock.Arguments) { cancel() }).Return(nil).Once()
+	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything, mock.Anything).Run(func(mock.Arguments) { cancel() }).Return(nil).Once()
 
 	err = service.Start()
 
@@ -98,7 +98,7 @@ func TestStart_ScaleToMinRunners(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	mockRsClient.On("GetRunnerScaleSetMessage", ctx, mock.Anything).Run(func(args mock.Arguments) {
+	mockRsClient.On("GetRunnerScaleSetMessage", ctx, mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
 		_ = service.scaleForAssignedJobCount(5)
 	}).Return(nil)
 
@@ -137,7 +137,7 @@ func TestStart_ScaleToMinRunnersFailed(t *testing.T) {
 	require.NoError(t, err)
 
 	c := mockKubeManager.On("ScaleEphemeralRunnerSet", ctx, service.settings.Namespace, service.settings.ResourceName, 5).Return(fmt.Errorf("error")).Once()
-	mockRsClient.On("GetRunnerScaleSetMessage", ctx, mock.Anything).Run(func(args mock.Arguments) {
+	mockRsClient.On("GetRunnerScaleSetMessage", ctx, mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
 		_ = service.scaleForAssignedJobCount(5)
 	}).Return(c.ReturnArguments.Get(0))
 
@@ -172,8 +172,8 @@ func TestStart_GetMultipleMessages(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything).Return(nil).Times(5)
-	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything).Run(func(args mock.Arguments) { cancel() }).Return(nil).Once()
+	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything, mock.Anything).Return(nil).Times(5)
+	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { cancel() }).Return(nil).Once()
 
 	err = service.Start()
 
@@ -207,8 +207,8 @@ func TestStart_ErrorOnMessage(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything).Return(nil).Times(2)
-	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything).Return(fmt.Errorf("error")).Once()
+	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything, mock.Anything).Return(nil).Times(2)
+	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything, mock.Anything).Return(fmt.Errorf("error")).Once()
 
 	err = service.Start()
 

cmd/githubrunnerscalesetlistener/messageListener.go

@@ -8,6 +8,6 @@ import (
 
 //go:generate mockery --inpackage --name=RunnerScaleSetClient
 type RunnerScaleSetClient interface {
-	GetRunnerScaleSetMessage(ctx context.Context, handler func(msg *actions.RunnerScaleSetMessage) error) error
+	GetRunnerScaleSetMessage(ctx context.Context, handler func(msg *actions.RunnerScaleSetMessage) error, maxCapacity int) error
 	AcquireJobsForRunnerScaleSet(ctx context.Context, requestIds []int64) error
 }

cmd/githubrunnerscalesetlistener/mock_RunnerScaleSetClient.go

@@ -29,13 +29,13 @@ func (_m *MockRunnerScaleSetClient) AcquireJobsForRunnerScaleSet(ctx context.Con
 	return r0
 }
 
-// GetRunnerScaleSetMessage provides a mock function with given fields: ctx, handler
-func (_m *MockRunnerScaleSetClient) GetRunnerScaleSetMessage(ctx context.Context, handler func(*actions.RunnerScaleSetMessage) error) error {
-	ret := _m.Called(ctx, handler)
+// GetRunnerScaleSetMessage provides a mock function with given fields: ctx, handler, maxCapacity
+func (_m *MockRunnerScaleSetClient) GetRunnerScaleSetMessage(ctx context.Context, handler func(*actions.RunnerScaleSetMessage) error, maxCapacity int) error {
+	ret := _m.Called(ctx, handler, maxCapacity)
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, func(*actions.RunnerScaleSetMessage) error) error); ok {
-		r0 = rf(ctx, handler)
+	if rf, ok := ret.Get(0).(func(context.Context, func(*actions.RunnerScaleSetMessage) error, int) error); ok {
+		r0 = rf(ctx, handler, maxCapacity)
 	} else {
 		r0 = ret.Error(0)
 	}

cmd/githubrunnerscalesetlistener/sessionrefreshingclient.go

@@ -24,8 +24,12 @@ func newSessionClient(client actions.ActionsService, logger *logr.Logger, sessio
 	}
 }
 
-func (m *SessionRefreshingClient) GetMessage(ctx context.Context, lastMessageId int64) (*actions.RunnerScaleSetMessage, error) {
-	message, err := m.client.GetMessage(ctx, m.session.MessageQueueUrl, m.session.MessageQueueAccessToken, lastMessageId)
+func (m *SessionRefreshingClient) GetMessage(ctx context.Context, lastMessageId int64, maxCapacity int) (*actions.RunnerScaleSetMessage, error) {
+	if maxCapacity < 0 {
+		return nil, fmt.Errorf("maxCapacity must be greater than or equal to 0")
+	}
+
+	message, err := m.client.GetMessage(ctx, m.session.MessageQueueUrl, m.session.MessageQueueAccessToken, lastMessageId, maxCapacity)
 	if err == nil {
 		return message, nil
 	}
@@ -42,7 +46,7 @@ func (m *SessionRefreshingClient) GetMessage(ctx context.Context, lastMessageId
 	}
 
 	m.session = session
-	message, err = m.client.GetMessage(ctx, m.session.MessageQueueUrl, m.session.MessageQueueAccessToken, lastMessageId)
+	message, err = m.client.GetMessage(ctx, m.session.MessageQueueUrl, m.session.MessageQueueAccessToken, lastMessageId, maxCapacity)
 	if err != nil {
 		return nil, fmt.Errorf("delete message failed after refresh message session. %w", err)
 	}

cmd/githubrunnerscalesetlistener/sessionrefreshingclient_test.go

@@ -31,17 +31,17 @@ func TestGetMessage(t *testing.T) {
 		},
 	}
 
-	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0)).Return(nil, nil).Once()
-	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0)).Return(&actions.RunnerScaleSetMessage{MessageId: 1}, nil).Once()
+	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0), 10).Return(nil, nil).Once()
+	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0), 10).Return(&actions.RunnerScaleSetMessage{MessageId: 1}, nil).Once()
 
 	client := newSessionClient(mockActionsClient, &logger, session)
 
-	msg, err := client.GetMessage(ctx, 0)
+	msg, err := client.GetMessage(ctx, 0, 10)
 	require.NoError(t, err, "GetMessage should not return an error")
 
 	assert.Nil(t, msg, "GetMessage should return nil message")
 
-	msg, err = client.GetMessage(ctx, 0)
+	msg, err = client.GetMessage(ctx, 0, 10)
 	require.NoError(t, err, "GetMessage should not return an error")
 
 	assert.Equal(t, int64(1), msg.MessageId, "GetMessage should return a message with id 1")
@@ -146,11 +146,11 @@ func TestGetMessage_Error(t *testing.T) {
 		},
 	}
 
-	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0)).Return(nil, fmt.Errorf("error")).Once()
+	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0), 10).Return(nil, fmt.Errorf("error")).Once()
 
 	client := newSessionClient(mockActionsClient, &logger, session)
 
-	msg, err := client.GetMessage(ctx, 0)
+	msg, err := client.GetMessage(ctx, 0, 10)
 	assert.ErrorContains(t, err, "get message failed. error", "GetMessage should return an error")
 	assert.Nil(t, msg, "GetMessage should return nil message")
 	assert.True(t, mockActionsClient.AssertExpectations(t), "All expected calls to mockActionsClient should have been made")
@@ -227,8 +227,8 @@ func TestGetMessage_RefreshToken(t *testing.T) {
 			Id: 1,
 		},
 	}
-	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0)).Return(nil, &actions.MessageQueueTokenExpiredError{}).Once()
-	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, "token2", int64(0)).Return(&actions.RunnerScaleSetMessage{
+	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0), 10).Return(nil, &actions.MessageQueueTokenExpiredError{}).Once()
+	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, "token2", int64(0), 10).Return(&actions.RunnerScaleSetMessage{
 		MessageId:   1,
 		MessageType: "test",
 		Body:        "test",
@@ -243,7 +243,7 @@ func TestGetMessage_RefreshToken(t *testing.T) {
 	}, nil).Once()
 
 	client := newSessionClient(mockActionsClient, &logger, session)
-	msg, err := client.GetMessage(ctx, 0)
+	msg, err := client.GetMessage(ctx, 0, 10)
 	assert.NoError(t, err, "Error getting message")
 	assert.Equal(t, int64(1), msg.MessageId, "message id should be updated")
 	assert.Equal(t, "token2", client.session.MessageQueueAccessToken, "Message queue access token should be updated")
@@ -340,11 +340,11 @@ func TestGetMessage_RefreshToken_Failed(t *testing.T) {
 			Id: 1,
 		},
 	}
-	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0)).Return(nil, &actions.MessageQueueTokenExpiredError{}).Once()
+	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0), 10).Return(nil, &actions.MessageQueueTokenExpiredError{}).Once()
 	mockActionsClient.On("RefreshMessageSession", ctx, session.RunnerScaleSet.Id, session.SessionId).Return(nil, fmt.Errorf("error"))
 
 	client := newSessionClient(mockActionsClient, &logger, session)
-	msg, err := client.GetMessage(ctx, 0)
+	msg, err := client.GetMessage(ctx, 0, 10)
 	assert.ErrorContains(t, err, "refresh message session failed. error", "Error should be returned")
 	assert.Nil(t, msg, "Message should be nil")
 	assert.Equal(t, "token", client.session.MessageQueueAccessToken, "Message queue access token should not be updated")

controllers/actions.github.com/autoscalinglistener_controller.go

@@ -690,30 +690,6 @@ func (r *AutoscalingListenerReconciler) publishRunningListener(autoscalingListen
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	groupVersionIndexer := func(rawObj client.Object) []string {
-		groupVersion := v1alpha1.GroupVersion.String()
-		owner := metav1.GetControllerOf(rawObj)
-		if owner == nil {
-			return nil
-		}
-
-		// ...make sure it is owned by this controller
-		if owner.APIVersion != groupVersion || owner.Kind != "AutoscalingListener" {
-			return nil
-		}
-
-		// ...and if so, return it
-		return []string{owner.Name}
-	}
-
-	if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Pod{}, resourceOwnerKey, groupVersionIndexer); err != nil {
-		return err
-	}
-
-	if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.ServiceAccount{}, resourceOwnerKey, groupVersionIndexer); err != nil {
-		return err
-	}
-
 	labelBasedWatchFunc := func(_ context.Context, obj client.Object) []reconcile.Request {
 		var requests []reconcile.Request
 		labels := obj.GetLabels()

controllers/actions.github.com/autoscalinglistener_controller_test.go

@@ -21,7 +21,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
-	actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
+	"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
 )
 
 const (
@@ -34,9 +34,9 @@ var _ = Describe("Test AutoScalingListener controller", func() {
 	var ctx context.Context
 	var mgr ctrl.Manager
 	var autoscalingNS *corev1.Namespace
-	var autoscalingRunnerSet *actionsv1alpha1.AutoscalingRunnerSet
+	var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
 	var configSecret *corev1.Secret
-	var autoscalingListener *actionsv1alpha1.AutoscalingListener
+	var autoscalingListener *v1alpha1.AutoscalingListener
 
 	BeforeEach(func() {
 		ctx = context.Background()
@@ -53,12 +53,12 @@ var _ = Describe("Test AutoScalingListener controller", func() {
 
 		min := 1
 		max := 10
-		autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{
+		autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "test-asrs",
 				Namespace: autoscalingNS.Name,
 			},
-			Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{
+			Spec: v1alpha1.AutoscalingRunnerSetSpec{
 				GitHubConfigUrl:    "https://github.com/owner/repo",
 				GitHubConfigSecret: configSecret.Name,
 				MaxRunners:         &max,
@@ -79,12 +79,12 @@ var _ = Describe("Test AutoScalingListener controller", func() {
 		err = k8sClient.Create(ctx, autoscalingRunnerSet)
 		Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
 
-		autoscalingListener = &actionsv1alpha1.AutoscalingListener{
+		autoscalingListener = &v1alpha1.AutoscalingListener{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "test-asl",
 				Namespace: autoscalingNS.Name,
 			},
-			Spec: actionsv1alpha1.AutoscalingListenerSpec{
+			Spec: v1alpha1.AutoscalingListenerSpec{
 				GitHubConfigUrl:               "https://github.com/owner/repo",
 				GitHubConfigSecret:            configSecret.Name,
 				RunnerScaleSetId:              1,
@@ -119,7 +119,7 @@ var _ = Describe("Test AutoScalingListener controller", func() {
 			).Should(Succeed(), "Config secret should be created")
 
 			// Check if finalizer is added
-			created := new(actionsv1alpha1.AutoscalingListener)
+			created := new(v1alpha1.AutoscalingListener)
 			Eventually(
 				func() (string, error) {
 					err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, created)
@@ -298,7 +298,7 @@ var _ = Describe("Test AutoScalingListener controller", func() {
 			// The AutoScalingListener should be deleted
 			Eventually(
 				func() error {
-					listenerList := new(actionsv1alpha1.AutoscalingListenerList)
+					listenerList := new(v1alpha1.AutoscalingListenerList)
 					err := k8sClient.List(ctx, listenerList, client.InNamespace(autoscalingListener.Namespace), client.MatchingFields{".metadata.name": autoscalingListener.Name})
 					if err != nil {
 						return err
@@ -415,9 +415,9 @@ var _ = Describe("Test AutoScalingListener customization", func() {
 	var ctx context.Context
 	var mgr ctrl.Manager
 	var autoscalingNS *corev1.Namespace
-	var autoscalingRunnerSet *actionsv1alpha1.AutoscalingRunnerSet
+	var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
 	var configSecret *corev1.Secret
-	var autoscalingListener *actionsv1alpha1.AutoscalingListener
+	var autoscalingListener *v1alpha1.AutoscalingListener
 
 	var runAsUser int64 = 1001
 
@@ -458,12 +458,12 @@ var _ = Describe("Test AutoScalingListener customization", func() {
 
 		min := 1
 		max := 10
-		autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{
+		autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "test-asrs",
 				Namespace: autoscalingNS.Name,
 			},
-			Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{
+			Spec: v1alpha1.AutoscalingRunnerSetSpec{
 				GitHubConfigUrl:    "https://github.com/owner/repo",
 				GitHubConfigSecret: configSecret.Name,
 				MaxRunners:         &max,
@@ -484,12 +484,12 @@ var _ = Describe("Test AutoScalingListener customization", func() {
 		err = k8sClient.Create(ctx, autoscalingRunnerSet)
 		Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
 
-		autoscalingListener = &actionsv1alpha1.AutoscalingListener{
+		autoscalingListener = &v1alpha1.AutoscalingListener{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "test-asltest",
 				Namespace: autoscalingNS.Name,
 			},
-			Spec: actionsv1alpha1.AutoscalingListenerSpec{
+			Spec: v1alpha1.AutoscalingListenerSpec{
 				GitHubConfigUrl:               "https://github.com/owner/repo",
 				GitHubConfigSecret:            configSecret.Name,
 				RunnerScaleSetId:              1,
@@ -512,7 +512,7 @@ var _ = Describe("Test AutoScalingListener customization", func() {
 	Context("When creating a new AutoScalingListener", func() {
 		It("It should create customized pod with applied configuration", func() {
 			// Check if finalizer is added
-			created := new(actionsv1alpha1.AutoscalingListener)
+			created := new(v1alpha1.AutoscalingListener)
 			Eventually(
 				func() (string, error) {
 					err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, created)
@@ -570,19 +570,19 @@ var _ = Describe("Test AutoScalingListener controller with proxy", func() {
 	var ctx context.Context
 	var mgr ctrl.Manager
 	var autoscalingNS *corev1.Namespace
-	var autoscalingRunnerSet *actionsv1alpha1.AutoscalingRunnerSet
+	var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
 	var configSecret *corev1.Secret
-	var autoscalingListener *actionsv1alpha1.AutoscalingListener
+	var autoscalingListener *v1alpha1.AutoscalingListener
 
-	createRunnerSetAndListener := func(proxy *actionsv1alpha1.ProxyConfig) {
+	createRunnerSetAndListener := func(proxy *v1alpha1.ProxyConfig) {
 		min := 1
 		max := 10
-		autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{
+		autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "test-asrs",
 				Namespace: autoscalingNS.Name,
 			},
-			Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{
+			Spec: v1alpha1.AutoscalingRunnerSetSpec{
 				GitHubConfigUrl:    "https://github.com/owner/repo",
 				GitHubConfigSecret: configSecret.Name,
 				MaxRunners:         &max,
@@ -604,12 +604,12 @@ var _ = Describe("Test AutoScalingListener controller with proxy", func() {
 		err := k8sClient.Create(ctx, autoscalingRunnerSet)
 		Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
 
-		autoscalingListener = &actionsv1alpha1.AutoscalingListener{
+		autoscalingListener = &v1alpha1.AutoscalingListener{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "test-asl",
 				Namespace: autoscalingNS.Name,
 			},
-			Spec: actionsv1alpha1.AutoscalingListenerSpec{
+			Spec: v1alpha1.AutoscalingListenerSpec{
 				GitHubConfigUrl:               "https://github.com/owner/repo",
 				GitHubConfigSecret:            configSecret.Name,
 				RunnerScaleSetId:              1,
@@ -658,12 +658,12 @@ var _ = Describe("Test AutoScalingListener controller with proxy", func() {
 		err := k8sClient.Create(ctx, proxyCredentials)
 		Expect(err).NotTo(HaveOccurred(), "failed to create proxy credentials secret")
 
-		proxy := &actionsv1alpha1.ProxyConfig{
-			HTTP: &actionsv1alpha1.ProxyServerConfig{
+		proxy := &v1alpha1.ProxyConfig{
+			HTTP: &v1alpha1.ProxyServerConfig{
 				Url:                 "http://localhost:8080",
 				CredentialSecretRef: "proxy-credentials",
 			},
-			HTTPS: &actionsv1alpha1.ProxyServerConfig{
+			HTTPS: &v1alpha1.ProxyServerConfig{
 				Url:                 "https://localhost:8443",
 				CredentialSecretRef: "proxy-credentials",
 			},
@@ -766,19 +766,19 @@ var _ = Describe("Test AutoScalingListener controller with template modification
 	var ctx context.Context
 	var mgr ctrl.Manager
 	var autoscalingNS *corev1.Namespace
-	var autoscalingRunnerSet *actionsv1alpha1.AutoscalingRunnerSet
+	var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
 	var configSecret *corev1.Secret
-	var autoscalingListener *actionsv1alpha1.AutoscalingListener
+	var autoscalingListener *v1alpha1.AutoscalingListener
 
 	createRunnerSetAndListener := func(listenerTemplate *corev1.PodTemplateSpec) {
 		min := 1
 		max := 10
-		autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{
+		autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "test-asrs",
 				Namespace: autoscalingNS.Name,
 			},
-			Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{
+			Spec: v1alpha1.AutoscalingRunnerSetSpec{
 				GitHubConfigUrl:    "https://github.com/owner/repo",
 				GitHubConfigSecret: configSecret.Name,
 				MaxRunners:         &max,
@@ -800,12 +800,12 @@ var _ = Describe("Test AutoScalingListener controller with template modification
 		err := k8sClient.Create(ctx, autoscalingRunnerSet)
 		Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
 
-		autoscalingListener = &actionsv1alpha1.AutoscalingListener{
+		autoscalingListener = &v1alpha1.AutoscalingListener{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "test-asl",
 				Namespace: autoscalingNS.Name,
 			},
-			Spec: actionsv1alpha1.AutoscalingListenerSpec{
+			Spec: v1alpha1.AutoscalingListenerSpec{
 				GitHubConfigUrl:               "https://github.com/owner/repo",
 				GitHubConfigSecret:            configSecret.Name,
 				RunnerScaleSetId:              1,
@@ -915,9 +915,9 @@ var _ = Describe("Test GitHub Server TLS configuration", func() {
 	var ctx context.Context
 	var mgr ctrl.Manager
 	var autoscalingNS *corev1.Namespace
-	var autoscalingRunnerSet *actionsv1alpha1.AutoscalingRunnerSet
+	var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
 	var configSecret *corev1.Secret
-	var autoscalingListener *actionsv1alpha1.AutoscalingListener
+	var autoscalingListener *v1alpha1.AutoscalingListener
 	var rootCAConfigMap *corev1.ConfigMap
 
 	BeforeEach(func() {
@@ -955,16 +955,16 @@ var _ = Describe("Test GitHub Server TLS configuration", func() {
 
 		min := 1
 		max := 10
-		autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{
+		autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "test-asrs",
 				Namespace: autoscalingNS.Name,
 			},
-			Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{
+			Spec: v1alpha1.AutoscalingRunnerSetSpec{
 				GitHubConfigUrl:    "https://github.com/owner/repo",
 				GitHubConfigSecret: configSecret.Name,
-				GitHubServerTLS: &actionsv1alpha1.GitHubServerTLSConfig{
-					CertificateFrom: &actionsv1alpha1.TLSCertificateSource{
+				GitHubServerTLS: &v1alpha1.GitHubServerTLSConfig{
+					CertificateFrom: &v1alpha1.TLSCertificateSource{
 						ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
 							LocalObjectReference: corev1.LocalObjectReference{
 								Name: rootCAConfigMap.Name,
@@ -991,16 +991,16 @@ var _ = Describe("Test GitHub Server TLS configuration", func() {
 		err = k8sClient.Create(ctx, autoscalingRunnerSet)
 		Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
 
-		autoscalingListener = &actionsv1alpha1.AutoscalingListener{
+		autoscalingListener = &v1alpha1.AutoscalingListener{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "test-asl",
 				Namespace: autoscalingNS.Name,
 			},
-			Spec: actionsv1alpha1.AutoscalingListenerSpec{
+			Spec: v1alpha1.AutoscalingListenerSpec{
 				GitHubConfigUrl:    "https://github.com/owner/repo",
 				GitHubConfigSecret: configSecret.Name,
-				GitHubServerTLS: &actionsv1alpha1.GitHubServerTLSConfig{
-					CertificateFrom: &actionsv1alpha1.TLSCertificateSource{
+				GitHubServerTLS: &v1alpha1.GitHubServerTLSConfig{
+					CertificateFrom: &v1alpha1.TLSCertificateSource{
 						ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
 							LocalObjectReference: corev1.LocalObjectReference{
 								Name: rootCAConfigMap.Name,

controllers/actions.github.com/autoscalingrunnerset_controller.go

@@ -30,7 +30,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -135,17 +134,11 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
 			return ctrl.Result{}, err
 		}
 
-		requeue, err := r.removeFinalizersFromDependentResources(ctx, autoscalingRunnerSet, log)
-		if err != nil {
+		if err := r.removeFinalizersFromDependentResources(ctx, autoscalingRunnerSet, log); err != nil {
 			log.Error(err, "Failed to remove finalizers on dependent resources")
 			return ctrl.Result{}, err
 		}
 
-		if requeue {
-			log.Info("Waiting for dependent resources to be deleted")
-			return ctrl.Result{Requeue: true}, nil
-		}
-
 		log.Info("Removing finalizer")
 		err = patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) {
 			controllerutil.RemoveFinalizer(obj, autoscalingRunnerSetFinalizerName)
@@ -277,6 +270,7 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
 			// need to scale down to 0
 			err := patch(ctx, r.Client, latestRunnerSet, func(obj *v1alpha1.EphemeralRunnerSet) {
 				obj.Spec.Replicas = 0
+				obj.Spec.PatchID = 0
 			})
 			if err != nil {
 				log.Error(err, "Failed to patch runner set to set desired count to 0")
@@ -389,7 +383,7 @@ func (r *AutoscalingRunnerSetReconciler) deleteEphemeralRunnerSets(ctx context.C
 	return nil
 }
 
-func (r *AutoscalingRunnerSetReconciler) removeFinalizersFromDependentResources(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, logger logr.Logger) (requeue bool, err error) {
+func (r *AutoscalingRunnerSetReconciler) removeFinalizersFromDependentResources(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, logger logr.Logger) error {
 	c := autoscalingRunnerSetFinalizerDependencyCleaner{
 		client:               r.Client,
 		autoscalingRunnerSet: autoscalingRunnerSet,
@@ -404,12 +398,7 @@ func (r *AutoscalingRunnerSetReconciler) removeFinalizersFromDependentResources(
 	c.removeManagerRoleBindingFinalizer(ctx)
 	c.removeManagerRoleFinalizer(ctx)
 
-	requeue, err = c.result()
-	if err != nil {
-		logger.Error(err, "Failed to cleanup finalizer from dependent resource")
-		return true, err
-	}
-	return requeue, nil
+	return c.Err()
 }
 
 func (r *AutoscalingRunnerSetReconciler) createRunnerScaleSet(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, logger logr.Logger) (ctrl.Result, error) {
@@ -758,26 +747,6 @@ func (r *AutoscalingRunnerSetReconciler) actionsClientOptionsFor(ctx context.Con
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *AutoscalingRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	groupVersionIndexer := func(rawObj client.Object) []string {
-		groupVersion := v1alpha1.GroupVersion.String()
-		owner := metav1.GetControllerOf(rawObj)
-		if owner == nil {
-			return nil
-		}
-
-		// ...make sure it is owned by this controller
-		if owner.APIVersion != groupVersion || owner.Kind != "AutoscalingRunnerSet" {
-			return nil
-		}
-
-		// ...and if so, return it
-		return []string{owner.Name}
-	}
-
-	if err := mgr.GetFieldIndexer().IndexField(context.Background(), &v1alpha1.EphemeralRunnerSet{}, resourceOwnerKey, groupVersionIndexer); err != nil {
-		return err
-	}
-
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.AutoscalingRunnerSet{}).
 		Owns(&v1alpha1.EphemeralRunnerSet{}).
@@ -804,17 +773,16 @@ type autoscalingRunnerSetFinalizerDependencyCleaner struct {
 	autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
 	logger               logr.Logger
 
-	// fields to operate on
-	requeue bool
-	err     error
+	err error
 }
 
-func (c *autoscalingRunnerSetFinalizerDependencyCleaner) result() (requeue bool, err error) {
-	return c.requeue, c.err
+func (c *autoscalingRunnerSetFinalizerDependencyCleaner) Err() error {
+	return c.err
 }
 
 func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRoleBindingFinalizer(ctx context.Context) {
-	if c.requeue || c.err != nil {
+	if c.err != nil {
+		c.logger.Info("Skipping cleaning up kubernetes mode service account")
 		return
 	}
 
@@ -845,7 +813,6 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRol
 			c.err = fmt.Errorf("failed to patch kubernetes mode role binding without finalizer: %w", err)
 			return
 		}
-		c.requeue = true
 		c.logger.Info("Removed finalizer from container mode kubernetes role binding", "name", roleBindingName)
 		return
 	case err != nil && !kerrors.IsNotFound(err):
@@ -858,7 +825,7 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRol
 }
 
 func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRoleFinalizer(ctx context.Context) {
-	if c.requeue || c.err != nil {
+	if c.err != nil {
 		return
 	}
 
@@ -888,7 +855,6 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRol
 			c.err = fmt.Errorf("failed to patch kubernetes mode role without finalizer: %w", err)
 			return
 		}
-		c.requeue = true
 		c.logger.Info("Removed finalizer from container mode kubernetes role")
 		return
 	case err != nil && !kerrors.IsNotFound(err):
@@ -901,7 +867,7 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRol
 }
 
 func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeServiceAccountFinalizer(ctx context.Context) {
-	if c.requeue || c.err != nil {
+	if c.err != nil {
 		return
 	}
 
@@ -932,7 +898,6 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeSer
 			c.err = fmt.Errorf("failed to patch kubernetes mode service account without finalizer: %w", err)
 			return
 		}
-		c.requeue = true
 		c.logger.Info("Removed finalizer from container mode kubernetes service account")
 		return
 	case err != nil && !kerrors.IsNotFound(err):
@@ -945,7 +910,7 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeSer
 }
 
 func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeNoPermissionServiceAccountFinalizer(ctx context.Context) {
-	if c.requeue || c.err != nil {
+	if c.err != nil {
 		return
 	}
 
@@ -976,7 +941,6 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeNoPermissionServi
 			c.err = fmt.Errorf("failed to patch service account without finalizer: %w", err)
 			return
 		}
-		c.requeue = true
 		c.logger.Info("Removed finalizer from no permission service account", "name", serviceAccountName)
 		return
 	case err != nil && !kerrors.IsNotFound(err):
@@ -989,7 +953,7 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeNoPermissionServi
 }
 
 func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeGitHubSecretFinalizer(ctx context.Context) {
-	if c.requeue || c.err != nil {
+	if c.err != nil {
 		return
 	}
 
@@ -1020,7 +984,6 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeGitHubSecretFinal
 			c.err = fmt.Errorf("failed to patch GitHub secret without finalizer: %w", err)
 			return
 		}
-		c.requeue = true
 		c.logger.Info("Removed finalizer from GitHub secret", "name", githubSecretName)
 		return
 	case err != nil && !kerrors.IsNotFound(err) && !kerrors.IsForbidden(err):
@@ -1033,7 +996,7 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeGitHubSecretFinal
 }
 
 func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleBindingFinalizer(ctx context.Context) {
-	if c.requeue || c.err != nil {
+	if c.err != nil {
 		return
 	}
 
@@ -1064,7 +1027,6 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleBindin
 			c.err = fmt.Errorf("failed to patch manager role binding without finalizer: %w", err)
 			return
 		}
-		c.requeue = true
 		c.logger.Info("Removed finalizer from manager role binding", "name", managerRoleBindingName)
 		return
 	case err != nil && !kerrors.IsNotFound(err):
@@ -1077,7 +1039,7 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleBindin
 }
 
 func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleFinalizer(ctx context.Context) {
-	if c.requeue || c.err != nil {
+	if c.err != nil {
 		return
 	}
 
@@ -1108,7 +1070,6 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleFinali
 			c.err = fmt.Errorf("failed to patch manager role without finalizer: %w", err)
 			return
 		}
-		c.requeue = true
 		c.logger.Info("Removed finalizer from manager role", "name", managerRoleName)
 		return
 	case err != nil && !kerrors.IsNotFound(err):

controllers/actions.github.com/ephemeralrunner_controller.go

@@ -21,7 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
@@ -295,14 +294,17 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 }
 
 func (r *EphemeralRunnerReconciler) cleanupRunnerFromService(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner, log logr.Logger) (ctrl.Result, error) {
-	actionsError := &actions.ActionsError{}
-	err := r.deleteRunnerFromService(ctx, ephemeralRunner, log)
-	if err != nil {
-		if errors.As(err, &actionsError) &&
-			actionsError.StatusCode == http.StatusBadRequest &&
-			strings.Contains(actionsError.ExceptionName, "JobStillRunningException") {
+	if err := r.deleteRunnerFromService(ctx, ephemeralRunner, log); err != nil {
+		actionsError := &actions.ActionsError{}
+		if !errors.As(err, &actionsError) {
+			log.Error(err, "Failed to clean up runner from the service (not an ActionsError)")
+			return ctrl.Result{}, err
+		}
+
+		if actionsError.StatusCode == http.StatusBadRequest && actionsError.IsException("JobStillRunningException") {
 			log.Info("Runner is still running the job. Re-queue in 30 seconds")
 			return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
+
 		}
 
 		log.Error(err, "Failed clean up runner from the service")
@@ -310,10 +312,9 @@ func (r *EphemeralRunnerReconciler) cleanupRunnerFromService(ctx context.Context
 	}
 
 	log.Info("Successfully removed runner registration from service")
-	err = patch(ctx, r.Client, ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) {
+	if err := patch(ctx, r.Client, ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) {
 		controllerutil.RemoveFinalizer(obj, ephemeralRunnerActionsFinalizerName)
-	})
-	if err != nil {
+	}); err != nil {
 		return ctrl.Result{}, err
 	}
 
@@ -528,7 +529,7 @@ func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Con
 		}
 
 		if actionsError.StatusCode != http.StatusConflict ||
-			!strings.Contains(actionsError.ExceptionName, "AgentExistsException") {
+			!actionsError.IsException("AgentExistsException") {
 			return ctrl.Result{}, fmt.Errorf("failed to generate JIT config with Actions service error: %v", err)
 		}
 
@@ -784,7 +785,7 @@ func (r EphemeralRunnerReconciler) runnerRegisteredWithService(ctx context.Conte
 		}
 
 		if actionsError.StatusCode != http.StatusNotFound ||
-			!strings.Contains(actionsError.ExceptionName, "AgentNotFoundException") {
+			!actionsError.IsException("AgentNotFoundException") {
 			return false, fmt.Errorf("failed to check if runner exists in GitHub service: %v", err)
 		}
 
@@ -814,7 +815,6 @@ func (r *EphemeralRunnerReconciler) deleteRunnerFromService(ctx context.Context,
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *EphemeralRunnerReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	// TODO(nikola-jokic): Add indexing and filtering fields on corev1.Pod{}
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.EphemeralRunner{}).
 		Owns(&corev1.Pod{}).

controllers/actions.github.com/ephemeralrunner_controller_test.go

@@ -671,8 +671,10 @@ var _ = Describe("EphemeralRunner", func() {
 							fake.WithGetRunner(
 								nil,
 								&actions.ActionsError{
-									StatusCode:    http.StatusNotFound,
-									ExceptionName: "AgentNotFoundException",
+									StatusCode: http.StatusNotFound,
+									Err: &actions.ActionsExceptionError{
+										ExceptionName: "AgentNotFoundException",
+									},
 								},
 							),
 						),

controllers/actions.github.com/ephemeralrunnerset_controller.go

@@ -23,7 +23,6 @@ import (
 	"net/http"
 	"sort"
 	"strconv"
-	"strings"
 
 	"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
 	"github.com/actions/actions-runner-controller/controllers/actions.github.com/metrics"
@@ -197,7 +196,6 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
 				log.Error(err, "failed to cleanup finished ephemeral runners")
 			}
 		}()
-
 		log.Info("Scaling comparison", "current", total, "desired", ephemeralRunnerSet.Spec.Replicas)
 		switch {
 		case total < ephemeralRunnerSet.Spec.Replicas: // Handle scale up
@@ -208,7 +206,12 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
 				return ctrl.Result{}, err
 			}
 
-		case total > ephemeralRunnerSet.Spec.Replicas: // Handle scale down scenario.
+		case ephemeralRunnerSet.Spec.PatchID > 0 && total >= ephemeralRunnerSet.Spec.Replicas: // Handle scale down scenario.
+			// If ephemeral runner did not yet update the phase to succeeded, but the scale down
+			// request is issued, we should ignore the scale down request.
+			// Eventually, the ephemeral runner will be cleaned up on the next patch request, which happens
+			// on the next batch
+		case ephemeralRunnerSet.Spec.PatchID == 0 && total > ephemeralRunnerSet.Spec.Replicas:
 			count := total - ephemeralRunnerSet.Spec.Replicas
 			log.Info("Deleting ephemeral runners (scale down)", "count", count)
 			if err := r.deleteIdleEphemeralRunners(
@@ -428,6 +431,9 @@ func (r *EphemeralRunnerSetReconciler) createProxySecret(ctx context.Context, ep
 // When this happens, the next reconcile loop will try to delete the remaining ephemeral runners
 // after we get notified by any of the `v1alpha1.EphemeralRunner.Status` updates.
 func (r *EphemeralRunnerSetReconciler) deleteIdleEphemeralRunners(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, pendingEphemeralRunners, runningEphemeralRunners []*v1alpha1.EphemeralRunner, count int, log logr.Logger) error {
+	if count <= 0 {
+		return nil
+	}
 	runners := newEphemeralRunnerStepper(pendingEphemeralRunners, runningEphemeralRunners)
 	if runners.len() == 0 {
 		log.Info("No pending or running ephemeral runners running at this time for scale down")
@@ -473,10 +479,14 @@ func (r *EphemeralRunnerSetReconciler) deleteIdleEphemeralRunners(ctx context.Co
 func (r *EphemeralRunnerSetReconciler) deleteEphemeralRunnerWithActionsClient(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner, actionsClient actions.ActionsService, log logr.Logger) (bool, error) {
 	if err := actionsClient.RemoveRunner(ctx, int64(ephemeralRunner.Status.RunnerId)); err != nil {
 		actionsError := &actions.ActionsError{}
-		if errors.As(err, &actionsError) &&
-			actionsError.StatusCode == http.StatusBadRequest &&
-			strings.Contains(actionsError.ExceptionName, "JobStillRunningException") {
-			// Runner is still running a job, proceed with the next one
+		if !errors.As(err, &actionsError) {
+			log.Error(err, "failed to remove runner from the service", "name", ephemeralRunner.Name, "runnerId", ephemeralRunner.Status.RunnerId)
+			return false, err
+		}
+
+		if actionsError.StatusCode == http.StatusBadRequest &&
+			actionsError.IsException("JobStillRunningException") {
+			log.Info("Runner is still running a job, skipping deletion", "name", ephemeralRunner.Name, "runnerId", ephemeralRunner.Status.RunnerId)
 			return false, nil
 		}
 
@@ -561,28 +571,6 @@ func (r *EphemeralRunnerSetReconciler) actionsClientOptionsFor(ctx context.Conte
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *EphemeralRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	// Index EphemeralRunner owned by EphemeralRunnerSet so we can perform faster look ups.
-	if err := mgr.GetFieldIndexer().IndexField(context.Background(), &v1alpha1.EphemeralRunner{}, resourceOwnerKey, func(rawObj client.Object) []string {
-		groupVersion := v1alpha1.GroupVersion.String()
-
-		// grab the job object, extract the owner...
-		ephemeralRunner := rawObj.(*v1alpha1.EphemeralRunner)
-		owner := metav1.GetControllerOf(ephemeralRunner)
-		if owner == nil {
-			return nil
-		}
-
-		// ...make sure it is owned by this controller
-		if owner.APIVersion != groupVersion || owner.Kind != "EphemeralRunnerSet" {
-			return nil
-		}
-
-		// ...and if so, return it
-		return []string{owner.Name}
-	}); err != nil {
-		return err
-	}
-
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.EphemeralRunnerSet{}).
 		Owns(&v1alpha1.EphemeralRunner{}).

controllers/actions.github.com/helpers_test.go

@@ -18,6 +18,9 @@ const defaultGitHubToken = "gh_token"
 
 func startManagers(t ginkgo.GinkgoTInterface, first manager.Manager, others ...manager.Manager) {
 	for _, mgr := range append([]manager.Manager{first}, others...) {
+		if err := SetupIndexers(mgr); err != nil {
+			t.Fatalf("failed to setup indexers: %v", err)
+		}
 		ctx, cancel := context.WithCancel(context.Background())
 
 		g, ctx := errgroup.WithContext(ctx)

controllers/actions.github.com/indexer.go

@@ -0,0 +1,71 @@
+package actionsgithubcom
+
+import (
+	"context"
+	"slices"
+
+	v1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func SetupIndexers(mgr ctrl.Manager) error {
+	if err := mgr.GetFieldIndexer().IndexField(
+		context.Background(),
+		&corev1.Pod{},
+		resourceOwnerKey,
+		newGroupVersionOwnerKindIndexer("AutoscalingListener", "EphemeralRunner"),
+	); err != nil {
+		return err
+	}
+
+	if err := mgr.GetFieldIndexer().IndexField(
+		context.Background(),
+		&corev1.ServiceAccount{},
+		resourceOwnerKey,
+		newGroupVersionOwnerKindIndexer("AutoscalingListener"),
+	); err != nil {
+		return err
+	}
+
+	if err := mgr.GetFieldIndexer().IndexField(
+		context.Background(),
+		&v1alpha1.EphemeralRunnerSet{},
+		resourceOwnerKey,
+		newGroupVersionOwnerKindIndexer("AutoscalingRunnerSet"),
+	); err != nil {
+		return err
+	}
+
+	if err := mgr.GetFieldIndexer().IndexField(
+		context.Background(),
+		&v1alpha1.EphemeralRunner{},
+		resourceOwnerKey,
+		newGroupVersionOwnerKindIndexer("EphemeralRunnerSet"),
+	); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func newGroupVersionOwnerKindIndexer(ownerKind string, otherOwnerKinds ...string) client.IndexerFunc {
+	owners := append([]string{ownerKind}, otherOwnerKinds...)
+	return func(o client.Object) []string {
+		groupVersion := v1alpha1.GroupVersion.String()
+		owner := metav1.GetControllerOfNoCopy(o)
+		if owner == nil {
+			return nil
+		}
+
+		// ...make sure it is owned by this controller
+		if owner.APIVersion != groupVersion || !slices.Contains(owners, owner.Kind) {
+			return nil
+		}
+
+		// ...and if so, return it
+		return []string{owner.Name}
+	}
+}

controllers/actions.github.com/resourcebuilder.go

@@ -85,13 +85,13 @@ func (b *resourceBuilder) newAutoScalingListener(autoscalingRunnerSet *v1alpha1.
 		effectiveMinRunners = *autoscalingRunnerSet.Spec.MinRunners
 	}
 
-	labels := map[string]string{
+	labels := mergeLabels(autoscalingRunnerSet.Labels, map[string]string{
 		LabelKeyGitHubScaleSetNamespace: autoscalingRunnerSet.Namespace,
 		LabelKeyGitHubScaleSetName:      autoscalingRunnerSet.Name,
 		LabelKeyKubernetesPartOf:        labelValueKubernetesPartOf,
 		LabelKeyKubernetesComponent:     "runner-scale-set-listener",
 		LabelKeyKubernetesVersion:       autoscalingRunnerSet.Labels[LabelKeyKubernetesVersion],
-	}
+	})
 
 	annotations := map[string]string{
 		annotationKeyRunnerSpecHash: autoscalingRunnerSet.ListenerSpecHash(),
@@ -411,10 +411,10 @@ func (b *resourceBuilder) newScaleSetListenerServiceAccount(autoscalingListener
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      scaleSetListenerServiceAccountName(autoscalingListener),
 			Namespace: autoscalingListener.Namespace,
-			Labels: map[string]string{
+			Labels: mergeLabels(autoscalingListener.Labels, map[string]string{
 				LabelKeyGitHubScaleSetNamespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
 				LabelKeyGitHubScaleSetName:      autoscalingListener.Spec.AutoscalingRunnerSetName,
-			},
+			}),
 		},
 	}
 }
@@ -426,13 +426,13 @@ func (b *resourceBuilder) newScaleSetListenerRole(autoscalingListener *v1alpha1.
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      scaleSetListenerRoleName(autoscalingListener),
 			Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
-			Labels: map[string]string{
+			Labels: mergeLabels(autoscalingListener.Labels, map[string]string{
 				LabelKeyGitHubScaleSetNamespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
 				LabelKeyGitHubScaleSetName:      autoscalingListener.Spec.AutoscalingRunnerSetName,
 				labelKeyListenerNamespace:       autoscalingListener.Namespace,
 				labelKeyListenerName:            autoscalingListener.Name,
 				"role-policy-rules-hash":        rulesHash,
-			},
+			}),
 		},
 		Rules: rules,
 	}
@@ -460,14 +460,14 @@ func (b *resourceBuilder) newScaleSetListenerRoleBinding(autoscalingListener *v1
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      scaleSetListenerRoleName(autoscalingListener),
 			Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
-			Labels: map[string]string{
+			Labels: mergeLabels(autoscalingListener.Labels, map[string]string{
 				LabelKeyGitHubScaleSetNamespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
 				LabelKeyGitHubScaleSetName:      autoscalingListener.Spec.AutoscalingRunnerSetName,
 				labelKeyListenerNamespace:       autoscalingListener.Namespace,
 				labelKeyListenerName:            autoscalingListener.Name,
 				"role-binding-role-ref-hash":    roleRefHash,
 				"role-binding-subject-hash":     subjectHash,
-			},
+			}),
 		},
 		RoleRef:  roleRef,
 		Subjects: subjects,
@@ -483,11 +483,11 @@ func (b *resourceBuilder) newScaleSetListenerSecretMirror(autoscalingListener *v
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      scaleSetListenerSecretMirrorName(autoscalingListener),
 			Namespace: autoscalingListener.Namespace,
-			Labels: map[string]string{
+			Labels: mergeLabels(autoscalingListener.Labels, map[string]string{
 				LabelKeyGitHubScaleSetNamespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
 				LabelKeyGitHubScaleSetName:      autoscalingListener.Spec.AutoscalingRunnerSetName,
 				"secret-data-hash":              dataHash,
-			},
+			}),
 		},
 		Data: secret.DeepCopy().Data,
 	}
@@ -502,13 +502,13 @@ func (b *resourceBuilder) newEphemeralRunnerSet(autoscalingRunnerSet *v1alpha1.A
 	}
 	runnerSpecHash := autoscalingRunnerSet.RunnerSetSpecHash()
 
-	labels := map[string]string{
+	labels := mergeLabels(autoscalingRunnerSet.Labels, map[string]string{
 		LabelKeyKubernetesPartOf:        labelValueKubernetesPartOf,
 		LabelKeyKubernetesComponent:     "runner-set",
 		LabelKeyKubernetesVersion:       autoscalingRunnerSet.Labels[LabelKeyKubernetesVersion],
 		LabelKeyGitHubScaleSetName:      autoscalingRunnerSet.Name,
 		LabelKeyGitHubScaleSetNamespace: autoscalingRunnerSet.Namespace,
-	}
+	})
 
 	if err := applyGitHubURLLabels(autoscalingRunnerSet.Spec.GitHubConfigUrl, labels); err != nil {
 		return nil, fmt.Errorf("failed to apply GitHub URL labels: %v", err)
@@ -547,18 +547,14 @@ func (b *resourceBuilder) newEphemeralRunnerSet(autoscalingRunnerSet *v1alpha1.A
 
 func (b *resourceBuilder) newEphemeralRunner(ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet) *v1alpha1.EphemeralRunner {
 	labels := make(map[string]string)
-	for _, key := range commonLabelKeys {
-		switch key {
-		case LabelKeyKubernetesComponent:
-			labels[key] = "runner"
-		default:
-			v, ok := ephemeralRunnerSet.Labels[key]
-			if !ok {
-				continue
-			}
-			labels[key] = v
+	for k, v := range ephemeralRunnerSet.Labels {
+		if k == LabelKeyKubernetesComponent {
+			labels[k] = "runner"
+		} else {
+			labels[k] = v
 		}
 	}
+
 	annotations := make(map[string]string)
 	for key, val := range ephemeralRunnerSet.Annotations {
 		annotations[key] = val
@@ -751,3 +747,17 @@ func trimLabelValue(val string) string {
 	}
 	return val
 }
+
+func mergeLabels(base, overwrite map[string]string) map[string]string {
+	mergedLabels := map[string]string{}
+
+	for k, v := range base {
+		mergedLabels[k] = v
+	}
+
+	for k, v := range overwrite {
+		mergedLabels[k] = v
+	}
+
+	return mergedLabels
+}

controllers/actions.github.com/resourcebuilder_test.go

@@ -21,6 +21,7 @@ func TestLabelPropagation(t *testing.T) {
 			Labels: map[string]string{
 				LabelKeyKubernetesPartOf:  labelValueKubernetesPartOf,
 				LabelKeyKubernetesVersion: "0.2.0",
+				"arbitrary-label":         "random-value",
 			},
 			Annotations: map[string]string{
 				runnerScaleSetIdAnnotationKey:         "1",
@@ -47,6 +48,7 @@ func TestLabelPropagation(t *testing.T) {
 	assert.Equal(t, "repo", ephemeralRunnerSet.Labels[LabelKeyGitHubRepository])
 	assert.Equal(t, autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerGroupName], ephemeralRunnerSet.Annotations[AnnotationKeyGitHubRunnerGroupName])
 	assert.Equal(t, autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerScaleSetName], ephemeralRunnerSet.Annotations[AnnotationKeyGitHubRunnerScaleSetName])
+	assert.Equal(t, autoscalingRunnerSet.Labels["arbitrary-label"], ephemeralRunnerSet.Labels["arbitrary-label"])
 
 	listener, err := b.newAutoScalingListener(&autoscalingRunnerSet, ephemeralRunnerSet, autoscalingRunnerSet.Namespace, "test:latest", nil)
 	require.NoError(t, err)
@@ -59,6 +61,7 @@ func TestLabelPropagation(t *testing.T) {
 	assert.Equal(t, "", listener.Labels[LabelKeyGitHubEnterprise])
 	assert.Equal(t, "org", listener.Labels[LabelKeyGitHubOrganization])
 	assert.Equal(t, "repo", listener.Labels[LabelKeyGitHubRepository])
+	assert.Equal(t, autoscalingRunnerSet.Labels["arbitrary-label"], listener.Labels["arbitrary-label"])
 
 	listenerServiceAccount := &corev1.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{

docs/gha-runner-scale-set-controller/README.md

@@ -43,6 +43,24 @@ You can follow [this troubleshooting guide](https://docs.github.com/en/actions/h
 
 ## Changelog
 
+### v0.9.2
+
+1. Refresh session if token expires during delete message [#3529](https://github.com/actions/actions-runner-controller/pull/3529)
+1. Re-use the last desired patch on empty batch [#3453](https://github.com/actions/actions-runner-controller/pull/3453)
+1. Extract single place to set up indexers [#3454](https://github.com/actions/actions-runner-controller/pull/3454)
+1. Include controller version in logs [#3473](https://github.com/actions/actions-runner-controller/pull/3473)
+1. Propogate arbitrary labels from runnersets to all created resources [#3157](https://github.com/actions/actions-runner-controller/pull/3157)
+
+### v0.9.1
+
+#### Major changes
+
+1. Shutdown metrics server when listener exits [#3445](https://github.com/actions/actions-runner-controller/pull/3445)
+1. Propagate max capacity information to the actions back-end [#3431](https://github.com/actions/actions-runner-controller/pull/3431)
+1. Refactor actions client error to include request id [#3430](https://github.com/actions/actions-runner-controller/pull/3430)
+1. Include self correction on empty batch and avoid removing pending runners when cluster is busy [#3426](https://github.com/actions/actions-runner-controller/pull/3426)
+1. Add topologySpreadConstraint to gha-runner-scale-set-controller chart [#3405](https://github.com/actions/actions-runner-controller/pull/3405)
+
 ### v0.9.0
 
 #### ⚠️ Warning

github/actions/client.go

@@ -29,6 +29,9 @@ const (
 	apiVersionQueryParam = "api-version=6.0-preview"
 )
 
+// Header used to propagate capacity information to the back-end
+const HeaderScaleSetMaxCapacity = "X-ScaleSetMaxCapacity"
+
 //go:generate mockery --inpackage --name=ActionsService
 type ActionsService interface {
 	GetRunnerScaleSet(ctx context.Context, runnerGroupId int, runnerScaleSetName string) (*RunnerScaleSet, error)
@@ -45,7 +48,7 @@ type ActionsService interface {
 	AcquireJobs(ctx context.Context, runnerScaleSetId int, messageQueueAccessToken string, requestIds []int64) ([]int64, error)
 	GetAcquirableJobs(ctx context.Context, runnerScaleSetId int) (*AcquirableJobList, error)
 
-	GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64) (*RunnerScaleSetMessage, error)
+	GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64, maxCapacity int) (*RunnerScaleSetMessage, error)
 	DeleteMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, messageId int64) error
 
 	GenerateJitRunnerConfig(ctx context.Context, jitRunnerSetting *RunnerScaleSetJitRunnerSetting, scaleSetId int) (*RunnerScaleSetJitRunnerConfig, error)
@@ -104,6 +107,8 @@ type Client struct {
 	proxyFunc ProxyFunc
 }
 
+var _ ActionsService = &Client{}
+
 type ProxyFunc func(req *http.Request) (*url.URL, error)
 
 type ClientOption func(*Client)
@@ -355,15 +360,22 @@ func (c *Client) GetRunnerScaleSet(ctx context.Context, runnerGroupId int, runne
 	}
 
 	var runnerScaleSetList *runnerScaleSetsResponse
-	err = json.NewDecoder(resp.Body).Decode(&runnerScaleSetList)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&runnerScaleSetList); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 	if runnerScaleSetList.Count == 0 {
 		return nil, nil
 	}
 	if runnerScaleSetList.Count > 1 {
-		return nil, fmt.Errorf("multiple runner scale sets found with name %s", runnerScaleSetName)
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        fmt.Errorf("multiple runner scale sets found with name %q", runnerScaleSetName),
+		}
 	}
 
 	return &runnerScaleSetList.RunnerScaleSets[0], nil
@@ -386,9 +398,12 @@ func (c *Client) GetRunnerScaleSetById(ctx context.Context, runnerScaleSetId int
 	}
 
 	var runnerScaleSet *RunnerScaleSet
-	err = json.NewDecoder(resp.Body).Decode(&runnerScaleSet)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&runnerScaleSet); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 	return runnerScaleSet, nil
 }
@@ -408,23 +423,43 @@ func (c *Client) GetRunnerGroupByName(ctx context.Context, runnerGroup string) (
 	if resp.StatusCode != http.StatusOK {
 		body, err := io.ReadAll(resp.Body)
 		if err != nil {
-			return nil, err
+			return nil, &ActionsError{
+				StatusCode: resp.StatusCode,
+				ActivityID: resp.Header.Get(HeaderActionsActivityID),
+				Err:        err,
+			}
 		}
-		return nil, fmt.Errorf("unexpected status code: %d - body: %s", resp.StatusCode, string(body))
+		return nil, fmt.Errorf("unexpected status code: %w", &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        errors.New(string(body)),
+		})
 	}
 
 	var runnerGroupList *RunnerGroupList
 	err = json.NewDecoder(resp.Body).Decode(&runnerGroupList)
 	if err != nil {
-		return nil, err
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 
 	if runnerGroupList.Count == 0 {
-		return nil, fmt.Errorf("no runner group found with name '%s'", runnerGroup)
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        fmt.Errorf("no runner group found with name %q", runnerGroup),
+		}
 	}
 
 	if runnerGroupList.Count > 1 {
-		return nil, fmt.Errorf("multiple runner group found with name %s", runnerGroup)
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        fmt.Errorf("multiple runner group found with name %q", runnerGroup),
+		}
 	}
 
 	return &runnerGroupList.RunnerGroups[0], nil
@@ -450,9 +485,12 @@ func (c *Client) CreateRunnerScaleSet(ctx context.Context, runnerScaleSet *Runne
 		return nil, ParseActionsErrorFromResponse(resp)
 	}
 	var createdRunnerScaleSet *RunnerScaleSet
-	err = json.NewDecoder(resp.Body).Decode(&createdRunnerScaleSet)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&createdRunnerScaleSet); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 	return createdRunnerScaleSet, nil
 }
@@ -480,9 +518,12 @@ func (c *Client) UpdateRunnerScaleSet(ctx context.Context, runnerScaleSetId int,
 	}
 
 	var updatedRunnerScaleSet *RunnerScaleSet
-	err = json.NewDecoder(resp.Body).Decode(&updatedRunnerScaleSet)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&updatedRunnerScaleSet); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 	return updatedRunnerScaleSet, nil
 }
@@ -507,7 +548,7 @@ func (c *Client) DeleteRunnerScaleSet(ctx context.Context, runnerScaleSetId int)
 	return nil
 }
 
-func (c *Client) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64) (*RunnerScaleSetMessage, error) {
+func (c *Client) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64, maxCapacity int) (*RunnerScaleSetMessage, error) {
 	u, err := url.Parse(messageQueueUrl)
 	if err != nil {
 		return nil, err
@@ -519,6 +560,10 @@ func (c *Client) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAc
 		u.RawQuery = q.Encode()
 	}
 
+	if maxCapacity < 0 {
+		return nil, fmt.Errorf("maxCapacity must be greater than or equal to 0")
+	}
+
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
 	if err != nil {
 		return nil, err
@@ -527,6 +572,7 @@ func (c *Client) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAc
 	req.Header.Set("Accept", "application/json; api-version=6.0-preview")
 	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", messageQueueAccessToken))
 	req.Header.Set("User-Agent", c.userAgent.String())
+	req.Header.Set(HeaderScaleSetMaxCapacity, strconv.Itoa(maxCapacity))
 
 	resp, err := c.Do(req)
 	if err != nil {
@@ -547,15 +593,26 @@ func (c *Client) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAc
 		body, err := io.ReadAll(resp.Body)
 		body = trimByteOrderMark(body)
 		if err != nil {
-			return nil, err
+			return nil, &ActionsError{
+				ActivityID: resp.Header.Get(HeaderActionsActivityID),
+				StatusCode: resp.StatusCode,
+				Err:        err,
+			}
+		}
+		return nil, &MessageQueueTokenExpiredError{
+			activityID: resp.Header.Get(HeaderActionsActivityID),
+			statusCode: resp.StatusCode,
+			msg:        string(body),
 		}
-		return nil, &MessageQueueTokenExpiredError{msg: string(body)}
 	}
 
 	var message *RunnerScaleSetMessage
-	err = json.NewDecoder(resp.Body).Decode(&message)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&message); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 	return message, nil
 }
@@ -591,9 +648,17 @@ func (c *Client) DeleteMessage(ctx context.Context, messageQueueUrl, messageQueu
 		body, err := io.ReadAll(resp.Body)
 		body = trimByteOrderMark(body)
 		if err != nil {
-			return err
+			return &ActionsError{
+				ActivityID: resp.Header.Get(HeaderActionsActivityID),
+				StatusCode: resp.StatusCode,
+				Err:        err,
+			}
+		}
+		return &MessageQueueTokenExpiredError{
+			activityID: resp.Header.Get(HeaderActionsActivityID),
+			statusCode: resp.StatusCode,
+			msg:        string(body),
 		}
-		return &MessageQueueTokenExpiredError{msg: string(body)}
 	}
 	return nil
 }
@@ -641,9 +706,18 @@ func (c *Client) doSessionRequest(ctx context.Context, method, path string, requ
 	}
 
 	if resp.StatusCode == expectedResponseStatusCode {
-		if responseUnmarshalTarget != nil {
-			return json.NewDecoder(resp.Body).Decode(responseUnmarshalTarget)
+		if responseUnmarshalTarget == nil {
+			return nil
 		}
+
+		if err := json.NewDecoder(resp.Body).Decode(responseUnmarshalTarget); err != nil {
+			return &ActionsError{
+				StatusCode: resp.StatusCode,
+				ActivityID: resp.Header.Get(HeaderActionsActivityID),
+				Err:        err,
+			}
+		}
+
 		return nil
 	}
 
@@ -655,10 +729,18 @@ func (c *Client) doSessionRequest(ctx context.Context, method, path string, requ
 	body, err := io.ReadAll(resp.Body)
 	body = trimByteOrderMark(body)
 	if err != nil {
-		return err
+		return &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 
-	return fmt.Errorf("unexpected status code: %d - body: %s", resp.StatusCode, string(body))
+	return fmt.Errorf("unexpected status code: %w", &ActionsError{
+		StatusCode: resp.StatusCode,
+		ActivityID: resp.Header.Get(HeaderActionsActivityID),
+		Err:        errors.New(string(body)),
+	})
 }
 
 func (c *Client) AcquireJobs(ctx context.Context, runnerScaleSetId int, messageQueueAccessToken string, requestIds []int64) ([]int64, error) {
@@ -692,16 +774,28 @@ func (c *Client) AcquireJobs(ctx context.Context, runnerScaleSetId int, messageQ
 		body, err := io.ReadAll(resp.Body)
 		body = trimByteOrderMark(body)
 		if err != nil {
-			return nil, err
+			return nil, &ActionsError{
+				ActivityID: resp.Header.Get(HeaderActionsActivityID),
+				StatusCode: resp.StatusCode,
+				Err:        err,
+			}
 		}
 
-		return nil, &MessageQueueTokenExpiredError{msg: string(body)}
+		return nil, &MessageQueueTokenExpiredError{
+			activityID: resp.Header.Get(HeaderActionsActivityID),
+			statusCode: resp.StatusCode,
+			msg:        string(body),
+		}
 	}
 
 	var acquiredJobs *Int64List
 	err = json.NewDecoder(resp.Body).Decode(&acquiredJobs)
 	if err != nil {
-		return nil, err
+		return nil, &ActionsError{
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			StatusCode: resp.StatusCode,
+			Err:        err,
+		}
 	}
 
 	return acquiredJobs.Value, nil
@@ -732,7 +826,11 @@ func (c *Client) GetAcquirableJobs(ctx context.Context, runnerScaleSetId int) (*
 	var acquirableJobList *AcquirableJobList
 	err = json.NewDecoder(resp.Body).Decode(&acquirableJobList)
 	if err != nil {
-		return nil, err
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 
 	return acquirableJobList, nil
@@ -761,9 +859,12 @@ func (c *Client) GenerateJitRunnerConfig(ctx context.Context, jitRunnerSetting *
 	}
 
 	var runnerJitConfig *RunnerScaleSetJitRunnerConfig
-	err = json.NewDecoder(resp.Body).Decode(&runnerJitConfig)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&runnerJitConfig); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 	return runnerJitConfig, nil
 }
@@ -786,9 +887,12 @@ func (c *Client) GetRunner(ctx context.Context, runnerId int64) (*RunnerReferenc
 	}
 
 	var runnerReference *RunnerReference
-	err = json.NewDecoder(resp.Body).Decode(&runnerReference)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&runnerReference); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 
 	return runnerReference, nil
@@ -812,9 +916,12 @@ func (c *Client) GetRunnerByName(ctx context.Context, runnerName string) (*Runne
 	}
 
 	var runnerList *RunnerReferenceList
-	err = json.NewDecoder(resp.Body).Decode(&runnerList)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&runnerList); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 
 	if runnerList.Count == 0 {
@@ -822,7 +929,11 @@ func (c *Client) GetRunnerByName(ctx context.Context, runnerName string) (*Runne
 	}
 
 	if runnerList.Count > 1 {
-		return nil, fmt.Errorf("multiple runner found with name %s", runnerName)
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        fmt.Errorf("multiple runner found with name %s", runnerName),
+		}
 	}
 
 	return &runnerList.RunnerReferences[0], nil
@@ -895,12 +1006,20 @@ func (c *Client) getRunnerRegistrationToken(ctx context.Context) (*registrationT
 		if err != nil {
 			return nil, err
 		}
-		return nil, fmt.Errorf("unexpected response from Actions service during registration token call: %v - %v", resp.StatusCode, string(body))
+		return nil, &GitHubAPIError{
+			StatusCode: resp.StatusCode,
+			RequestID:  resp.Header.Get(HeaderGitHubRequestID),
+			Err:        errors.New(string(body)),
+		}
 	}
 
 	var registrationToken *registrationToken
 	if err := json.NewDecoder(resp.Body).Decode(&registrationToken); err != nil {
-		return nil, err
+		return nil, &GitHubAPIError{
+			StatusCode: resp.StatusCode,
+			RequestID:  resp.Header.Get(HeaderGitHubRequestID),
+			Err:        err,
+		}
 	}
 
 	return registrationToken, nil
@@ -937,8 +1056,14 @@ func (c *Client) fetchAccessToken(ctx context.Context, gitHubConfigURL string, c
 
 	// Format: https://docs.github.com/en/rest/apps/apps#create-an-installation-access-token-for-an-app
 	var accessToken *accessToken
-	err = json.NewDecoder(resp.Body).Decode(&accessToken)
-	return accessToken, err
+	if err = json.NewDecoder(resp.Body).Decode(&accessToken); err != nil {
+		return nil, &GitHubAPIError{
+			StatusCode: resp.StatusCode,
+			RequestID:  resp.Header.Get(HeaderGitHubRequestID),
+			Err:        err,
+		}
+	}
+	return accessToken, nil
 }
 
 type ActionsServiceAdminConnection struct {
@@ -989,21 +1114,29 @@ func (c *Client) getActionsServiceAdminConnection(ctx context.Context, rt *regis
 			break
 		}
 
-		errStr := fmt.Sprintf("unexpected response from Actions service during registration call: %v", resp.StatusCode)
+		var innerErr error
 		body, err := io.ReadAll(resp.Body)
 		if err != nil {
-			err = fmt.Errorf("%s - %w", errStr, err)
+			innerErr = err
 		} else {
-			err = fmt.Errorf("%s - %v", errStr, string(body))
+			innerErr = errors.New(string(body))
 		}
 
 		if resp.StatusCode != http.StatusUnauthorized && resp.StatusCode != http.StatusForbidden {
-			return nil, err
+			return nil, &GitHubAPIError{
+				StatusCode: resp.StatusCode,
+				RequestID:  resp.Header.Get(HeaderGitHubRequestID),
+				Err:        innerErr,
+			}
 		}
 
 		retry++
 		if retry > 3 {
-			return nil, fmt.Errorf("unable to register runner after 3 retries: %v", err)
+			return nil, fmt.Errorf("unable to register runner after 3 retries: %w", &GitHubAPIError{
+				StatusCode: resp.StatusCode,
+				RequestID:  resp.Header.Get(HeaderGitHubRequestID),
+				Err:        innerErr,
+			})
 		}
 		time.Sleep(time.Duration(500 * int(time.Millisecond) * (retry + 1)))
 
@@ -1011,7 +1144,11 @@ func (c *Client) getActionsServiceAdminConnection(ctx context.Context, rt *regis
 
 	var actionsServiceAdminConnection *ActionsServiceAdminConnection
 	if err := json.NewDecoder(resp.Body).Decode(&actionsServiceAdminConnection); err != nil {
-		return nil, err
+		return nil, &GitHubAPIError{
+			StatusCode: resp.StatusCode,
+			RequestID:  resp.Header.Get(HeaderGitHubRequestID),
+			Err:        err,
+		}
 	}
 
 	return actionsServiceAdminConnection, nil

github/actions/client_runner_scale_set_message_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"net/http"
+	"strconv"
 	"testing"
 	"time"
 
@@ -35,7 +36,7 @@ func TestGetMessage(t *testing.T) {
 		client, err := actions.NewClient(s.configURLForOrg("my-org"), auth)
 		require.NoError(t, err)
 
-		got, err := client.GetMessage(ctx, s.URL, token, 0)
+		got, err := client.GetMessage(ctx, s.URL, token, 0, 10)
 		require.NoError(t, err)
 		assert.Equal(t, want, got)
 	})
@@ -52,7 +53,7 @@ func TestGetMessage(t *testing.T) {
 		client, err := actions.NewClient(s.configURLForOrg("my-org"), auth)
 		require.NoError(t, err)
 
-		got, err := client.GetMessage(ctx, s.URL, token, 1)
+		got, err := client.GetMessage(ctx, s.URL, token, 1, 10)
 		require.NoError(t, err)
 		assert.Equal(t, want, got)
 	})
@@ -76,7 +77,7 @@ func TestGetMessage(t *testing.T) {
 		)
 		require.NoError(t, err)
 
-		_, err = client.GetMessage(ctx, server.URL, token, 0)
+		_, err = client.GetMessage(ctx, server.URL, token, 0, 10)
 		assert.NotNil(t, err)
 		assert.Equalf(t, actualRetry, expectedRetry, "A retry was expected after the first request but got: %v", actualRetry)
 	})
@@ -89,7 +90,7 @@ func TestGetMessage(t *testing.T) {
 		client, err := actions.NewClient(server.configURLForOrg("my-org"), auth)
 		require.NoError(t, err)
 
-		_, err = client.GetMessage(ctx, server.URL, token, 0)
+		_, err = client.GetMessage(ctx, server.URL, token, 0, 10)
 		require.NotNil(t, err)
 
 		var expectedErr *actions.MessageQueueTokenExpiredError
@@ -98,7 +99,7 @@ func TestGetMessage(t *testing.T) {
 
 	t.Run("Status code not found", func(t *testing.T) {
 		want := actions.ActionsError{
-			Message:    "Request returned status: 404 Not Found",
+			Err:        errors.New("unknown exception"),
 			StatusCode: 404,
 		}
 		server := newActionsServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
@@ -108,7 +109,7 @@ func TestGetMessage(t *testing.T) {
 		client, err := actions.NewClient(server.configURLForOrg("my-org"), auth)
 		require.NoError(t, err)
 
-		_, err = client.GetMessage(ctx, server.URL, token, 0)
+		_, err = client.GetMessage(ctx, server.URL, token, 0, 10)
 		require.NotNil(t, err)
 		assert.Equal(t, want.Error(), err.Error())
 	})
@@ -122,9 +123,35 @@ func TestGetMessage(t *testing.T) {
 		client, err := actions.NewClient(server.configURLForOrg("my-org"), auth)
 		require.NoError(t, err)
 
-		_, err = client.GetMessage(ctx, server.URL, token, 0)
+		_, err = client.GetMessage(ctx, server.URL, token, 0, 10)
 		assert.NotNil(t, err)
 	})
+
+	t.Run("Capacity error handling", func(t *testing.T) {
+		server := newActionsServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hc := r.Header.Get(actions.HeaderScaleSetMaxCapacity)
+			c, err := strconv.Atoi(hc)
+			require.NoError(t, err)
+			assert.GreaterOrEqual(t, c, 0)
+
+			w.WriteHeader(http.StatusBadRequest)
+			w.Header().Set("Content-Type", "text/plain")
+		}))
+
+		client, err := actions.NewClient(server.configURLForOrg("my-org"), auth)
+		require.NoError(t, err)
+
+		_, err = client.GetMessage(ctx, server.URL, token, 0, -1)
+		require.Error(t, err)
+		// Ensure we don't send requests with negative capacity
+		assert.False(t, errors.Is(err, &actions.ActionsError{}))
+
+		_, err = client.GetMessage(ctx, server.URL, token, 0, 0)
+		assert.Error(t, err)
+		var expectedErr *actions.ActionsError
+		assert.ErrorAs(t, err, &expectedErr)
+		assert.Equal(t, http.StatusBadRequest, expectedErr.StatusCode)
+	})
 }
 
 func TestDeleteMessage(t *testing.T) {

github/actions/client_runner_scale_set_session_test.go

@@ -13,6 +13,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const exampleRequestID = "5ddf2050-dae0-013c-9159-04421ad31b68"
+
 func TestCreateMessageSession(t *testing.T) {
 	ctx := context.Background()
 	auth := &actions.ActionsAuth{
@@ -69,13 +71,17 @@ func TestCreateMessageSession(t *testing.T) {
 		}
 
 		want := &actions.ActionsError{
-			ExceptionName: "CSharpExceptionNameHere",
-			Message:       "could not do something",
-			StatusCode:    http.StatusBadRequest,
+			ActivityID: exampleRequestID,
+			StatusCode: http.StatusBadRequest,
+			Err: &actions.ActionsExceptionError{
+				ExceptionName: "CSharpExceptionNameHere",
+				Message:       "could not do something",
+			},
 		}
 
 		server := newActionsServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 			w.Header().Set("Content-Type", "application/json")
+			w.Header().Set(actions.HeaderActionsActivityID, exampleRequestID)
 			w.WriteHeader(http.StatusBadRequest)
 			resp := []byte(`{"typeName": "CSharpExceptionNameHere","message": "could not do something"}`)
 			w.Write(resp)

github/actions/client_runner_scale_set_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/actions/actions-runner-controller/github/actions"
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -124,9 +125,15 @@ func TestGetRunnerScaleSet(t *testing.T) {
 	})
 
 	t.Run("Multiple runner scale sets found", func(t *testing.T) {
-		wantErr := fmt.Errorf("multiple runner scale sets found with name %s", scaleSetName)
+		reqID := uuid.NewString()
+		wantErr := &actions.ActionsError{
+			StatusCode: http.StatusOK,
+			ActivityID: reqID,
+			Err:        fmt.Errorf("multiple runner scale sets found with name %q", scaleSetName),
+		}
 		runnerScaleSetsResp := []byte(`{"count":2,"value":[{"id":1,"name":"ScaleSet"}]}`)
 		server := newActionsServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+			w.Header().Set(actions.HeaderActionsActivityID, reqID)
 			w.Write(runnerScaleSetsResp)
 		}))
 

github/actions/errors.go

@@ -2,63 +2,117 @@ package actions
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
 )
 
+// Header names for request IDs
+const (
+	HeaderActionsActivityID = "ActivityId"
+	HeaderGitHubRequestID   = "X-GitHub-Request-Id"
+)
+
+type GitHubAPIError struct {
+	StatusCode int
+	RequestID  string
+	Err        error
+}
+
+func (e *GitHubAPIError) Error() string {
+	return fmt.Sprintf("github api error: StatusCode %d, RequestID %q: %v", e.StatusCode, e.RequestID, e.Err)
+}
+
+func (e *GitHubAPIError) Unwrap() error {
+	return e.Err
+}
+
 type ActionsError struct {
-	ExceptionName string `json:"typeName,omitempty"`
-	Message       string `json:"message,omitempty"`
-	StatusCode    int
+	ActivityID string
+	StatusCode int
+	Err        error
 }
 
 func (e *ActionsError) Error() string {
-	return fmt.Sprintf("%v - had issue communicating with Actions backend: %v", e.StatusCode, e.Message)
+	return fmt.Sprintf("actions error: StatusCode %d, AcivityId %q: %v", e.StatusCode, e.ActivityID, e.Err)
+}
+
+func (e *ActionsError) Unwrap() error {
+	return e.Err
+}
+
+func (e *ActionsError) IsException(target string) bool {
+	if ex, ok := e.Err.(*ActionsExceptionError); ok {
+		return strings.Contains(ex.ExceptionName, target)
+	}
+	return false
+}
+
+type ActionsExceptionError struct {
+	ExceptionName string `json:"typeName,omitempty"`
+	Message       string `json:"message,omitempty"`
+}
+
+func (e *ActionsExceptionError) Error() string {
+	return fmt.Sprintf("%s: %s", e.ExceptionName, e.Message)
 }
 
 func ParseActionsErrorFromResponse(response *http.Response) error {
 	if response.ContentLength == 0 {
-		message := "Request returned status: " + response.Status
 		return &ActionsError{
-			ExceptionName: "unknown",
-			Message:       message,
-			StatusCode:    response.StatusCode,
+			ActivityID: response.Header.Get(HeaderActionsActivityID),
+			StatusCode: response.StatusCode,
+			Err:        errors.New("unknown exception"),
 		}
 	}
 
 	defer response.Body.Close()
 	body, err := io.ReadAll(response.Body)
 	if err != nil {
-		return err
+		return &ActionsError{
+			ActivityID: response.Header.Get(HeaderActionsActivityID),
+			StatusCode: response.StatusCode,
+			Err:        err,
+		}
 	}
 
 	body = trimByteOrderMark(body)
 	contentType, ok := response.Header["Content-Type"]
 	if ok && len(contentType) > 0 && strings.Contains(contentType[0], "text/plain") {
 		message := string(body)
-		statusCode := response.StatusCode
 		return &ActionsError{
-			Message:    message,
-			StatusCode: statusCode,
+			ActivityID: response.Header.Get(HeaderActionsActivityID),
+			StatusCode: response.StatusCode,
+			Err:        errors.New(message),
 		}
 	}
 
-	actionsError := &ActionsError{StatusCode: response.StatusCode}
-	if err := json.Unmarshal(body, &actionsError); err != nil {
-		return err
+	var exception ActionsExceptionError
+	if err := json.Unmarshal(body, &exception); err != nil {
+		return &ActionsError{
+			ActivityID: response.Header.Get(HeaderActionsActivityID),
+			StatusCode: response.StatusCode,
+			Err:        err,
+		}
 	}
 
-	return actionsError
+	return &ActionsError{
+		ActivityID: response.Header.Get(HeaderActionsActivityID),
+		StatusCode: response.StatusCode,
+		Err:        &exception,
+	}
 }
 
 type MessageQueueTokenExpiredError struct {
-	msg string
+	activityID string
+	statusCode int
+	msg        string
 }
 
 func (e *MessageQueueTokenExpiredError) Error() string {
-	return e.msg
+	return fmt.Sprintf("MessageQueueTokenExpiredError: AcivityId %q, StatusCode %d: %s", e.activityID, e.statusCode, e.msg)
 }
 
 type HttpClientSideError struct {

github/actions/errors_test.go

@@ -0,0 +1,206 @@
+package actions_test
+
+import (
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/actions/actions-runner-controller/github/actions"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestActionsError(t *testing.T) {
+	t.Run("contains the status code, activity ID, and error", func(t *testing.T) {
+		err := &actions.ActionsError{
+			ActivityID: "activity-id",
+			StatusCode: 404,
+			Err:        errors.New("example error description"),
+		}
+
+		s := err.Error()
+		assert.Contains(t, s, "StatusCode 404")
+		assert.Contains(t, s, "AcivityId \"activity-id\"")
+		assert.Contains(t, s, "example error description")
+	})
+
+	t.Run("unwraps the error", func(t *testing.T) {
+		err := &actions.ActionsError{
+			ActivityID: "activity-id",
+			StatusCode: 404,
+			Err: &actions.ActionsExceptionError{
+				ExceptionName: "exception-name",
+				Message:       "example error message",
+			},
+		}
+
+		assert.Equal(t, err.Unwrap(), err.Err)
+	})
+
+	t.Run("is exception is ok", func(t *testing.T) {
+		err := &actions.ActionsError{
+			ActivityID: "activity-id",
+			StatusCode: 404,
+			Err: &actions.ActionsExceptionError{
+				ExceptionName: "exception-name",
+				Message:       "example error message",
+			},
+		}
+
+		var exception *actions.ActionsExceptionError
+		assert.True(t, errors.As(err, &exception))
+
+		assert.True(t, err.IsException("exception-name"))
+	})
+
+	t.Run("is exception is not ok", func(t *testing.T) {
+		tt := map[string]*actions.ActionsError{
+			"not an exception": {
+				ActivityID: "activity-id",
+				StatusCode: 404,
+				Err:        errors.New("example error description"),
+			},
+			"not target exception": {
+				ActivityID: "activity-id",
+				StatusCode: 404,
+				Err: &actions.ActionsExceptionError{
+					ExceptionName: "exception-name",
+					Message:       "example error message",
+				},
+			},
+		}
+
+		targetException := "target-exception"
+		for name, err := range tt {
+			t.Run(name, func(t *testing.T) {
+				assert.False(t, err.IsException(targetException))
+			})
+		}
+	})
+}
+
+func TestActionsExceptionError(t *testing.T) {
+	t.Run("contains the exception name and message", func(t *testing.T) {
+		err := &actions.ActionsExceptionError{
+			ExceptionName: "exception-name",
+			Message:       "example error message",
+		}
+
+		s := err.Error()
+		assert.Contains(t, s, "exception-name")
+		assert.Contains(t, s, "example error message")
+	})
+}
+
+func TestGitHubAPIError(t *testing.T) {
+	t.Run("contains the status code, request ID, and error", func(t *testing.T) {
+		err := &actions.GitHubAPIError{
+			StatusCode: 404,
+			RequestID:  "request-id",
+			Err:        errors.New("example error description"),
+		}
+
+		s := err.Error()
+		assert.Contains(t, s, "StatusCode 404")
+		assert.Contains(t, s, "RequestID \"request-id\"")
+		assert.Contains(t, s, "example error description")
+	})
+
+	t.Run("unwraps the error", func(t *testing.T) {
+		err := &actions.GitHubAPIError{
+			StatusCode: 404,
+			RequestID:  "request-id",
+			Err:        errors.New("example error description"),
+		}
+
+		assert.Equal(t, err.Unwrap(), err.Err)
+	})
+}
+
+func ParseActionsErrorFromResponse(t *testing.T) {
+	t.Run("empty content length", func(t *testing.T) {
+		response := &http.Response{
+			ContentLength: 0,
+			Header: http.Header{
+				actions.HeaderActionsActivityID: []string{"activity-id"},
+			},
+			StatusCode: 404,
+		}
+
+		err := actions.ParseActionsErrorFromResponse(response)
+		require.Error(t, err)
+		assert.Equal(t, err.(*actions.ActionsError).ActivityID, "activity-id")
+		assert.Equal(t, err.(*actions.ActionsError).StatusCode, 404)
+		assert.Equal(t, err.(*actions.ActionsError).Err.Error(), "unknown exception")
+	})
+
+	t.Run("contains text plain error", func(t *testing.T) {
+		errorMessage := "example error message"
+		response := &http.Response{
+			ContentLength: int64(len(errorMessage)),
+			Header: http.Header{
+				actions.HeaderActionsActivityID: []string{"activity-id"},
+				"Content-Type":                  []string{"text/plain"},
+			},
+			StatusCode: 404,
+			Body:       io.NopCloser(strings.NewReader(errorMessage)),
+		}
+
+		err := actions.ParseActionsErrorFromResponse(response)
+		require.Error(t, err)
+		var actionsError *actions.ActionsError
+		assert.ErrorAs(t, err, &actionsError)
+		assert.Equal(t, actionsError.ActivityID, "activity-id")
+		assert.Equal(t, actionsError.StatusCode, 404)
+		assert.Equal(t, actionsError.Err.Error(), errorMessage)
+	})
+
+	t.Run("contains json error", func(t *testing.T) {
+		errorMessage := `{"typeName":"exception-name","message":"example error message"}`
+		response := &http.Response{
+			ContentLength: int64(len(errorMessage)),
+			Header: http.Header{
+				actions.HeaderActionsActivityID: []string{"activity-id"},
+				"Content-Type":                  []string{"application/json"},
+			},
+			StatusCode: 404,
+			Body:       io.NopCloser(strings.NewReader(errorMessage)),
+		}
+
+		err := actions.ParseActionsErrorFromResponse(response)
+		require.Error(t, err)
+		var actionsError *actions.ActionsError
+		assert.ErrorAs(t, err, &actionsError)
+		assert.Equal(t, actionsError.ActivityID, "activity-id")
+		assert.Equal(t, actionsError.StatusCode, 404)
+
+		inner, ok := actionsError.Err.(*actions.ActionsExceptionError)
+		require.True(t, ok)
+		assert.Equal(t, inner.ExceptionName, "exception-name")
+		assert.Equal(t, inner.Message, "example error message")
+	})
+
+	t.Run("wrapped exception error", func(t *testing.T) {
+		errorMessage := `{"typeName":"exception-name","message":"example error message"}`
+		response := &http.Response{
+			ContentLength: int64(len(errorMessage)),
+			Header: http.Header{
+				actions.HeaderActionsActivityID: []string{"activity-id"},
+				"Content-Type":                  []string{"application/json"},
+			},
+			StatusCode: 404,
+			Body:       io.NopCloser(strings.NewReader(errorMessage)),
+		}
+
+		err := actions.ParseActionsErrorFromResponse(response)
+		require.Error(t, err)
+
+		var actionsExceptionError *actions.ActionsExceptionError
+		assert.ErrorAs(t, err, &actionsExceptionError)
+
+		assert.Equal(t, actionsExceptionError.ExceptionName, "exception-name")
+		assert.Equal(t, actionsExceptionError.Message, "example error message")
+	})
+}

github/actions/fake/client.go

@@ -259,7 +259,7 @@ func (f *FakeClient) GetAcquirableJobs(ctx context.Context, runnerScaleSetId int
 	return f.getAcquirableJobsResult.AcquirableJobList, f.getAcquirableJobsResult.err
 }
 
-func (f *FakeClient) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64) (*actions.RunnerScaleSetMessage, error) {
+func (f *FakeClient) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64, maxCapacity int) (*actions.RunnerScaleSetMessage, error) {
 	return f.getMessageResult.RunnerScaleSetMessage, f.getMessageResult.err
 }
 

github/actions/github_api_request_test.go

@@ -139,7 +139,13 @@ func TestNewActionsServiceRequest(t *testing.T) {
 				w.WriteHeader(http.StatusUnauthorized)
 				w.Write([]byte(errMessage))
 			}
-			server := testserver.New(t, nil, testserver.WithActionsToken("random-token"), testserver.WithActionsToken(newToken), testserver.WithActionsRegistrationTokenHandler(unauthorizedHandler))
+			server := testserver.New(
+				t,
+				nil,
+				testserver.WithActionsToken("random-token"),
+				testserver.WithActionsToken(newToken),
+				testserver.WithActionsRegistrationTokenHandler(unauthorizedHandler),
+			)
 			client, err := actions.NewClient(server.ConfigURLForOrg("my-org"), defaultCreds)
 			require.NoError(t, err)
 			expiringToken := "expiring-token"

github/actions/mock_ActionsService.go

@@ -186,25 +186,25 @@ func (_m *MockActionsService) GetAcquirableJobs(ctx context.Context, runnerScale
 	return r0, r1
 }
 
-// GetMessage provides a mock function with given fields: ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId
-func (_m *MockActionsService) GetMessage(ctx context.Context, messageQueueUrl string, messageQueueAccessToken string, lastMessageId int64) (*RunnerScaleSetMessage, error) {
-	ret := _m.Called(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+// GetMessage provides a mock function with given fields: ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity
+func (_m *MockActionsService) GetMessage(ctx context.Context, messageQueueUrl string, messageQueueAccessToken string, lastMessageId int64, maxCapacity int) (*RunnerScaleSetMessage, error) {
+	ret := _m.Called(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 
 	var r0 *RunnerScaleSetMessage
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64) (*RunnerScaleSetMessage, error)); ok {
-		return rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int) (*RunnerScaleSetMessage, error)); ok {
+		return rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64) *RunnerScaleSetMessage); ok {
-		r0 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int) *RunnerScaleSetMessage); ok {
+		r0 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(*RunnerScaleSetMessage)
 		}
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, string, string, int64) error); ok {
-		r1 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+	if rf, ok := ret.Get(1).(func(context.Context, string, string, int64, int) error); ok {
+		r1 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 	} else {
 		r1 = ret.Error(1)
 	}

github/actions/mock_SessionService.go

@@ -67,25 +67,25 @@ func (_m *MockSessionService) DeleteMessage(ctx context.Context, messageId int64
 	return r0
 }
 
-// GetMessage provides a mock function with given fields: ctx, lastMessageId
-func (_m *MockSessionService) GetMessage(ctx context.Context, lastMessageId int64) (*RunnerScaleSetMessage, error) {
-	ret := _m.Called(ctx, lastMessageId)
+// GetMessage provides a mock function with given fields: ctx, lastMessageId, maxCapacity
+func (_m *MockSessionService) GetMessage(ctx context.Context, lastMessageId int64, maxCapacity int) (*RunnerScaleSetMessage, error) {
+	ret := _m.Called(ctx, lastMessageId, maxCapacity)
 
 	var r0 *RunnerScaleSetMessage
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, int64) (*RunnerScaleSetMessage, error)); ok {
-		return rf(ctx, lastMessageId)
+	if rf, ok := ret.Get(0).(func(context.Context, int64, int) (*RunnerScaleSetMessage, error)); ok {
+		return rf(ctx, lastMessageId, maxCapacity)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, int64) *RunnerScaleSetMessage); ok {
-		r0 = rf(ctx, lastMessageId)
+	if rf, ok := ret.Get(0).(func(context.Context, int64, int) *RunnerScaleSetMessage); ok {
+		r0 = rf(ctx, lastMessageId, maxCapacity)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(*RunnerScaleSetMessage)
 		}
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
-		r1 = rf(ctx, lastMessageId)
+	if rf, ok := ret.Get(1).(func(context.Context, int64, int) error); ok {
+		r1 = rf(ctx, lastMessageId, maxCapacity)
 	} else {
 		r1 = ret.Error(1)
 	}

github/actions/sessionservice.go

@@ -7,7 +7,7 @@ import (
 
 //go:generate mockery --inpackage --name=SessionService
 type SessionService interface {
-	GetMessage(ctx context.Context, lastMessageId int64) (*RunnerScaleSetMessage, error)
+	GetMessage(ctx context.Context, lastMessageId int64, maxCapacity int) (*RunnerScaleSetMessage, error)
 	DeleteMessage(ctx context.Context, messageId int64) error
 	AcquireJobs(ctx context.Context, requestIds []int64) ([]int64, error)
 	io.Closer

go.mod

@@ -5,7 +5,7 @@ go 1.22.1
 require (
 	github.com/bradleyfalzon/ghinstallation/v2 v2.8.0
 	github.com/davecgh/go-spew v1.1.1
-	github.com/evanphx/json-patch v5.7.0+incompatible
+	github.com/evanphx/json-patch v5.9.0+incompatible
 	github.com/go-logr/logr v1.4.1
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/google/go-cmp v0.6.0
@@ -17,17 +17,17 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.7.5
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/ginkgo/v2 v2.13.1
-	github.com/onsi/gomega v1.30.0
+	github.com/onsi/ginkgo/v2 v2.17.1
+	github.com/onsi/gomega v1.33.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.17.0
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/teambition/rrule-go v1.8.2
 	go.uber.org/multierr v1.11.0
-	go.uber.org/zap v1.26.0
-	golang.org/x/net v0.19.0
-	golang.org/x/oauth2 v0.15.0
-	golang.org/x/sync v0.6.0
+	go.uber.org/zap v1.27.0
+	golang.org/x/net v0.24.0
+	golang.org/x/oauth2 v0.19.0
+	golang.org/x/sync v0.7.0
 	gomodules.xyz/jsonpatch/v2 v2.4.0
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.28.4
@@ -43,7 +43,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boombuler/barcode v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/cloudflare/circl v1.3.6 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
@@ -87,15 +87,15 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/objx v0.5.1 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/urfave/cli v1.22.2 // indirect
-	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.4.0 // indirect
-	golang.org/x/tools v0.15.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect

go.sum

@@ -19,8 +19,8 @@ github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7N
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
-github.com/cloudflare/circl v1.3.6 h1:/xbKIqSHbZXHwkhbrhrt2YOHIwYJlXH94E3tI/gDlUg=
-github.com/cloudflare/circl v1.3.6/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
+github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
+github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -30,8 +30,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
-github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls=
+github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc=
 github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
@@ -169,12 +169,12 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.13.1 h1:LNGfMbR2OVGBfXjvRZIZ2YCTQdGKtPLvuI1rMCCj3OU=
-github.com/onsi/ginkgo/v2 v2.13.1/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM=
+github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8=
+github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
-github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/onsi/gomega v1.33.0 h1:snPCflnZrpMsy94p4lXVEkHo12lmPnc3vY5XBbreexE=
+github.com/onsi/gomega v1.33.0/go.mod h1:+925n5YtiFsLzzafLUHzVMBpvvRAzrydIBiSIxjX3wY=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -202,8 +202,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0=
-github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -212,9 +212,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/teambition/rrule-go v1.8.2 h1:lIjpjvWTj9fFUZCmuoVDrKVOtdiyzbzc93qTmRVe/J8=
 github.com/teambition/rrule-go v1.8.2/go.mod h1:Ieq5AbrKGciP1V//Wq8ktsTXwSwJHDD5mD/wLBGl3p4=
 github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
@@ -222,28 +221,26 @@ github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
-go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
-go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -256,18 +253,18 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
-golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
-golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
-golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg=
+golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -289,15 +286,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -316,8 +313,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8=
-golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
+golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

main.go

@@ -239,6 +239,10 @@ func main() {
 	}
 
 	if autoScalingRunnerSetOnly {
+		if err := actionsgithubcom.SetupIndexers(mgr); err != nil {
+			log.Error(err, "unable to setup indexers")
+			os.Exit(1)
+		}
 		managerImage := os.Getenv("CONTROLLER_MANAGER_CONTAINER_IMAGE")
 		if managerImage == "" {
 			log.Error(err, "unable to obtain listener image")
@@ -256,7 +260,7 @@ func main() {
 
 		if err = (&actionsgithubcom.AutoscalingRunnerSetReconciler{
 			Client:                             mgr.GetClient(),
-			Log:                                log.WithName("AutoscalingRunnerSet"),
+			Log:                                log.WithName("AutoscalingRunnerSet").WithValues("version", build.Version),
 			Scheme:                             mgr.GetScheme(),
 			ControllerNamespace:                managerNamespace,
 			DefaultRunnerScaleSetListenerImage: managerImage,
@@ -270,7 +274,7 @@ func main() {
 
 		if err = (&actionsgithubcom.EphemeralRunnerReconciler{
 			Client:        mgr.GetClient(),
-			Log:           log.WithName("EphemeralRunner"),
+			Log:           log.WithName("EphemeralRunner").WithValues("version", build.Version),
 			Scheme:        mgr.GetScheme(),
 			ActionsClient: actionsMultiClient,
 		}).SetupWithManager(mgr); err != nil {
@@ -280,7 +284,7 @@ func main() {
 
 		if err = (&actionsgithubcom.EphemeralRunnerSetReconciler{
 			Client:         mgr.GetClient(),
-			Log:            log.WithName("EphemeralRunnerSet"),
+			Log:            log.WithName("EphemeralRunnerSet").WithValues("version", build.Version),
 			Scheme:         mgr.GetScheme(),
 			ActionsClient:  actionsMultiClient,
 			PublishMetrics: metricsAddr != "0",
@@ -291,7 +295,7 @@ func main() {
 
 		if err = (&actionsgithubcom.AutoscalingListenerReconciler{
 			Client:                  mgr.GetClient(),
-			Log:                     log.WithName("AutoscalingListener"),
+			Log:                     log.WithName("AutoscalingListener").WithValues("version", build.Version),
 			Scheme:                  mgr.GetScheme(),
 			ListenerMetricsAddr:     listenerMetricsAddr,
 			ListenerMetricsEndpoint: listenerMetricsEndpoint,
@@ -441,7 +445,7 @@ func main() {
 		}
 	}
 
-	log.Info("starting manager")
+	log.Info("starting manager", "version", build.Version)
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		log.Error(err, "problem running manager")
 		os.Exit(1)

runner/Makefile

@@ -6,8 +6,8 @@ DIND_ROOTLESS_RUNNER_NAME ?= ${DOCKER_USER}/actions-runner-dind-rootless
 OS_IMAGE ?= ubuntu-22.04
 TARGETPLATFORM ?= $(shell arch)
 
-RUNNER_VERSION ?= 2.314.1
-RUNNER_CONTAINER_HOOKS_VERSION ?= 0.5.1
+RUNNER_VERSION ?= 2.316.1
+RUNNER_CONTAINER_HOOKS_VERSION ?= 0.6.0
 DOCKER_VERSION ?= 24.0.7
 
 # default list of platforms for which multiarch image is built

runner/VERSION

@@ -1,2 +1,2 @@
-RUNNER_VERSION=2.314.1
-RUNNER_CONTAINER_HOOKS_VERSION=0.5.1
+RUNNER_VERSION=2.316.1
+RUNNER_CONTAINER_HOOKS_VERSION=0.6.0

test/e2e/e2e_test.go

@@ -36,8 +36,8 @@ var (
 
 	testResultCMNamePrefix = "test-result-"
 
-	RunnerVersion               = "2.314.1"
-	RunnerContainerHooksVersion = "0.5.1"
+	RunnerVersion               = "2.316.1"
+	RunnerContainerHooksVersion = "0.6.0"
 )
 
 // If you're willing to run this test via VS Code "run test" or "debug test",