Upgrade resources in-place, causing 1-1 mapping between autoscaling runner set and ephemeral runner set (#4516)

controllers/actions.github.com/autoscalingrunnerset_controller.go

@@ -19,7 +19,7 @@ package actionsgithubcom
 import (
 	"context"
 	"fmt"
-	"sort"
+	"maps"
 	"strconv"
 	"strings"
 	"time"
@@ -28,6 +28,7 @@ import (
 	"github.com/actions/actions-runner-controller/build"
 	"github.com/actions/scaleset"
 	"github.com/go-logr/logr"
+	"github.com/google/go-cmp/cmp"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -42,35 +43,10 @@ import (
 )
 
 const (
-	annotationKeyRunnerSpecHash = "actions.github.com/runner-spec-hash"
-	// annotationKeyValuesHash is hash of the entire values json.
-	// This is used to determine if the values have changed, so we can
-	// re-create listener.
-	annotationKeyValuesHash = "actions.github.com/values-hash"
-	annotationKeyChangeHash = "actions.github.com/change-hash"
-
 	autoscalingRunnerSetFinalizerName = "autoscalingrunnerset.actions.github.com/finalizer"
 	runnerScaleSetIDAnnotationKey     = "runner-scale-set-id"
 )
 
-type UpdateStrategy string
-
-// Defines how the controller should handle upgrades while having running jobs.
-const (
-	// "immediate": (default) The controller will immediately apply the change causing the
-	// recreation of the listener and ephemeral runner set. This can lead to an
-	// overprovisioning of runners, if there are pending / running jobs. This should not
-	// be a problem at a small scale, but it could lead to a significant increase of
-	// resources if you have a lot of jobs running concurrently.
-	UpdateStrategyImmediate = UpdateStrategy("immediate")
-	// "eventual": The controller will remove the listener and ephemeral runner set
-	// immediately, but will not recreate them (to apply changes) until all
-	// pending / running jobs have completed.
-	// This can lead to a longer time to apply the change but it will ensure
-	// that you don't have any overprovisioning of runners.
-	UpdateStrategyEventual = UpdateStrategy("eventual")
-)
-
 // AutoscalingRunnerSetReconciler reconciles a AutoscalingRunnerSet object
 type AutoscalingRunnerSetReconciler struct {
 	client.Client
@@ -79,7 +55,6 @@ type AutoscalingRunnerSetReconciler struct {
 	ControllerNamespace                           string
 	DefaultRunnerScaleSetListenerImage            string
 	DefaultRunnerScaleSetListenerImagePullSecrets []string
-	UpdateStrategy                                UpdateStrategy
 	ResourceBuilder
 }
 
@@ -95,18 +70,19 @@ type AutoscalingRunnerSetReconciler struct {
 func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := r.Log.WithValues("autoscalingrunnerset", req.NamespacedName)
 
-	autoscalingRunnerSet := new(v1alpha1.AutoscalingRunnerSet)
-	if err := r.Get(ctx, req.NamespacedName, autoscalingRunnerSet); err != nil {
+	var autoscalingRunnerSet v1alpha1.AutoscalingRunnerSet
+	if err := r.Get(ctx, req.NamespacedName, &autoscalingRunnerSet); err != nil {
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
+	original := autoscalingRunnerSet.DeepCopy()
 
 	if !autoscalingRunnerSet.DeletionTimestamp.IsZero() {
-		if !controllerutil.ContainsFinalizer(autoscalingRunnerSet, autoscalingRunnerSetFinalizerName) {
+		if !controllerutil.ContainsFinalizer(&autoscalingRunnerSet, autoscalingRunnerSetFinalizerName) {
 			return ctrl.Result{}, nil
 		}
 
 		log.Info("Deleting resources")
-		done, err := r.cleanUpResources(ctx, autoscalingRunnerSet, nil, log)
+		done, err := r.cleanUpResources(ctx, &autoscalingRunnerSet, log)
 		if err != nil {
 			log.Error(err, "Failed to clean up resources during deletion")
 			return ctrl.Result{}, err
@@ -118,18 +94,17 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
 			}, nil
 		}
 
-		if err := r.removeFinalizersFromDependentResources(ctx, autoscalingRunnerSet, log); err != nil {
+		if err := r.removeFinalizersFromDependentResources(ctx, &autoscalingRunnerSet, log); err != nil {
 			log.Error(err, "Failed to remove finalizers on dependent resources")
 			return ctrl.Result{}, err
 		}
 
-		log.Info("Removing finalizer")
-		err = patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) {
-			controllerutil.RemoveFinalizer(obj, autoscalingRunnerSetFinalizerName)
-		})
-		if err != nil && !kerrors.IsNotFound(err) {
-			log.Error(err, "Failed to update autoscaling runner set without finalizer")
-			return ctrl.Result{}, err
+		if controllerutil.RemoveFinalizer(&autoscalingRunnerSet, autoscalingRunnerSetFinalizerName) {
+			log.Info("Removing finalizer")
+			if err := r.Patch(ctx, &autoscalingRunnerSet, client.MergeFrom(original)); err != nil && !kerrors.IsNotFound(err) {
+				log.Error(err, "Failed to update autoscaling runner set without finalizer")
+				return ctrl.Result{}, err
+			}
 		}
 
 		log.Info("Successfully removed finalizer after cleanup")
@@ -137,7 +112,7 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
 	}
 
 	if !v1alpha1.IsVersionAllowed(autoscalingRunnerSet.Labels[LabelKeyKubernetesVersion], build.Version) {
-		if err := r.Delete(ctx, autoscalingRunnerSet); err != nil {
+		if err := r.Delete(ctx, &autoscalingRunnerSet); err != nil {
 			log.Error(
 				err, "Failed to delete autoscaling runner set on version mismatch",
 				"buildVersion", build.Version,
@@ -154,178 +129,267 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
 		return ctrl.Result{}, nil
 	}
 
-	if !controllerutil.ContainsFinalizer(autoscalingRunnerSet, autoscalingRunnerSetFinalizerName) {
+	if controllerutil.AddFinalizer(&autoscalingRunnerSet, autoscalingRunnerSetFinalizerName) {
 		log.Info("Adding finalizer")
-		if err := patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) {
-			controllerutil.AddFinalizer(obj, autoscalingRunnerSetFinalizerName)
-		}); err != nil {
-			log.Error(err, "Failed to update autoscaling runner set with finalizer added")
+
+		if err := r.Patch(ctx, &autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
+			log.Error(err, "Failed to update autoscaling runner set with finalizer")
 			return ctrl.Result{}, err
 		}
 
 		log.Info("Successfully added finalizer")
-	}
-
-	if targetHash := autoscalingRunnerSet.Hash(); autoscalingRunnerSet.Annotations[annotationKeyChangeHash] != targetHash {
-		if err := patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) {
-			if obj.Annotations == nil {
-				obj.Annotations = map[string]string{}
-			}
-			obj.Annotations[annotationKeyChangeHash] = targetHash
-		}); err != nil {
-			log.Error(err, "Failed to update autoscaling runner set with change hash annotation")
-			return ctrl.Result{}, err
-		}
-
-		if err := r.updateStatus(ctx, autoscalingRunnerSet, nil, v1alpha1.AutoscalingRunnerSetPhasePending, log); err != nil {
-			log.Error(err, "Failed to update autoscaling runner set status to pending")
-			return ctrl.Result{}, err
-		}
-	}
-
-	outdated := autoscalingRunnerSet.Status.Phase == v1alpha1.AutoscalingRunnerSetPhaseOutdated
-
-	if !outdated {
-		scaleSetIDRaw, ok := autoscalingRunnerSet.Annotations[runnerScaleSetIDAnnotationKey]
-		if !ok {
-			// Need to create a new runner scale set on Actions service
-			log.Info("Runner scale set id annotation does not exist. Creating a new runner scale set.")
-			return r.createRunnerScaleSet(ctx, autoscalingRunnerSet, log)
-		}
-
-		if id, err := strconv.Atoi(scaleSetIDRaw); err != nil || id <= 0 {
-			log.Info("Runner scale set id annotation is not an id, or is <= 0. Creating a new runner scale set.")
-			// something modified the scaleSetId. Try to create one
-			return r.createRunnerScaleSet(ctx, autoscalingRunnerSet, log)
-		}
-
-		// Make sure the runner group of the scale set is up to date
-		currentRunnerGroupName, ok := autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerGroupName]
-		if !ok || (len(autoscalingRunnerSet.Spec.RunnerGroup) > 0 && !strings.EqualFold(currentRunnerGroupName, autoscalingRunnerSet.Spec.RunnerGroup)) {
-			log.Info("AutoScalingRunnerSet runner group changed. Updating the runner scale set.")
-			return r.updateRunnerScaleSetRunnerGroup(ctx, autoscalingRunnerSet, log)
-		}
-
-		// Make sure the runner scale set name is up to date
-		currentRunnerScaleSetName, ok := autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerScaleSetName]
-		if !ok || (len(autoscalingRunnerSet.Spec.RunnerScaleSetName) > 0 && !strings.EqualFold(currentRunnerScaleSetName, autoscalingRunnerSet.Spec.RunnerScaleSetName)) {
-			log.Info("AutoScalingRunnerSet runner scale set name changed. Updating the runner scale set.")
-			return r.updateRunnerScaleSetName(ctx, autoscalingRunnerSet, log)
-		}
-	}
-
-	existingRunnerSets, err := r.listEphemeralRunnerSets(ctx, autoscalingRunnerSet)
-	if err != nil {
-		log.Error(err, "Failed to list existing ephemeral runner sets")
-		return ctrl.Result{}, err
-	}
-
-	latestRunnerSet := existingRunnerSets.latest()
-	if latestRunnerSet == nil && !outdated {
-		log.Info("Latest runner set does not exist. Creating a new runner set.")
-		return r.createEphemeralRunnerSet(ctx, autoscalingRunnerSet, log)
-	}
-
-	for _, runnerSet := range existingRunnerSets.all() {
-		log.Info("Find existing ephemeral runner set", "name", runnerSet.Name, "specHash", runnerSet.Annotations[annotationKeyRunnerSpecHash])
-	}
-
-	outdated = outdated || (latestRunnerSet != nil && latestRunnerSet.Status.Phase == v1alpha1.EphemeralRunnerSetPhaseOutdated)
-
-	// Make sure the AutoscalingListener is up and running in the controller namespace
-	listener := new(v1alpha1.AutoscalingListener)
-	listenerFound := true
-	if err := r.Get(ctx, client.ObjectKey{Namespace: r.ControllerNamespace, Name: scaleSetListenerName(autoscalingRunnerSet)}, listener); err != nil {
-		if !kerrors.IsNotFound(err) {
-			log.Error(err, "Failed to get AutoscalingListener resource")
-			return ctrl.Result{}, err
-		}
-
-		listenerFound = false
-		log.Info("AutoscalingListener does not exist.")
-	}
-
-	if outdated {
-		log.Info("Ephemeral runner set is outdated")
-		if autoscalingRunnerSet.Status.Phase != v1alpha1.AutoscalingRunnerSetPhaseOutdated {
-			if err := r.updateStatus(ctx, autoscalingRunnerSet, latestRunnerSet, v1alpha1.AutoscalingRunnerSetPhaseOutdated, log); err != nil {
-				log.Error(err, "Failed to update autoscaling runner set status to outdated")
-				return ctrl.Result{}, err
-			}
-		}
-
-		done, err := r.cleanUpResources(ctx, autoscalingRunnerSet, latestRunnerSet, log)
-		if err != nil {
-			log.Error(err, "Failed to clean up resources for outdated runner set")
-			return ctrl.Result{}, err
-		}
-		if done {
-			return ctrl.Result{}, nil
-		}
-
-		log.Info("Waiting for resources to be cleaned up for outdated runner set")
-		return ctrl.Result{
-			RequeueAfter: 5 * time.Second,
-		}, nil
-	}
-
-	// Our listener pod is out of date, so we need to delete it to get a new recreate.
-	listenerValuesHashChanged := listener.Annotations[annotationKeyValuesHash] != autoscalingRunnerSet.Annotations[annotationKeyValuesHash]
-	listenerSpecHashChanged := listener.Annotations[annotationKeyRunnerSpecHash] != autoscalingRunnerSet.ListenerSpecHash()
-	if listenerFound && (listenerValuesHashChanged ||
-		listenerSpecHashChanged ||
-		latestRunnerSet == nil ||
-		listener.Spec.EphemeralRunnerSetName != latestRunnerSet.Name) {
-		log.Info("RunnerScaleSetListener is out of date. Deleting it so that it is recreated", "name", listener.Name)
-		if err := r.Delete(ctx, listener); err != nil {
-			log.Error(err, "Failed to delete AutoscalingListener resource")
-			return ctrl.Result{}, err
-		}
-
-		log.Info("Deleted RunnerScaleSetListener since existing one is out of date")
 		return ctrl.Result{}, nil
 	}
 
-	if latestRunnerSet.Annotations[annotationKeyRunnerSpecHash] != autoscalingRunnerSet.RunnerSetSpecHash() {
-		if r.drainingJobs(&latestRunnerSet.Status) {
-			log.Info("Latest runner set spec hash does not match the current autoscaling runner set. Waiting for the running and pending runners to finish:", "running", latestRunnerSet.Status.RunningEphemeralRunners, "pending", latestRunnerSet.Status.PendingEphemeralRunners)
-			log.Info("Scaling down the number of desired replicas to 0")
-			// We are in the process of draining the jobs. The listener has been deleted and the ephemeral runner set replicas
-			// need to scale down to 0
-			err := patch(ctx, r.Client, latestRunnerSet, func(obj *v1alpha1.EphemeralRunnerSet) {
-				obj.Spec.Replicas = 0
-				obj.Spec.PatchID = 0
-			})
-			if err != nil {
-				log.Error(err, "Failed to patch runner set to set desired count to 0")
-			}
+	// Something has changed, we need to re-apply the pending phase and change hash annotation to trigger the update of runner scale set and listener.
+	if targetHash := autoscalingRunnerSet.Hash(); autoscalingRunnerSet.Annotations[annotationKeyIntegrityHash] != targetHash {
+		// TODO: apply the version label
+		original := autoscalingRunnerSet.DeepCopy()
+		if autoscalingRunnerSet.Annotations == nil {
+			autoscalingRunnerSet.Annotations = map[string]string{}
+		}
+		autoscalingRunnerSet.Annotations[annotationKeyIntegrityHash] = targetHash
+		if err := r.Patch(ctx, &autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
+			log.Error(err, "Failed to update autoscaling runner set with new change hash and pending phase")
 			return ctrl.Result{}, err
 		}
-		log.Info("Latest runner set spec hash does not match the current autoscaling runner set. Creating a new runner set")
-		return r.createEphemeralRunnerSet(ctx, autoscalingRunnerSet, log)
+
+		original = autoscalingRunnerSet.DeepCopy()
+		autoscalingRunnerSet.Status.Phase = v1alpha1.AutoscalingRunnerSetPhasePending
+		if err := r.Status().Patch(ctx, &autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
+			log.Error(err, "Failed to update autoscaling runner set status with pending phase")
+			return ctrl.Result{}, err
+		}
+
+		return ctrl.Result{}, nil
 	}
 
-	oldRunnerSets := existingRunnerSets.old()
-	if len(oldRunnerSets) > 0 {
-		log.Info("Cleanup old ephemeral runner sets", "count", len(oldRunnerSets))
-		err := r.deleteEphemeralRunnerSets(ctx, oldRunnerSets, log)
+	outdated := autoscalingRunnerSet.Status.Phase == v1alpha1.AutoscalingRunnerSetPhaseOutdated
+	if outdated {
+		log.Info("Autoscaling runner set is in outdated phase, removing the listener")
+		done, err := r.cleanupListener(ctx, &autoscalingRunnerSet, log)
 		if err != nil {
-			log.Error(err, "Failed to clean up old runner sets")
+			log.Error(err, "Failed to clean up listener")
 			return ctrl.Result{}, err
 		}
-	}
+		if !done {
+			log.Info("Waiting for listener to be cleaned up for the outdated runner set")
+			return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+		}
+
+		var ephemeralRunnerSet v1alpha1.EphemeralRunnerSet
+		err = r.Get(
+			ctx,
+			types.NamespacedName{
+				Namespace: autoscalingRunnerSet.Namespace,
+				Name:      autoscalingRunnerSet.Name,
+			},
+			&ephemeralRunnerSet,
+		)
+		switch {
+		case kerrors.IsNotFound(err):
+			// If the ephemeral runner set is not found, something removed the ephemeral runner set. The ephemeral runner set should
+			// not be removed by the controller once it is outdated. However, if the ephemeral runner set is removed, it means no ephemeral
+			// runners should be running (or at least no ephemeral runners associated with the ephemeral runner set).
+			// Therefore, this state is acceptable, because the update to the autoscaling runner set will trigger the loop
+			// that will eventually create a new ephemeral runner set.
+			log.Info("Ephemeral runner set is not found. Ignoring the state until the autoscaling runner set is updated")
+			return ctrl.Result{}, nil
+		case err != nil:
+			log.Error(err, "Failed to get ephemeral runner set for the outdated runner set")
+			return ctrl.Result{}, err
+		default:
+			if !ephemeralRunnerSet.DeletionTimestamp.IsZero() {
+				// Same as NotFound case, ignore.
+				return ctrl.Result{}, nil
+			}
+
+			original := ephemeralRunnerSet.DeepCopy()
+			ephemeralRunnerSet.Spec.Replicas = 0
+			ephemeralRunnerSet.Spec.PatchID = 0
+			if err := r.Patch(ctx, &ephemeralRunnerSet, client.MergeFrom(original)); err != nil {
+				log.Error(err, "Failed to patch ephemeral runner set with 0 replicas and reset patch ID for the outdated runner set")
+				return ctrl.Result{}, err
+			}
 
-	// Make sure the AutoscalingListener is up and running in the controller namespace
-	if !listenerFound {
-		if r.drainingJobs(&latestRunnerSet.Status) {
-			log.Info("Creating a new AutoscalingListener is waiting for the running and pending runners to finish. Waiting for the running and pending runners to finish:", "running", latestRunnerSet.Status.RunningEphemeralRunners, "pending", latestRunnerSet.Status.PendingEphemeralRunners)
 			return ctrl.Result{}, nil
 		}
-		log.Info("Creating a new AutoscalingListener for the runner set", "ephemeralRunnerSetName", latestRunnerSet.Name)
-		return r.createAutoScalingListenerForRunnerSet(ctx, autoscalingRunnerSet, latestRunnerSet, log)
 	}
 
-	if err := r.updateStatus(ctx, autoscalingRunnerSet, latestRunnerSet, v1alpha1.AutoscalingRunnerSetPhaseRunning, log); err != nil {
+	if shouldCreateScaleSet(&autoscalingRunnerSet) {
+		log.Info("Creating runner scale set")
+		return r.createRunnerScaleSet(ctx, &autoscalingRunnerSet, log)
+	}
+
+	// Make sure the runner group of the scale set is up to date
+	currentRunnerGroupName, ok := autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerGroupName]
+	if !ok || (len(autoscalingRunnerSet.Spec.RunnerGroup) > 0 && !strings.EqualFold(currentRunnerGroupName, autoscalingRunnerSet.Spec.RunnerGroup)) {
+		log.Info("AutoScalingRunnerSet runner group changed. Updating the runner scale set.")
+		return r.updateRunnerScaleSetRunnerGroup(ctx, &autoscalingRunnerSet, log)
+	}
+
+	// Make sure the runner scale set name is up to date
+	currentRunnerScaleSetName, ok := autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerScaleSetName]
+	if !ok || (len(autoscalingRunnerSet.Spec.RunnerScaleSetName) > 0 && !strings.EqualFold(currentRunnerScaleSetName, autoscalingRunnerSet.Spec.RunnerScaleSetName)) {
+		log.Info("AutoScalingRunnerSet runner scale set name changed. Updating the runner scale set.")
+		return r.updateRunnerScaleSetName(ctx, &autoscalingRunnerSet, log)
+	}
+
+	var ephemeralRunnerSet v1alpha1.EphemeralRunnerSet
+	err := r.Get(
+		ctx,
+		types.NamespacedName{
+			Namespace: autoscalingRunnerSet.Namespace,
+			Name:      autoscalingRunnerSet.Name,
+		},
+		&ephemeralRunnerSet,
+	)
+	switch {
+	case kerrors.IsNotFound(err):
+		log.Info("Creating ephemeral runner set")
+		return r.createEphemeralRunnerSet(ctx, &autoscalingRunnerSet, log)
+	case err != nil:
+		log.Error(err, "Failed to get ephemeral runner")
+		return ctrl.Result{}, err
+	case ephemeralRunnerSet.Status.Phase == v1alpha1.EphemeralRunnerSetPhaseOutdated && autoscalingRunnerSet.Status.Phase == v1alpha1.AutoscalingRunnerSetPhaseRunning:
+		// Runners are outdated. We need to stop the listener so it stops getting new jobs.
+		log.Info("Ephemeral runner set is outdated. Cleaning up resources for the outdated runner set")
+		done, err := r.cleanupListener(ctx, &autoscalingRunnerSet, log)
+		if err != nil {
+			log.Error(err, "Failed to clean up listener for outdated ephemeral runner set")
+			return ctrl.Result{}, err
+		}
+		if !done {
+			log.Info("Waiting for listener to be cleaned up for the outdated ephemeral runner set")
+			return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+		}
+
+		// Then, we need to remove the ephemeral runner set to force scale-down. The ephemeral runner set
+		// will eventually remove all runners as soon as possible.
+		//
+		// The scale set should not be removed yet, since user did not explicitly remove the scale set (or the autoscaling runner set)
+		// Therefore, the autoscaling runner set should stay in outdated state until the spec is updated,
+		// or until the autoscaling runner set is removed.
+		done, err = r.cleanupEphemeralRunnerSet(ctx, &autoscalingRunnerSet, log)
+		if err != nil {
+			log.Error(err, "Failed to clean up ephemeral runner set for outdated runner set")
+			return ctrl.Result{}, err
+		}
+		if !done {
+			log.Info("Waiting for ephemeral runner set to be cleaned up for the outdated runner set")
+			return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+		}
+
+		log.Info("Successfully cleaned up resources for the outdated runner set")
+
+		return ctrl.Result{}, nil
+
+	default:
+		desired, err := r.newEphemeralRunnerSet(&autoscalingRunnerSet)
+		if err != nil {
+			log.Error(err, "Failed to generate ephemeral runner set spec")
+			return ctrl.Result{}, nil
+		}
+
+		if ephemeralRunnerSet.Annotations[annotationKeyIntegrityHash] != desired.Annotations[annotationKeyIntegrityHash] {
+			// When runners are actively processing jobs, defer the spec update:
+			// delete the listener to stop accepting new jobs, but leave the ERS
+			// (and its running pods) untouched until all jobs have drained.
+			if ephemeralRunnerSet.Status.RunningEphemeralRunners+ephemeralRunnerSet.Status.PendingEphemeralRunners > 0 {
+				log.Info("Ephemeral runner set spec changed but runners are still active; deleting listener to stop new jobs")
+				if _, err := r.cleanupListener(ctx, &autoscalingRunnerSet, log); err != nil {
+					log.Error(err, "Failed to clean up listener while waiting for runners to drain")
+					return ctrl.Result{}, err
+				}
+				return ctrl.Result{RequeueAfter: 1 * time.Second}, nil
+			}
+
+			original := ephemeralRunnerSet.DeepCopy()
+			ephemeralRunnerSet.Spec.EphemeralRunnerMetadata = desired.Spec.EphemeralRunnerMetadata
+			ephemeralRunnerSet.Spec.EphemeralRunnerSpec = desired.Spec.EphemeralRunnerSpec
+			ephemeralRunnerSet.Labels = r.filterAndMergeLabels(ephemeralRunnerSet.Labels, desired.Labels)
+			ephemeralRunnerSet.Annotations = r.mergeAnnotations(ephemeralRunnerSet.Annotations, desired.Annotations)
+
+			log.Info("Updating ephemeral runner set spec to match the desired spec")
+			if err := r.Patch(ctx, &ephemeralRunnerSet, client.MergeFrom(original)); err != nil {
+				log.Error(err, "Failed to patch ephemeral runner set to match the desired spec")
+				return ctrl.Result{}, err
+			}
+
+			log.Info("Successfully patched ephemeral runner set spec")
+			return ctrl.Result{}, nil
+		}
+
+		ephemeralRunnerMetadataModified := !cmp.Equal(ephemeralRunnerSet.Spec.EphemeralRunnerMetadata, desired.Spec.EphemeralRunnerMetadata)
+		ephemeralRunnerLabelsModified := !maps.Equal(ephemeralRunnerSet.Labels, desired.Labels)
+		ephemeralRunnerAnnotationsModified := !maps.Equal(ephemeralRunnerSet.Annotations, desired.Annotations)
+
+		if ephemeralRunnerLabelsModified || ephemeralRunnerAnnotationsModified || ephemeralRunnerMetadataModified {
+			original := ephemeralRunnerSet.DeepCopy()
+			ephemeralRunnerSet.Labels = r.filterAndMergeLabels(ephemeralRunnerSet.Labels, desired.Labels)
+			ephemeralRunnerSet.Annotations = r.mergeAnnotations(ephemeralRunnerSet.Annotations, desired.Annotations)
+			ephemeralRunnerSet.Spec.EphemeralRunnerMetadata = desired.Spec.EphemeralRunnerMetadata
+			log.Info("Updating ephemeral runner set metadata to match desired labels and annotations")
+			if err := r.Patch(ctx, &ephemeralRunnerSet, client.MergeFrom(original)); err != nil {
+				log.Error(err, "Failed to patch ephemeral runner set metadata to match desired labels and annotations")
+				return ctrl.Result{}, err
+			}
+
+			log.Info("Successfully patched ephemeral runner set metadata")
+			return ctrl.Result{}, nil
+		}
+	}
+
+	var listener v1alpha1.AutoscalingListener
+	err = r.Get(
+		ctx,
+		types.NamespacedName{
+			Namespace: r.ControllerNamespace,
+			Name:      scaleSetListenerName(&autoscalingRunnerSet),
+		},
+		&listener,
+	)
+	switch {
+	case kerrors.IsNotFound(err):
+		log.Info("AutoscalingListener does not exist, creating autoscaling listener")
+		return r.createAutoScalingListenerForRunnerSet(ctx, &autoscalingRunnerSet, &ephemeralRunnerSet, log)
+	case err != nil:
+		log.Error(err, "Failed to get AutoscalingListener resource")
+		return ctrl.Result{}, err
+	default:
+		desired, err := r.newAutoscalingListener(
+			&autoscalingRunnerSet,
+			&ephemeralRunnerSet,
+			r.ControllerNamespace,
+			r.DefaultRunnerScaleSetListenerImage,
+			nil, // TODO: remove
+		)
+		if err != nil {
+			log.Error(err, "Failed to generate AutoscalingListener spec")
+			return ctrl.Result{}, nil
+		}
+
+		if !cmp.Equal(listener.Spec, desired.Spec) ||
+			!cmp.Equal(listener.Labels, desired.Labels) ||
+			!cmp.Equal(listener.Annotations, desired.Annotations) {
+			log.Info("Deleting AutoscalingListener to re-create with updated spec")
+			if err := r.Delete(ctx, &listener); err != nil {
+				log.Error(err, "Failed to delete AutoscalingListener for re-creation")
+				return ctrl.Result{}, err
+			}
+			log.Info("Deleted AutoscalingListener, will re-create on next reconcile")
+			return ctrl.Result{}, nil
+		}
+	}
+
+	log.Info("Autoscaling runner set is up to date and ready")
+	if err := r.updateStatus(
+		ctx,
+		&autoscalingRunnerSet,
+		&ephemeralRunnerSet,
+		v1alpha1.AutoscalingRunnerSetPhaseRunning,
+		log,
+	); err != nil {
 		log.Error(err, "Failed to update autoscaling runner set status to running")
 		return ctrl.Result{}, err
 	}
@@ -333,7 +397,7 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
 	return ctrl.Result{}, nil
 }
 
-func (r *AutoscalingRunnerSetReconciler) cleanUpResources(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, latestRunnerSet *v1alpha1.EphemeralRunnerSet, log logr.Logger) (bool, error) {
+func (r *AutoscalingRunnerSetReconciler) cleanUpResources(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, log logr.Logger) (bool, error) {
 	log.Info("Deleting the listener")
 	done, err := r.cleanupListener(ctx, autoscalingRunnerSet, log)
 	if err != nil {
@@ -347,7 +411,7 @@ func (r *AutoscalingRunnerSetReconciler) cleanUpResources(ctx context.Context, a
 	}
 
 	log.Info("deleting ephemeral runner sets")
-	done, err = r.cleanupEphemeralRunnerSets(ctx, autoscalingRunnerSet, log)
+	done, err = r.cleanupEphemeralRunnerSet(ctx, autoscalingRunnerSet, log)
 	if err != nil {
 		log.Error(err, "Failed to clean up ephemeral runner sets")
 		return false, err
@@ -371,39 +435,41 @@ func (r *AutoscalingRunnerSetReconciler) cleanUpResources(ctx context.Context, a
 func (r *AutoscalingRunnerSetReconciler) updateStatus(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, phase v1alpha1.AutoscalingRunnerSetPhase, log logr.Logger) error {
 	countDiff := ephemeralRunnerSet != nil && ephemeralRunnerSet.Status.CurrentReplicas != autoscalingRunnerSet.Status.CurrentRunners
 	phaseDiff := phase != autoscalingRunnerSet.Status.Phase
-	if countDiff || phaseDiff {
-		if err := patchSubResource(ctx, r.Status(), autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) {
-			obj.Status.Phase = phase
-			var ephemeralRunnerSetStatus v1alpha1.EphemeralRunnerSetStatus
-			if ephemeralRunnerSet != nil {
-				ephemeralRunnerSetStatus = ephemeralRunnerSet.Status
-			}
-			obj.Status.CurrentRunners = ephemeralRunnerSetStatus.CurrentReplicas
-			obj.Status.PendingEphemeralRunners = ephemeralRunnerSetStatus.PendingEphemeralRunners
-			obj.Status.RunningEphemeralRunners = ephemeralRunnerSetStatus.RunningEphemeralRunners
-			obj.Status.FailedEphemeralRunners = ephemeralRunnerSetStatus.FailedEphemeralRunners
-		}); err != nil {
-			log.Error(err, "Failed to update autoscaling runner set status with current runner count")
-			return err
-		}
+	if !countDiff && !phaseDiff {
+		return nil
 	}
-	return nil
-}
 
-// Prevents overprovisioning of runners.
-// We reach this code path when runner scale set has been patched with a new runner spec but there are still running ephemeral runners.
-// The safest approach is to wait for the running ephemeral runners to finish before creating a new runner set.
-func (r *AutoscalingRunnerSetReconciler) drainingJobs(latestRunnerSetStatus *v1alpha1.EphemeralRunnerSetStatus) bool {
-	if r.UpdateStrategy == UpdateStrategyEventual && ((latestRunnerSetStatus.RunningEphemeralRunners + latestRunnerSetStatus.PendingEphemeralRunners) > 0) {
-		return true
+	original := autoscalingRunnerSet.DeepCopy()
+	if phaseDiff {
+		autoscalingRunnerSet.Status.Phase = phase
 	}
-	return false
+
+	if countDiff && ephemeralRunnerSet != nil {
+		autoscalingRunnerSet.Status.CurrentRunners = ephemeralRunnerSet.Status.CurrentReplicas
+		autoscalingRunnerSet.Status.PendingEphemeralRunners = ephemeralRunnerSet.Status.PendingEphemeralRunners
+		autoscalingRunnerSet.Status.RunningEphemeralRunners = ephemeralRunnerSet.Status.RunningEphemeralRunners
+		autoscalingRunnerSet.Status.FailedEphemeralRunners = ephemeralRunnerSet.Status.FailedEphemeralRunners
+	}
+
+	if err := r.Status().Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
+		log.Error(err, "Failed to patch autoscaling runner set status")
+		return err
+	}
+
+	return nil
 }
 
 func (r *AutoscalingRunnerSetReconciler) cleanupListener(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, logger logr.Logger) (done bool, err error) {
 	logger.Info("Cleaning up the listener")
 	var listener v1alpha1.AutoscalingListener
-	err = r.Get(ctx, client.ObjectKey{Namespace: r.ControllerNamespace, Name: scaleSetListenerName(autoscalingRunnerSet)}, &listener)
+	err = r.Get(
+		ctx,
+		client.ObjectKey{
+			Namespace: r.ControllerNamespace,
+			Name:      scaleSetListenerName(autoscalingRunnerSet),
+		},
+		&listener,
+	)
 	switch {
 	case err == nil:
 		if listener.DeletionTimestamp.IsZero() {
@@ -421,39 +487,32 @@ func (r *AutoscalingRunnerSetReconciler) cleanupListener(ctx context.Context, au
 	return true, nil
 }
 
-func (r *AutoscalingRunnerSetReconciler) cleanupEphemeralRunnerSets(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, logger logr.Logger) (done bool, err error) {
-	logger.Info("Cleaning up ephemeral runner sets")
-	runnerSets, err := r.listEphemeralRunnerSets(ctx, autoscalingRunnerSet)
-	if err != nil {
-		return false, fmt.Errorf("failed to list ephemeral runner sets: %w", err)
-	}
-	if runnerSets.empty() {
-		logger.Info("All ephemeral runner sets are deleted")
-		return true, nil
+func (r *AutoscalingRunnerSetReconciler) cleanupEphemeralRunnerSet(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, logger logr.Logger) (done bool, err error) {
+	logger.Info("Cleaning up ephemeral runner set")
+	var ers v1alpha1.EphemeralRunnerSet
+	err = r.Get(
+		ctx,
+		client.ObjectKey{
+			Namespace: autoscalingRunnerSet.Namespace,
+			Name:      autoscalingRunnerSet.Name,
+		},
+		&ers,
+	)
+	switch {
+	case err == nil:
+		if ers.DeletionTimestamp.IsZero() {
+			logger.Info("Deleting the ephemeral runner set")
+			if err := r.Delete(ctx, &ers); err != nil {
+				return false, fmt.Errorf("failed to delete ephemeral runner set: %w", err)
+			}
+		}
+		return false, nil
+	case !kerrors.IsNotFound(err):
+		return false, fmt.Errorf("failed to get ephemeral runner set: %w", err)
 	}
 
-	logger.Info("Deleting all ephemeral runner sets", "count", runnerSets.count())
-	if err := r.deleteEphemeralRunnerSets(ctx, runnerSets.all(), logger); err != nil {
-		return false, fmt.Errorf("failed to delete ephemeral runner sets: %w", err)
-	}
-	return false, nil
-}
-
-func (r *AutoscalingRunnerSetReconciler) deleteEphemeralRunnerSets(ctx context.Context, oldRunnerSets []v1alpha1.EphemeralRunnerSet, logger logr.Logger) error {
-	for i := range oldRunnerSets {
-		rs := &oldRunnerSets[i]
-		// already deleted but contains finalizer so it still exists
-		if !rs.DeletionTimestamp.IsZero() {
-			logger.Info("Skip ephemeral runner set since it is already marked for deletion", "name", rs.Name)
-			continue
-		}
-		logger.Info("Deleting ephemeral runner set", "name", rs.Name)
-		if err := r.Delete(ctx, rs); err != nil {
-			return fmt.Errorf("failed to delete EphemeralRunnerSet resource: %w", err)
-		}
-		logger.Info("Deleted ephemeral runner set", "name", rs.Name)
-	}
-	return nil
+	logger.Info("Ephemeral runner set is deleted")
+	return true, nil
 }
 
 func (r *AutoscalingRunnerSetReconciler) removeFinalizersFromDependentResources(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, logger logr.Logger) error {
@@ -612,11 +671,11 @@ func (r *AutoscalingRunnerSetReconciler) updateRunnerScaleSetRunnerGroup(ctx con
 	}
 
 	logger.Info("Updating runner scale set name and runner group name as annotations")
-	if err := patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) {
-		obj.Annotations[AnnotationKeyGitHubRunnerGroupName] = updatedRunnerScaleSet.RunnerGroupName
-		obj.Annotations[AnnotationKeyGitHubRunnerScaleSetName] = updatedRunnerScaleSet.Name
-	}); err != nil {
-		logger.Error(err, "Failed to update runner group name annotation")
+	original := autoscalingRunnerSet.DeepCopy()
+	autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerGroupName] = updatedRunnerScaleSet.RunnerGroupName
+	autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerScaleSetName] = updatedRunnerScaleSet.Name
+	if err := r.Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
+		logger.Error(err, "Failed to update runner group name and runner scale set name annotation")
 		return ctrl.Result{}, err
 	}
 
@@ -649,9 +708,9 @@ func (r *AutoscalingRunnerSetReconciler) updateRunnerScaleSetName(ctx context.Co
 	}
 
 	logger.Info("Updating runner scale set name as an annotation")
-	if err := patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) {
-		obj.Annotations[AnnotationKeyGitHubRunnerScaleSetName] = updatedRunnerScaleSet.Name
-	}); err != nil {
+	original := autoscalingRunnerSet.DeepCopy()
+	autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerScaleSetName] = updatedRunnerScaleSet.Name
+	if err := r.Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
 		logger.Error(err, "Failed to update runner scale set name annotation")
 		return ctrl.Result{}, err
 	}
@@ -669,7 +728,7 @@ func (r *AutoscalingRunnerSetReconciler) deleteRunnerScaleSet(ctx context.Contex
 		//
 		// 2. The scale set has been deleted by the controller.
 		//    In that case, the controller will clean up annotation because the scale set does not exist anymore.
-		//    Removal of the scale set id is also useful because permission cleanup will eventually lose permission
+		//    Removal of the scale set id is also useful because permission cleanup will later lose permission
 		//    assigned to it on a GitHub secret, causing actions client from secret to result in permission denied
 		//
 		// 3. Annotation is removed manually.
@@ -699,11 +758,11 @@ func (r *AutoscalingRunnerSetReconciler) deleteRunnerScaleSet(ctx context.Contex
 		return err
 	}
 
-	err = patch(ctx, r.Client, autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) {
-		delete(obj.Annotations, runnerScaleSetIDAnnotationKey)
-	})
-	if err != nil {
-		logger.Error(err, "Failed to patch autoscaling runner set with annotation removed", "annotation", runnerScaleSetIDAnnotationKey)
+	original := autoscalingRunnerSet.DeepCopy()
+	delete(autoscalingRunnerSet.Annotations, runnerScaleSetIDAnnotationKey)
+
+	if err := r.Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
+		logger.Error(err, "Failed to remove runner scale set ID annotation after deleting the runner scale set", "runnerScaleSetId", runnerScaleSetID)
 		return err
 	}
 
@@ -718,11 +777,6 @@ func (r *AutoscalingRunnerSetReconciler) createEphemeralRunnerSet(ctx context.Co
 		return ctrl.Result{}, err
 	}
 
-	if err := ctrl.SetControllerReference(autoscalingRunnerSet, desiredRunnerSet, r.Scheme); err != nil {
-		log.Error(err, "Failed to set controller reference to a new EphemeralRunnerSet")
-		return ctrl.Result{}, err
-	}
-
 	log.Info("Creating a new EphemeralRunnerSet resource")
 	if err := r.Create(ctx, desiredRunnerSet); err != nil {
 		log.Error(err, "Failed to create EphemeralRunnerSet resource")
@@ -741,7 +795,13 @@ func (r *AutoscalingRunnerSetReconciler) createAutoScalingListenerForRunnerSet(c
 		})
 	}
 
-	autoscalingListener, err := r.newAutoScalingListener(autoscalingRunnerSet, ephemeralRunnerSet, r.ControllerNamespace, r.DefaultRunnerScaleSetListenerImage, imagePullSecrets)
+	autoscalingListener, err := r.newAutoscalingListener(
+		autoscalingRunnerSet,
+		ephemeralRunnerSet,
+		r.ControllerNamespace,
+		r.DefaultRunnerScaleSetListenerImage,
+		imagePullSecrets,
+	)
 	if err != nil {
 		log.Error(err, "Could not create AutoscalingListener spec")
 		return ctrl.Result{}, err
@@ -757,17 +817,20 @@ func (r *AutoscalingRunnerSetReconciler) createAutoScalingListenerForRunnerSet(c
 	return ctrl.Result{}, nil
 }
 
-func (r *AutoscalingRunnerSetReconciler) listEphemeralRunnerSets(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) (*EphemeralRunnerSets, error) {
-	list := new(v1alpha1.EphemeralRunnerSetList)
-	if err := r.List(ctx, list, client.InNamespace(autoscalingRunnerSet.Namespace), client.MatchingFields{resourceOwnerKey: autoscalingRunnerSet.Name}); err != nil {
-		return nil, fmt.Errorf("failed to list ephemeral runner sets: %w", err)
+// TODO: change that
+func shouldCreateScaleSet(autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) bool {
+	scaleSetIDRaw, ok := autoscalingRunnerSet.Annotations[runnerScaleSetIDAnnotationKey]
+	if !ok {
+		return true
 	}
-
-	return &EphemeralRunnerSets{list: list}, nil
+	id, err := strconv.Atoi(scaleSetIDRaw)
+	return err != nil || id <= 0
 }
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *AutoscalingRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error {
+	r.ResourceBuilder.setSchemeIfUnset(r.Scheme)
+
 	return builderWithOptions(
 		ctrl.NewControllerManagedBy(mgr).
 			For(&v1alpha1.AutoscalingRunnerSet{}).
@@ -829,12 +892,13 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRol
 			c.logger.Info("Kubernetes mode role binding finalizer has already been removed", "name", roleBindingName)
 			return
 		}
-		err = patch(ctx, c.client, roleBinding, func(obj *rbacv1.RoleBinding) {
-			controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName)
-		})
-		if err != nil {
-			c.err = fmt.Errorf("failed to patch kubernetes mode role binding without finalizer: %w", err)
-			return
+
+		original := roleBinding.DeepCopy()
+		if controllerutil.RemoveFinalizer(roleBinding, AutoscalingRunnerSetCleanupFinalizerName) {
+			if err = c.client.Patch(ctx, roleBinding, client.MergeFrom(original)); err != nil {
+				c.err = fmt.Errorf("failed to patch kubernetes mode role binding without finalizer: %w", err)
+				return
+			}
 		}
 		c.logger.Info("Removed finalizer from container mode kubernetes role binding", "name", roleBindingName)
 		return
@@ -871,12 +935,12 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRol
 			c.logger.Info("Kubernetes mode role finalizer has already been removed", "name", roleName)
 			return
 		}
-		err = patch(ctx, c.client, role, func(obj *rbacv1.Role) {
-			controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName)
-		})
-		if err != nil {
-			c.err = fmt.Errorf("failed to patch kubernetes mode role without finalizer: %w", err)
-			return
+		original := role.DeepCopy()
+		if controllerutil.RemoveFinalizer(role, AutoscalingRunnerSetCleanupFinalizerName) {
+			if err = c.client.Patch(ctx, role, client.MergeFrom(original)); err != nil {
+				c.err = fmt.Errorf("failed to patch kubernetes mode role without finalizer: %w", err)
+				return
+			}
 		}
 		c.logger.Info("Removed finalizer from container mode kubernetes role")
 		return
@@ -914,13 +978,14 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeSer
 			c.logger.Info("Kubernetes mode service account finalizer has already been removed", "name", serviceAccountName)
 			return
 		}
-		err = patch(ctx, c.client, serviceAccount, func(obj *corev1.ServiceAccount) {
-			controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName)
-		})
-		if err != nil {
-			c.err = fmt.Errorf("failed to patch kubernetes mode service account without finalizer: %w", err)
-			return
+		original := serviceAccount.DeepCopy()
+		if controllerutil.RemoveFinalizer(serviceAccount, AutoscalingRunnerSetCleanupFinalizerName) {
+			if err = c.client.Patch(ctx, serviceAccount, client.MergeFrom(original)); err != nil {
+				c.err = fmt.Errorf("failed to patch kubernetes mode service account without finalizer: %w", err)
+				return
+			}
 		}
+
 		c.logger.Info("Removed finalizer from container mode kubernetes service account")
 		return
 	case kerrors.IsNotFound(err):
@@ -950,20 +1015,29 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeNoPermissionServi
 	c.logger.Info("Removing finalizer from no permission service account", "name", serviceAccountName)
 
 	serviceAccount := new(corev1.ServiceAccount)
-	err := c.client.Get(ctx, types.NamespacedName{Name: serviceAccountName, Namespace: c.autoscalingRunnerSet.Namespace}, serviceAccount)
+	err := c.client.Get(
+		ctx,
+		types.NamespacedName{
+			Name:      serviceAccountName,
+			Namespace: c.autoscalingRunnerSet.Namespace,
+		},
+		serviceAccount,
+	)
 	switch {
 	case err == nil:
 		if !controllerutil.ContainsFinalizer(serviceAccount, AutoscalingRunnerSetCleanupFinalizerName) {
 			c.logger.Info("No permission service account finalizer has already been removed", "name", serviceAccountName)
 			return
 		}
-		err = patch(ctx, c.client, serviceAccount, func(obj *corev1.ServiceAccount) {
-			controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName)
-		})
-		if err != nil {
-			c.err = fmt.Errorf("failed to patch service account without finalizer: %w", err)
-			return
+		original := serviceAccount.DeepCopy()
+
+		if controllerutil.RemoveFinalizer(serviceAccount, AutoscalingRunnerSetCleanupFinalizerName) {
+			if err = c.client.Patch(ctx, serviceAccount, client.MergeFrom(original)); err != nil {
+				c.err = fmt.Errorf("failed to patch no permission service account without finalizer: %w", err)
+				return
+			}
 		}
+
 		c.logger.Info("Removed finalizer from no permission service account", "name", serviceAccountName)
 		return
 	case kerrors.IsNotFound(err):
@@ -1000,12 +1074,13 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeGitHubSecretFinal
 			c.logger.Info("GitHub secret finalizer has already been removed", "name", githubSecretName)
 			return
 		}
-		err = patch(ctx, c.client, githubSecret, func(obj *corev1.Secret) {
-			controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName)
-		})
-		if err != nil {
-			c.err = fmt.Errorf("failed to patch GitHub secret without finalizer: %w", err)
-			return
+		original := githubSecret.DeepCopy()
+
+		if controllerutil.RemoveFinalizer(githubSecret, AutoscalingRunnerSetCleanupFinalizerName) {
+			if err = c.client.Patch(ctx, githubSecret, client.MergeFrom(original)); err != nil {
+				c.err = fmt.Errorf("failed to patch GitHub secret without finalizer: %w", err)
+				return
+			}
 		}
 		c.logger.Info("Removed finalizer from GitHub secret", "name", githubSecretName)
 		return
@@ -1039,17 +1114,14 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleBindin
 	err := c.client.Get(ctx, types.NamespacedName{Name: managerRoleBindingName, Namespace: c.autoscalingRunnerSet.Namespace}, roleBinding)
 	switch {
 	case err == nil:
-		if !controllerutil.ContainsFinalizer(roleBinding, AutoscalingRunnerSetCleanupFinalizerName) {
-			c.logger.Info("Manager role binding finalizer has already been removed", "name", managerRoleBindingName)
-			return
-		}
-		err = patch(ctx, c.client, roleBinding, func(obj *rbacv1.RoleBinding) {
-			controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName)
-		})
-		if err != nil {
-			c.err = fmt.Errorf("failed to patch manager role binding without finalizer: %w", err)
-			return
+		original := roleBinding.DeepCopy()
+		if controllerutil.RemoveFinalizer(roleBinding, AutoscalingRunnerSetCleanupFinalizerName) {
+			if err = c.client.Patch(ctx, roleBinding, client.MergeFrom(original)); err != nil {
+				c.err = fmt.Errorf("failed to patch manager role binding without finalizer: %w", err)
+				return
+			}
 		}
+
 		c.logger.Info("Removed finalizer from manager role binding", "name", managerRoleBindingName)
 		return
 	case kerrors.IsNotFound(err):
@@ -1082,16 +1154,12 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleFinali
 	err := c.client.Get(ctx, types.NamespacedName{Name: managerRoleName, Namespace: c.autoscalingRunnerSet.Namespace}, role)
 	switch {
 	case err == nil:
-		if !controllerutil.ContainsFinalizer(role, AutoscalingRunnerSetCleanupFinalizerName) {
-			c.logger.Info("Manager role finalizer has already been removed", "name", managerRoleName)
-			return
-		}
-		err = patch(ctx, c.client, role, func(obj *rbacv1.Role) {
-			controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName)
-		})
-		if err != nil {
-			c.err = fmt.Errorf("failed to patch manager role without finalizer: %w", err)
-			return
+		original := role.DeepCopy()
+		if controllerutil.RemoveFinalizer(role, AutoscalingRunnerSetCleanupFinalizerName) {
+			if err := c.client.Patch(ctx, role, client.MergeFrom(original)); err != nil {
+				c.err = fmt.Errorf("failed to patch manager role without finalizer: %w", err)
+				return
+			}
 		}
 		c.logger.Info("Removed finalizer from manager role", "name", managerRoleName)
 		return
@@ -1103,54 +1171,3 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleFinali
 		return
 	}
 }
-
-// NOTE: if this is logic should be used for other resources,
-// consider using generics
-
-type EphemeralRunnerSets struct {
-	list   *v1alpha1.EphemeralRunnerSetList
-	sorted bool
-}
-
-func (rs *EphemeralRunnerSets) latest() *v1alpha1.EphemeralRunnerSet {
-	if rs.empty() {
-		return nil
-	}
-	if !rs.sorted {
-		rs.sort()
-	}
-	return rs.list.Items[0].DeepCopy()
-}
-
-func (rs *EphemeralRunnerSets) old() []v1alpha1.EphemeralRunnerSet {
-	if rs.empty() {
-		return nil
-	}
-	if !rs.sorted {
-		rs.sort()
-	}
-	copy := rs.list.DeepCopy()
-	return copy.Items[1:]
-}
-
-func (rs *EphemeralRunnerSets) all() []v1alpha1.EphemeralRunnerSet {
-	if rs.empty() {
-		return nil
-	}
-	copy := rs.list.DeepCopy()
-	return copy.Items
-}
-
-func (rs *EphemeralRunnerSets) empty() bool {
-	return rs.list == nil || len(rs.list.Items) == 0
-}
-
-func (rs *EphemeralRunnerSets) sort() {
-	sort.Slice(rs.list.Items, func(i, j int) bool {
-		return rs.list.Items[i].GetCreationTimestamp().After(rs.list.Items[j].GetCreationTimestamp().Time)
-	})
-}
-
-func (rs *EphemeralRunnerSets) count() int {
-	return len(rs.list.Items)
-}