public_git/actions-runner-controller
1
0
Fork 0
mirror of https://github.com/actions-runner-controller/actions-runner-controller.git synced 2026-10-22 08:45:42 +02:00
Code Issues Projects Releases Wiki Activity

Make the Outdated phase revision-aware

Browse Source 
An EphemeralRunner that exits as Outdated marks its EphemeralRunnerSet
Outdated, and the AutoscalingRunnerSet then stops scaling it. Without a way
to tell which runner spec an Outdated report refers to, a report from a
runner built before the spec was updated keeps the set switched off after
the update that was supposed to fix it.

Stamp each runner with the actionable revision it was built from, and judge
Outdated reports against the revision the set has applied:

  - A runner whose revision is older than the applied one is reporting on a
    spec that has already been replaced. It is deleted and rebuilt from the
    current spec, and it does not hold the set Outdated.
  - A runner whose revision is current is reporting on the live spec, so the
    set stays Outdated. It is deliberately not delete-and-replaced: a fresh
    runner at the same revision would report Outdated again, forever.

Runners without the annotation parse to revision 0, which matches the zero
value of Status.AppliedActionableRevision, so existing runners keep their
current behaviour across an upgrade.

The phase is derived inside patchAppliedActionableRevisionStatus, from a
list read through the same authoritative reader as the set itself, because
the optimistic lock on that patch covers the EphemeralRunnerSet object only
and cannot vouch for a separately-read list. The runners are classified
against the live applied revision rather than the revision this call was
asked to apply: the caller reads the spec from the cache while this function
re-reads the status from the API server, so a lagging reconcile can arrive
with a target behind the live marker, and judging against it would flip a
set that has already moved on back to Outdated.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Nikola Jokic
2026-09-12 23:18:26 +02:00
co-authored by Copilot App
parent 03328aa14f
commit 854c0517a7
14 changed files with 1007 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
controllers/actions.github.com/ephemeralrunnerset_controller.go
+149 -20
View File
@@ -22,6 +22,7 @@ import (
"errors"
"fmt"
"maps"
"slices"
"sort"
"strconv"
"time"
@@ -196,11 +197,12 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
return ctrl.Result{}, err
}
ephemeralRunnersByState := newEphemeralRunnersByStates(&ephemeralRunnerList)
ephemeralRunnersByState := newEphemeralRunnersByStates(&ephemeralRunnerList, ephemeralRunnerSet.Status.AppliedActionableRevision)
log.Info(
"Ephemeral runner counts",
"outdated", len(ephemeralRunnersByState.outdated),
"staleOutdated", len(ephemeralRunnersByState.staleOutdated),
"pending", len(ephemeralRunnersByState.pending),
"running", len(ephemeralRunnersByState.running),
"finished", len(ephemeralRunnersByState.finished),
@@ -208,6 +210,23 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
"deleting", len(ephemeralRunnersByState.deleting),
)
// Runners that reported Outdated against a runner spec that has since been
// replaced are not evidence about the current spec. Drop them so the scaling
// logic below replaces them with runners built from the current spec, instead
// of letting them hold the set in the Outdated phase forever.
if len(ephemeralRunnersByState.staleOutdated) > 0 {
log.Info(
"Deleting outdated ephemeral runners created before the last spec update so they can be replaced",
"count", len(ephemeralRunnersByState.staleOutdated),
"appliedActionableRevision", ephemeralRunnerSet.Status.AppliedActionableRevision,
)
if err := r.deleteTerminatedEphemeralRunners(ctx, ephemeralRunnersByState.staleOutdated, log); err != nil {
log.Error(err, "failed to delete stale outdated ephemeral runners")
return ctrl.Result{}, err
}
return ctrl.Result{}, r.updateStatus(ctx, &ephemeralRunnerSet, ephemeralRunnersByState, log)
}
total := ephemeralRunnersByState.scaleTotal()
if ephemeralRunnerSet.Spec.PatchID == 0 || ephemeralRunnerSet.Spec.PatchID != ephemeralRunnersByState.latestPatchID {
// Spec.Replicas is the count the listener asked for when it published
@@ -284,8 +303,11 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
return ctrl.Result{}, r.updateStatus(ctx, &ephemeralRunnerSet, ephemeralRunnersByState, log)
}
// patchAppliedActionableRevisionStatus records that the runner spec carried by
// targetAppliedRevision has been fully applied.
// patchAppliedActionableRevisionStatus brings status into line with the runner
// spec carried by targetAppliedRevision once that spec has been fully applied.
// It records the applied revision, clears the scale-up suppression marker when
// the revision actually advances, and re-derives Status.Phase from the child
// runners.
//
// The marker lives in status rather than in an annotation on the spec, and it is
// written only once the cleanup above has actually succeeded. If the controller
@@ -315,6 +337,11 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
// if the re-fetched object is still the live one, so a successful patch proves
// the monotonicity check above was evaluated against live data. A stale attempt
// conflicts and is retried or requeued instead of regressing the marker.
//
// The lock covers the EphemeralRunnerSet object and nothing else. The phase is
// derived from a separate list of the child runners, which no precondition on
// this patch can vouch for, so that list is read through the same authoritative
// reader rather than the cache.
func (r *EphemeralRunnerSetReconciler) patchAppliedActionableRevisionStatus(ctx context.Context, key types.NamespacedName, targetAppliedRevision int64) error {
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
var latest v1alpha1.EphemeralRunnerSet
@@ -326,22 +353,88 @@ func (r *EphemeralRunnerSetReconciler) patchAppliedActionableRevisionStatus(ctx
return err
}
if latest.Status.AppliedActionableRevision >= targetAppliedRevision {
return nil
original := latest.DeepCopy()
// Only an advance means the idle and pending runners were just deleted and
// the listener restarted. Guarding both writes on it keeps this callable
// as a plain "make sure status reflects revision N" without disturbing a
// marker that still describes the live patch sequence.
if latest.Status.AppliedActionableRevision < targetAppliedRevision {
latest.Status.AppliedActionableRevision = targetAppliedRevision
// The marker records a patch ID from the sequence that was current
// before this spec change. Applying a new revision deletes the idle and
// pending runners, so the shortfall that follows belongs to the new spec
// and must be filled. Worse, a spec change restarts the listener, and a
// restarted listener numbers its patches from 0 upwards, counting
// through every integer. It therefore passes through a leftover marker
// value with near-certainty, and would suppress the very scale up that
// rebuilds the pool.
latest.Status.FinishedRunnerCleanupPatchID = 0
}
original := latest.DeepCopy()
latest.Status.AppliedActionableRevision = targetAppliedRevision
ephemeralRunnerList := new(v1alpha1.EphemeralRunnerList)
// Listed through the same authoritative reader as the Get above. The
// optimistic lock on the patch below covers the EphemeralRunnerSet object
// only, so it cannot vouch for a separately-read list: deriving the phase
// from the cache would let a successful, lock-protected write carry a
// value the lock says nothing about. The list also sits inside
// RetryOnConflict, and a cached list can return the same stale data on
// every attempt, spending the whole backoff re-deriving one wrong phase.
//
// resourceOwnerKey cannot be used here. It is a client-side index
// registered on the manager's cache, and the API server rejects it as an
// unsupported field label, so the ownership filter has to be applied in
// this process instead.
//
// Narrowing server-side by label is not a safe alternative either. Label
// propagation is operator-configurable through
// --exclude-label-propagation-prefix, so the scale set labels are not
// guaranteed to reach the runners, and a selector that silently matched
// none of them would derive the phase from an empty list rather than
// fail. A namespace can hold more than one scale set, so this does read
// runners that are not ours, but it only runs when a revision actually
// advances rather than on every reconcile.
if err := reader.List(ctx, ephemeralRunnerList, client.InNamespace(latest.Namespace)); err != nil {
return fmt.Errorf("failed to list child ephemeral runners: %w", err)
}
ephemeralRunnerList.Items = slices.DeleteFunc(ephemeralRunnerList.Items, func(runner v1alpha1.EphemeralRunner) bool {
return !isControlledBy(&runner, "EphemeralRunnerSet", latest.Name)
})
// The marker records a patch ID from the sequence that was current before
// this spec change. Applying a new revision deletes the idle and pending
// runners, so the shortfall that follows belongs to the new spec and must
// be filled. Worse, a spec change restarts the listener, and a restarted
// listener numbers its patches from 0 upwards, counting through every
// integer. It therefore passes through a leftover marker value with
// near-certainty, and would suppress the very scale up that rebuilds the
// pool.
latest.Status.FinishedRunnerCleanupPatchID = 0
// Judge the runners against the revision the set has now applied, rather
// than the one this call was asked to apply: every runner created before
// that revision is stale by definition, so its Outdated report says
// nothing about the current spec. This is what lets a spec update clear
// the Outdated phase immediately rather than waiting for the pre-update
// runners to be collected.
//
// After the guard above, this field is max(live, target), and the two
// differ in a case that matters. The caller reads the spec from the
// cache while this function re-reads the status from the API server, so
// a lagging reconcile can arrive with a target behind the live marker.
// Judging against that lower target would rate a runner left over from
// the superseded revision as current and flip a set that has already
// moved on back to Outdated. That phase is deliberately absorbing, so
// the set would then stay switched off until the next spec change.
state := newEphemeralRunnersByStates(ephemeralRunnerList, latest.Status.AppliedActionableRevision)
// Set the phase in both directions. This function returns early from
// Reconcile without reaching updateStatus, so leaving the phase untouched
// would let a stale value survive: a stale Running would hide genuinely
// outdated runners from the cleanup path, and a stale Outdated would keep
// the set switched off after the spec that caused it was replaced.
if len(state.outdated) > 0 {
latest.Status.Phase = v1alpha1.EphemeralRunnerSetPhaseOutdated
} else {
latest.Status.Phase = v1alpha1.EphemeralRunnerSetPhaseRunning
}
// Checked after every field above has been set, so that clearing the
// marker alone is still enough to issue the patch.
if original.Status == latest.Status {
return nil
}
return r.Status().Patch(ctx, &latest, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{}))
})
@@ -539,7 +632,7 @@ func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunners(ctx context.Conte
return true, nil
}
ephemeralRunnerState := newEphemeralRunnersByStates(ephemeralRunnerList)
ephemeralRunnerState := newEphemeralRunnersByStates(ephemeralRunnerList, ephemeralRunnerSet.Status.AppliedActionableRevision)
log.Info(
"Clean up runner counts",
@@ -914,12 +1007,27 @@ type ephemeralRunnersByState struct {
finished []*v1alpha1.EphemeralRunner
failed []*v1alpha1.EphemeralRunner
deleting []*v1alpha1.EphemeralRunner
// outdated holds runners that reported Outdated against the runner spec that
// is currently applied. They are evidence that the current spec is still
// rejected by the service, so they drive the set into the Outdated phase.
outdated []*v1alpha1.EphemeralRunner
// staleOutdated holds runners that reported Outdated against a runner spec
// that has since been replaced. They say nothing about the current spec, so
// they must not drive the set into the Outdated phase; they are deleted and
// replaced by runners built from the current spec instead.
staleOutdated []*v1alpha1.EphemeralRunner
latestPatchID int
}
func newEphemeralRunnersByStates(ephemeralRunnerList *v1alpha1.EphemeralRunnerList) *ephemeralRunnersByState {
// newEphemeralRunnersByStates groups the child runners by state.
//
// appliedActionableRevision is the EphemeralRunnerSet revision the runners are
// being judged against. A runner that reported Outdated before that revision was
// applied is classified as stale rather than outdated, so that updating the
// runner spec clears the Outdated phase immediately instead of waiting for the
// pre-update runners to disappear.
func newEphemeralRunnersByStates(ephemeralRunnerList *v1alpha1.EphemeralRunnerList, appliedActionableRevision int64) *ephemeralRunnersByState {
var ephemeralRunnerState ephemeralRunnersByState
for i := range ephemeralRunnerList.Items {
@@ -941,7 +1049,11 @@ func newEphemeralRunnersByStates(ephemeralRunnerList *v1alpha1.EphemeralRunnerLi
case v1alpha1.EphemeralRunnerPhaseFailed:
ephemeralRunnerState.failed = append(ephemeralRunnerState.failed, r)
case v1alpha1.EphemeralRunnerPhaseOutdated:
ephemeralRunnerState.outdated = append(ephemeralRunnerState.outdated, r)
if ephemeralRunnerActionableRevision(r) < appliedActionableRevision {
ephemeralRunnerState.staleOutdated = append(ephemeralRunnerState.staleOutdated, r)
} else {
ephemeralRunnerState.outdated = append(ephemeralRunnerState.outdated, r)
}
default:
// Pending or no phase should be considered as pending.
//
@@ -953,8 +1065,25 @@ func newEphemeralRunnersByStates(ephemeralRunnerList *v1alpha1.EphemeralRunnerLi
return &ephemeralRunnerState
}
// ephemeralRunnerActionableRevision reports the EphemeralRunnerSet revision the
// runner was created from. Runners created before this annotation existed report
// 0, which matches the zero value of Status.AppliedActionableRevision, so they
// are treated as current until the spec is updated for the first time.
func ephemeralRunnerActionableRevision(ephemeralRunner *v1alpha1.EphemeralRunner) int64 {
revision, err := strconv.ParseInt(ephemeralRunner.Annotations[AnnotationKeyActionableRevision], 10, 64)
if err != nil {
return 0
}
return revision
}
func (s *ephemeralRunnersByState) terminated() []*v1alpha1.EphemeralRunner {
return append(s.finished, append(s.failed, s.outdated...)...)
terminated := make([]*v1alpha1.EphemeralRunner, 0, len(s.finished)+len(s.failed)+len(s.outdated)+len(s.staleOutdated))
terminated = append(terminated, s.finished...)
terminated = append(terminated, s.failed...)
terminated = append(terminated, s.outdated...)
terminated = append(terminated, s.staleOutdated...)
return terminated
}
func (s *ephemeralRunnersByState) scaleTotal() int {