public_git/actions-runner-controller
1
0
Fork 0
mirror of https://github.com/actions-runner-controller/actions-runner-controller.git synced 2026-10-02 16:00:12 +02:00
Code Issues Projects Releases Wiki Activity

Make the Outdated phase revision-aware

Browse Source 
An outdated runner used to flip the whole scale set into the Outdated
phase permanently, which tears down the listener and switches the scale
set off. That verdict outlived the runner spec it was about: a runner
busy with a job survives the revision cleanup that follows a spec
update, and only reports Outdated once the job finishes. The result was
that a freshly applied fix could be discarded by a runner that never ran
it.

Runners are now stamped with the actionable revision they were built
from. An Outdated runner whose revision is behind the applied revision is
considered stale: it is deleted so the scaling logic replaces it with one
built from the current spec, and it no longer contributes to the set's
phase. A runner at the current revision still marks the set Outdated, so
a genuinely bad spec is still surfaced.

Two supporting fixes:

  - patchAppliedActionableRevisionStatus now recomputes the phase in both
    directions. It only ever forced Running, so the early-return path
    could leave a stale Outdated behind.

  - The AutoscalingRunnerSet only tears down on an Outdated set once that
    set has applied its current actionable revision. Otherwise a spec
    update races the EphemeralRunnerSet controller and the teardown fires
    against a phase that predates the update.

Runners created before this change parse to revision 0, which matches
the zero value of AppliedActionableRevision, so they are treated as
current until a revision is actually applied.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Nikola Jokic
2026-09-08 17:27:39 +02:00
co-authored by Copilot App
parent 2aef0aeedd
commit 41d05325c2
7 changed files with 439 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
controllers/actions.github.com/ephemeralrunnerset_controller.go
+81 -12
View File
@@ -187,11 +187,12 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
return ctrl.Result{}, err
}
ephemeralRunnersByState := newEphemeralRunnersByStates(&ephemeralRunnerList)
ephemeralRunnersByState := newEphemeralRunnersByStates(&ephemeralRunnerList, ephemeralRunnerSet.Status.AppliedActionableRevision)
log.Info(
"Ephemeral runner counts",
"outdated", len(ephemeralRunnersByState.outdated),
"staleOutdated", len(ephemeralRunnersByState.staleOutdated),
"pending", len(ephemeralRunnersByState.pending),
"running", len(ephemeralRunnersByState.running),
"finished", len(ephemeralRunnersByState.finished),
@@ -199,10 +200,27 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
"deleting", len(ephemeralRunnersByState.deleting),
)
// Runners that reported Outdated against a runner spec that has since been
// replaced are not evidence about the current spec. Drop them so the scaling
// logic below replaces them with runners built from the current spec, instead
// of letting them hold the set in the Outdated phase forever.
if len(ephemeralRunnersByState.staleOutdated) > 0 {
log.Info(
"Deleting outdated ephemeral runners created before the last spec update so they can be replaced",
"count", len(ephemeralRunnersByState.staleOutdated),
"appliedActionableRevision", ephemeralRunnerSet.Status.AppliedActionableRevision,
)
if err := r.deleteTerminatedEphemeralRunners(ctx, ephemeralRunnersByState.staleOutdated, log); err != nil {
log.Error(err, "failed to delete stale outdated ephemeral runners")
return ctrl.Result{}, err
}
return ctrl.Result{}, r.updateStatus(ctx, &ephemeralRunnerSet, ephemeralRunnersByState, log)
}
total := ephemeralRunnersByState.scaleTotal()
if ephemeralRunnerSet.Spec.PatchID == 0 || ephemeralRunnerSet.Spec.PatchID != ephemeralRunnersByState.latestPatchID {
if len(ephemeralRunnersByState.finished) > 0 {
if err := r.cleanupFinishedEphemeralRunners(ctx, ephemeralRunnersByState.finished, log); err != nil {
if err := r.deleteTerminatedEphemeralRunners(ctx, ephemeralRunnersByState.finished, log); err != nil {
log.Error(err, "failed to cleanup finished ephemeral runners")
return ctrl.Result{}, err
}
@@ -272,7 +290,21 @@ func (r *EphemeralRunnerSetReconciler) patchAppliedActionableRevisionStatus(ctx
return fmt.Errorf("failed to list child ephemeral runners: %w", err)
}
if len(newEphemeralRunnersByStates(ephemeralRunnerList).outdated) == 0 {
// Judge the runners against the revision being applied, not the one
// recorded in status: every runner created before this update is stale by
// definition, so its Outdated report says nothing about the new spec. This
// is what lets a spec update clear the Outdated phase immediately rather
// than waiting for the pre-update runners to be collected.
state := newEphemeralRunnersByStates(ephemeralRunnerList, targetAppliedRevision)
// Set the phase in both directions. This function returns early from
// Reconcile without reaching updateStatus, so leaving the phase untouched
// would let a stale value survive: a stale Running would hide genuinely
// outdated runners from the cleanup path, and a stale Outdated would keep
// the set switched off after the spec that caused it was replaced.
if len(state.outdated) > 0 {
latest.Status.Phase = v1alpha1.EphemeralRunnerSetPhaseOutdated
} else {
latest.Status.Phase = v1alpha1.EphemeralRunnerSetPhaseRunning
}
@@ -332,12 +364,13 @@ func (r *EphemeralRunnerSetReconciler) updateStatus(ctx context.Context, ephemer
return nil
}
func (r *EphemeralRunnerSetReconciler) cleanupFinishedEphemeralRunners(ctx context.Context, finishedEphemeralRunners []*v1alpha1.EphemeralRunner, log logr.Logger) error {
// cleanup finished runners and proceed
// deleteTerminatedEphemeralRunners deletes runners that have reached a terminal
// state and are no longer useful, so that the scaling logic can replace them.
func (r *EphemeralRunnerSetReconciler) deleteTerminatedEphemeralRunners(ctx context.Context, ephemeralRunners []*v1alpha1.EphemeralRunner, log logr.Logger) error {
var errs []error
for i := range finishedEphemeralRunners {
log.Info("Deleting finished ephemeral runner", "name", finishedEphemeralRunners[i].Name)
if err := r.Delete(ctx, finishedEphemeralRunners[i]); err != nil {
for i := range ephemeralRunners {
log.Info("Deleting terminated ephemeral runner", "name", ephemeralRunners[i].Name, "phase", ephemeralRunners[i].Status.Phase)
if err := r.Delete(ctx, ephemeralRunners[i]); err != nil {
if !kerrors.IsNotFound(err) {
errs = append(errs, err)
}
@@ -383,7 +416,7 @@ func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunners(ctx context.Conte
return true, nil
}
ephemeralRunnerState := newEphemeralRunnersByStates(ephemeralRunnerList)
ephemeralRunnerState := newEphemeralRunnersByStates(ephemeralRunnerList, ephemeralRunnerSet.Status.AppliedActionableRevision)
log.Info(
"Clean up runner counts",
@@ -754,12 +787,27 @@ type ephemeralRunnersByState struct {
finished []*v1alpha1.EphemeralRunner
failed []*v1alpha1.EphemeralRunner
deleting []*v1alpha1.EphemeralRunner
// outdated holds runners that reported Outdated against the runner spec that
// is currently applied. They are evidence that the current spec is still
// rejected by the service, so they drive the set into the Outdated phase.
outdated []*v1alpha1.EphemeralRunner
// staleOutdated holds runners that reported Outdated against a runner spec
// that has since been replaced. They say nothing about the current spec, so
// they must not drive the set into the Outdated phase; they are deleted and
// replaced by runners built from the current spec instead.
staleOutdated []*v1alpha1.EphemeralRunner
latestPatchID int
}
func newEphemeralRunnersByStates(ephemeralRunnerList *v1alpha1.EphemeralRunnerList) *ephemeralRunnersByState {
// newEphemeralRunnersByStates groups the child runners by state.
//
// appliedActionableRevision is the EphemeralRunnerSet revision the runners are
// being judged against. A runner that reported Outdated before that revision was
// applied is classified as stale rather than outdated, so that updating the
// runner spec clears the Outdated phase immediately instead of waiting for the
// pre-update runners to disappear.
func newEphemeralRunnersByStates(ephemeralRunnerList *v1alpha1.EphemeralRunnerList, appliedActionableRevision int64) *ephemeralRunnersByState {
var ephemeralRunnerState ephemeralRunnersByState
for i := range ephemeralRunnerList.Items {
@@ -781,7 +829,11 @@ func newEphemeralRunnersByStates(ephemeralRunnerList *v1alpha1.EphemeralRunnerLi
case v1alpha1.EphemeralRunnerPhaseFailed:
ephemeralRunnerState.failed = append(ephemeralRunnerState.failed, r)
case v1alpha1.EphemeralRunnerPhaseOutdated:
ephemeralRunnerState.outdated = append(ephemeralRunnerState.outdated, r)
if ephemeralRunnerActionableRevision(r) < appliedActionableRevision {
ephemeralRunnerState.staleOutdated = append(ephemeralRunnerState.staleOutdated, r)
} else {
ephemeralRunnerState.outdated = append(ephemeralRunnerState.outdated, r)
}
default:
// Pending or no phase should be considered as pending.
//
@@ -793,8 +845,25 @@ func newEphemeralRunnersByStates(ephemeralRunnerList *v1alpha1.EphemeralRunnerLi
return &ephemeralRunnerState
}
// ephemeralRunnerActionableRevision reports the EphemeralRunnerSet revision the
// runner was created from. Runners created before this annotation existed report
// 0, which matches the zero value of Status.AppliedActionableRevision, so they
// are treated as current until the spec is updated for the first time.
func ephemeralRunnerActionableRevision(ephemeralRunner *v1alpha1.EphemeralRunner) int64 {
revision, err := strconv.ParseInt(ephemeralRunner.Annotations[AnnotationKeyActionableRevision], 10, 64)
if err != nil {
return 0
}
return revision
}
func (s *ephemeralRunnersByState) terminated() []*v1alpha1.EphemeralRunner {
return append(s.finished, append(s.failed, s.outdated...)...)
terminated := make([]*v1alpha1.EphemeralRunner, 0, len(s.finished)+len(s.failed)+len(s.outdated)+len(s.staleOutdated))
terminated = append(terminated, s.finished...)
terminated = append(terminated, s.failed...)
terminated = append(terminated, s.outdated...)
terminated = append(terminated, s.staleOutdated...)
return terminated
}
func (s *ephemeralRunnersByState) scaleTotal() int {