This commit is contained in:
Nikola Jokic 2026-05-26 11:28:47 +02:00
parent d1af7d66ea
commit 00fbd94622
No known key found for this signature in database
GPG Key ID: 419BB425B0E501B0
1 changed files with 70 additions and 19 deletions

View File

@ -147,9 +147,25 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
log.Info("Successfully added finalizer")
}
// Something has changed, work-out what
// Something has changed, we need to re-apply the pending phase and change hash annotation to trigger the update of runner scale set and listener.
if targetHash := autoscalingRunnerSet.Hash(); autoscalingRunnerSet.Annotations[annotationKeyChangeHash] != targetHash {
// TODO: work
// TODO: apply the version label
original := autoscalingRunnerSet.DeepCopy()
if autoscalingRunnerSet.Annotations == nil {
autoscalingRunnerSet.Annotations = map[string]string{}
}
autoscalingRunnerSet.Annotations[annotationKeyChangeHash] = targetHash
autoscalingRunnerSet.Status.Phase = v1alpha1.AutoscalingRunnerSetPhasePending
if err := r.Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
log.Error(err, "Failed to update autoscaling runner set with new change hash and pending phase")
return ctrl.Result{}, err
}
if err := r.Status().Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
log.Error(err, "Failed to update autoscaling runner set status with pending phase")
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
outdated := autoscalingRunnerSet.Status.Phase == v1alpha1.AutoscalingRunnerSetPhaseOutdated
@ -206,9 +222,39 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
case err != nil:
log.Error(err, "Failed to get ephemeral runner")
return ctrl.Result{}, err
case ephemeralRunnerSet.Status.Phase == v1alpha1.EphemeralRunnerSetPhaseOutdated:
case ephemeralRunnerSet.Status.Phase == v1alpha1.EphemeralRunnerSetPhaseOutdated && autoscalingRunnerSet.Status.Phase == v1alpha1.AutoscalingRunnerSetPhaseRunning:
// Runners are outdated. We need to stop the listener so it stops getting new jobs.
log.Info("Ephemeral runner set is outdated. Cleaning up resources for the outdated runner set")
// TODO: patch autoscaling runner set as outdated
done, err := r.cleanupListener(ctx, autoscalingRunnerSet, log)
if err != nil {
log.Error(err, "Failed to clean up listener for outdated ephemeral runner set")
return ctrl.Result{}, err
}
if !done {
log.Info("Waiting for listener to be cleaned up for the outdated ephemeral runner set")
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
// Then, we need to remove the ephemeral runner set to force scale-down. The ephemeral runner set
// will eventually remove all runners as soon as possible.
//
// The scale set should not be removed yet, since user did not explicitly remove the scale set (or the autoscaling runner set)
// Therefore, the autoscaling runner set should stay in outdated state until the spec is updated,
// or until the autoscaling runner set is removed.
done, err = r.cleanupEphemeralRunnerSet(ctx, autoscalingRunnerSet, log)
if err != nil {
log.Error(err, "Failed to clean up ephemeral runner set for outdated runner set")
return ctrl.Result{}, err
}
if !done {
log.Info("Waiting for ephemeral runner set to be cleaned up for the outdated runner set")
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
log.Info("Successfully cleaned up resources for the outdated runner set")
return ctrl.Result{}, nil
default:
desired, err := r.newEphemeralRunnerSet(autoscalingRunnerSet)
if err != nil {
@ -331,22 +377,27 @@ func (r *AutoscalingRunnerSetReconciler) cleanUpResources(ctx context.Context, a
func (r *AutoscalingRunnerSetReconciler) updateStatus(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, phase v1alpha1.AutoscalingRunnerSetPhase, log logr.Logger) error {
countDiff := ephemeralRunnerSet != nil && ephemeralRunnerSet.Status.CurrentReplicas != autoscalingRunnerSet.Status.CurrentRunners
phaseDiff := phase != autoscalingRunnerSet.Status.Phase
if countDiff || phaseDiff {
if err := patchSubResource(ctx, r.Status(), autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) {
obj.Status.Phase = phase
var ephemeralRunnerSetStatus v1alpha1.EphemeralRunnerSetStatus
if ephemeralRunnerSet != nil {
ephemeralRunnerSetStatus = ephemeralRunnerSet.Status
}
obj.Status.CurrentRunners = ephemeralRunnerSetStatus.CurrentReplicas
obj.Status.PendingEphemeralRunners = ephemeralRunnerSetStatus.PendingEphemeralRunners
obj.Status.RunningEphemeralRunners = ephemeralRunnerSetStatus.RunningEphemeralRunners
obj.Status.FailedEphemeralRunners = ephemeralRunnerSetStatus.FailedEphemeralRunners
}); err != nil {
log.Error(err, "Failed to update autoscaling runner set status with current runner count")
return err
}
if !countDiff && !phaseDiff {
return nil
}
original := autoscalingRunnerSet.DeepCopy()
if phaseDiff {
autoscalingRunnerSet.Status.Phase = phase
}
if phaseDiff {
autoscalingRunnerSet.Status.CurrentRunners = ephemeralRunnerSet.Status.CurrentReplicas
autoscalingRunnerSet.Status.PendingEphemeralRunners = ephemeralRunnerSet.Status.PendingEphemeralRunners
autoscalingRunnerSet.Status.RunningEphemeralRunners = ephemeralRunnerSet.Status.RunningEphemeralRunners
autoscalingRunnerSet.Status.FailedEphemeralRunners = ephemeralRunnerSet.Status.FailedEphemeralRunners
}
if err := r.Status().Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
log.Error(err, "Failed to patch autoscaling runner set status")
return err
}
return nil
}