wip
This commit is contained in:
parent
d1af7d66ea
commit
00fbd94622
|
|
@ -147,9 +147,25 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
|
|||
log.Info("Successfully added finalizer")
|
||||
}
|
||||
|
||||
// Something has changed, work-out what
|
||||
// Something has changed, we need to re-apply the pending phase and change hash annotation to trigger the update of runner scale set and listener.
|
||||
if targetHash := autoscalingRunnerSet.Hash(); autoscalingRunnerSet.Annotations[annotationKeyChangeHash] != targetHash {
|
||||
// TODO: work
|
||||
// TODO: apply the version label
|
||||
original := autoscalingRunnerSet.DeepCopy()
|
||||
if autoscalingRunnerSet.Annotations == nil {
|
||||
autoscalingRunnerSet.Annotations = map[string]string{}
|
||||
}
|
||||
autoscalingRunnerSet.Annotations[annotationKeyChangeHash] = targetHash
|
||||
autoscalingRunnerSet.Status.Phase = v1alpha1.AutoscalingRunnerSetPhasePending
|
||||
if err := r.Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
|
||||
log.Error(err, "Failed to update autoscaling runner set with new change hash and pending phase")
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if err := r.Status().Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
|
||||
log.Error(err, "Failed to update autoscaling runner set status with pending phase")
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
outdated := autoscalingRunnerSet.Status.Phase == v1alpha1.AutoscalingRunnerSetPhaseOutdated
|
||||
|
|
@ -206,9 +222,39 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
|
|||
case err != nil:
|
||||
log.Error(err, "Failed to get ephemeral runner")
|
||||
return ctrl.Result{}, err
|
||||
case ephemeralRunnerSet.Status.Phase == v1alpha1.EphemeralRunnerSetPhaseOutdated:
|
||||
case ephemeralRunnerSet.Status.Phase == v1alpha1.EphemeralRunnerSetPhaseOutdated && autoscalingRunnerSet.Status.Phase == v1alpha1.AutoscalingRunnerSetPhaseRunning:
|
||||
// Runners are outdated. We need to stop the listener so it stops getting new jobs.
|
||||
log.Info("Ephemeral runner set is outdated. Cleaning up resources for the outdated runner set")
|
||||
// TODO: patch autoscaling runner set as outdated
|
||||
done, err := r.cleanupListener(ctx, autoscalingRunnerSet, log)
|
||||
if err != nil {
|
||||
log.Error(err, "Failed to clean up listener for outdated ephemeral runner set")
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if !done {
|
||||
log.Info("Waiting for listener to be cleaned up for the outdated ephemeral runner set")
|
||||
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
|
||||
}
|
||||
|
||||
// Then, we need to remove the ephemeral runner set to force scale-down. The ephemeral runner set
|
||||
// will eventually remove all runners as soon as possible.
|
||||
//
|
||||
// The scale set should not be removed yet, since user did not explicitly remove the scale set (or the autoscaling runner set)
|
||||
// Therefore, the autoscaling runner set should stay in outdated state until the spec is updated,
|
||||
// or until the autoscaling runner set is removed.
|
||||
done, err = r.cleanupEphemeralRunnerSet(ctx, autoscalingRunnerSet, log)
|
||||
if err != nil {
|
||||
log.Error(err, "Failed to clean up ephemeral runner set for outdated runner set")
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if !done {
|
||||
log.Info("Waiting for ephemeral runner set to be cleaned up for the outdated runner set")
|
||||
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
|
||||
}
|
||||
|
||||
log.Info("Successfully cleaned up resources for the outdated runner set")
|
||||
|
||||
return ctrl.Result{}, nil
|
||||
|
||||
default:
|
||||
desired, err := r.newEphemeralRunnerSet(autoscalingRunnerSet)
|
||||
if err != nil {
|
||||
|
|
@ -331,22 +377,27 @@ func (r *AutoscalingRunnerSetReconciler) cleanUpResources(ctx context.Context, a
|
|||
func (r *AutoscalingRunnerSetReconciler) updateStatus(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, phase v1alpha1.AutoscalingRunnerSetPhase, log logr.Logger) error {
|
||||
countDiff := ephemeralRunnerSet != nil && ephemeralRunnerSet.Status.CurrentReplicas != autoscalingRunnerSet.Status.CurrentRunners
|
||||
phaseDiff := phase != autoscalingRunnerSet.Status.Phase
|
||||
if countDiff || phaseDiff {
|
||||
if err := patchSubResource(ctx, r.Status(), autoscalingRunnerSet, func(obj *v1alpha1.AutoscalingRunnerSet) {
|
||||
obj.Status.Phase = phase
|
||||
var ephemeralRunnerSetStatus v1alpha1.EphemeralRunnerSetStatus
|
||||
if ephemeralRunnerSet != nil {
|
||||
ephemeralRunnerSetStatus = ephemeralRunnerSet.Status
|
||||
}
|
||||
obj.Status.CurrentRunners = ephemeralRunnerSetStatus.CurrentReplicas
|
||||
obj.Status.PendingEphemeralRunners = ephemeralRunnerSetStatus.PendingEphemeralRunners
|
||||
obj.Status.RunningEphemeralRunners = ephemeralRunnerSetStatus.RunningEphemeralRunners
|
||||
obj.Status.FailedEphemeralRunners = ephemeralRunnerSetStatus.FailedEphemeralRunners
|
||||
}); err != nil {
|
||||
log.Error(err, "Failed to update autoscaling runner set status with current runner count")
|
||||
return err
|
||||
}
|
||||
if !countDiff && !phaseDiff {
|
||||
return nil
|
||||
}
|
||||
|
||||
original := autoscalingRunnerSet.DeepCopy()
|
||||
if phaseDiff {
|
||||
autoscalingRunnerSet.Status.Phase = phase
|
||||
}
|
||||
|
||||
if phaseDiff {
|
||||
autoscalingRunnerSet.Status.CurrentRunners = ephemeralRunnerSet.Status.CurrentReplicas
|
||||
autoscalingRunnerSet.Status.PendingEphemeralRunners = ephemeralRunnerSet.Status.PendingEphemeralRunners
|
||||
autoscalingRunnerSet.Status.RunningEphemeralRunners = ephemeralRunnerSet.Status.RunningEphemeralRunners
|
||||
autoscalingRunnerSet.Status.FailedEphemeralRunners = ephemeralRunnerSet.Status.FailedEphemeralRunners
|
||||
}
|
||||
|
||||
if err := r.Status().Patch(ctx, autoscalingRunnerSet, client.MergeFrom(original)); err != nil {
|
||||
log.Error(err, "Failed to patch autoscaling runner set status")
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue