Experimental support for ScheduledOverrides (#515)
This adds the initial version of ScheduledOverrides to HorizontalRunnerAutoscaler. `MinReplicas` overriding should just work. When there are two or more ScheduledOverrides, the earliest one that matched is activated. Each ScheduledOverride can be recurring or one-time. If you have two or more ScheduledOverrides, only one of them should be one-time. And the one-time override should be the earliest item in the list to make sense. Tests will be added in another commit. Logging improvements and additional observability in HRA.Status will also be added in yet another commits. Ref #484
This commit is contained in:
parent
b3cae25741
commit
0e0f385f72
|
|
@ -54,6 +54,12 @@ type HorizontalRunnerAutoscalerSpec struct {
|
|||
ScaleUpTriggers []ScaleUpTrigger `json:"scaleUpTriggers,omitempty"`
|
||||
|
||||
CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
|
||||
|
||||
// ScheduledOverrides is the list of ScheduledOverride.
|
||||
// It can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule.
|
||||
// The earlier a scheduled override is, the higher it is prioritized.
|
||||
// +optional
|
||||
ScheduledOverrides []ScheduledOverride `json:"scheduledOverrides,omitempty"`
|
||||
}
|
||||
|
||||
type ScaleUpTrigger struct {
|
||||
|
|
@ -142,12 +148,6 @@ type MetricSpec struct {
|
|||
// You can only specify either ScaleDownFactor or ScaleDownAdjustment.
|
||||
// +optional
|
||||
ScaleDownAdjustment int `json:"scaleDownAdjustment,omitempty"`
|
||||
|
||||
// ScheduledOverrides is the list of ScheduledOverride.
|
||||
// It can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule.
|
||||
// The earlier a scheduled override is, the higher it is prioritized.
|
||||
// +optional
|
||||
ScheduledOverrides []ScheduledOverride `json:"scheduledOverrides,omitempty"`
|
||||
}
|
||||
|
||||
// ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule.
|
||||
|
|
|
|||
|
|
@ -212,6 +212,13 @@ func (in *HorizontalRunnerAutoscalerSpec) DeepCopyInto(out *HorizontalRunnerAuto
|
|||
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||
}
|
||||
}
|
||||
if in.ScheduledOverrides != nil {
|
||||
in, out := &in.ScheduledOverrides, &out.ScheduledOverrides
|
||||
*out = make([]ScheduledOverride, len(*in))
|
||||
for i := range *in {
|
||||
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerSpec.
|
||||
|
|
@ -263,13 +270,6 @@ func (in *MetricSpec) DeepCopyInto(out *MetricSpec) {
|
|||
*out = make([]string, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
if in.ScheduledOverrides != nil {
|
||||
in, out := &in.ScheduledOverrides, &out.ScheduledOverrides
|
||||
*out = make([]ScheduledOverride, len(*in))
|
||||
for i := range *in {
|
||||
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricSpec.
|
||||
|
|
|
|||
|
|
@ -105,57 +105,6 @@ spec:
|
|||
description: ScaleUpThreshold is the percentage of busy runners
|
||||
greater than which will trigger the hpa to scale runners up.
|
||||
type: string
|
||||
scheduledOverrides:
|
||||
description: ScheduledOverrides is the list of ScheduledOverride.
|
||||
It can be used to override a few fields of HorizontalRunnerAutoscalerSpec
|
||||
on schedule. The earlier a scheduled override is, the higher
|
||||
it is prioritized.
|
||||
items:
|
||||
description: ScheduledOverride can be used to override a few
|
||||
fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule
|
||||
can optionally be recurring, so that the correspoding override
|
||||
happens every day, week, month, or year.
|
||||
properties:
|
||||
endTime:
|
||||
description: EndTime is the time at which the first override
|
||||
ends.
|
||||
format: date-time
|
||||
type: string
|
||||
minReplicas:
|
||||
description: MinReplicas is the number of runners while
|
||||
overriding. If omitted, it doesn't override minReplicas.
|
||||
minimum: 0
|
||||
nullable: true
|
||||
type: integer
|
||||
recurrenceRule:
|
||||
properties:
|
||||
frequency:
|
||||
description: Frequency is the name of a predefined interval
|
||||
of each recurrence. The valid values are "Daily",
|
||||
"Weekly", "Monthly", and "Yearly". If empty, the corresponding
|
||||
override happens only once.
|
||||
enum:
|
||||
- Daily
|
||||
- Weekly
|
||||
- Monthly
|
||||
- Yearly
|
||||
type: string
|
||||
untilTime:
|
||||
description: UntilTime is the time of the final recurrence.
|
||||
If empty, the schedule recurs forever.
|
||||
format: date-time
|
||||
type: string
|
||||
type: object
|
||||
startTime:
|
||||
description: StartTime is the time at which the first override
|
||||
starts.
|
||||
format: date-time
|
||||
type: string
|
||||
required:
|
||||
- endTime
|
||||
- startTime
|
||||
type: object
|
||||
type: array
|
||||
type:
|
||||
description: Type is the type of metric to be used for autoscaling.
|
||||
The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
|
||||
|
|
@ -236,6 +185,56 @@ spec:
|
|||
type: object
|
||||
type: object
|
||||
type: array
|
||||
scheduledOverrides:
|
||||
description: ScheduledOverrides is the list of ScheduledOverride. It
|
||||
can be used to override a few fields of HorizontalRunnerAutoscalerSpec
|
||||
on schedule. The earlier a scheduled override is, the higher it is
|
||||
prioritized.
|
||||
items:
|
||||
description: ScheduledOverride can be used to override a few fields
|
||||
of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally
|
||||
be recurring, so that the correspoding override happens every day,
|
||||
week, month, or year.
|
||||
properties:
|
||||
endTime:
|
||||
description: EndTime is the time at which the first override ends.
|
||||
format: date-time
|
||||
type: string
|
||||
minReplicas:
|
||||
description: MinReplicas is the number of runners while overriding.
|
||||
If omitted, it doesn't override minReplicas.
|
||||
minimum: 0
|
||||
nullable: true
|
||||
type: integer
|
||||
recurrenceRule:
|
||||
properties:
|
||||
frequency:
|
||||
description: Frequency is the name of a predefined interval
|
||||
of each recurrence. The valid values are "Daily", "Weekly",
|
||||
"Monthly", and "Yearly". If empty, the corresponding override
|
||||
happens only once.
|
||||
enum:
|
||||
- Daily
|
||||
- Weekly
|
||||
- Monthly
|
||||
- Yearly
|
||||
type: string
|
||||
untilTime:
|
||||
description: UntilTime is the time of the final recurrence.
|
||||
If empty, the schedule recurs forever.
|
||||
format: date-time
|
||||
type: string
|
||||
type: object
|
||||
startTime:
|
||||
description: StartTime is the time at which the first override
|
||||
starts.
|
||||
format: date-time
|
||||
type: string
|
||||
required:
|
||||
- endTime
|
||||
- startTime
|
||||
type: object
|
||||
type: array
|
||||
type: object
|
||||
status:
|
||||
properties:
|
||||
|
|
|
|||
|
|
@ -105,57 +105,6 @@ spec:
|
|||
description: ScaleUpThreshold is the percentage of busy runners
|
||||
greater than which will trigger the hpa to scale runners up.
|
||||
type: string
|
||||
scheduledOverrides:
|
||||
description: ScheduledOverrides is the list of ScheduledOverride.
|
||||
It can be used to override a few fields of HorizontalRunnerAutoscalerSpec
|
||||
on schedule. The earlier a scheduled override is, the higher
|
||||
it is prioritized.
|
||||
items:
|
||||
description: ScheduledOverride can be used to override a few
|
||||
fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule
|
||||
can optionally be recurring, so that the correspoding override
|
||||
happens every day, week, month, or year.
|
||||
properties:
|
||||
endTime:
|
||||
description: EndTime is the time at which the first override
|
||||
ends.
|
||||
format: date-time
|
||||
type: string
|
||||
minReplicas:
|
||||
description: MinReplicas is the number of runners while
|
||||
overriding. If omitted, it doesn't override minReplicas.
|
||||
minimum: 0
|
||||
nullable: true
|
||||
type: integer
|
||||
recurrenceRule:
|
||||
properties:
|
||||
frequency:
|
||||
description: Frequency is the name of a predefined interval
|
||||
of each recurrence. The valid values are "Daily",
|
||||
"Weekly", "Monthly", and "Yearly". If empty, the corresponding
|
||||
override happens only once.
|
||||
enum:
|
||||
- Daily
|
||||
- Weekly
|
||||
- Monthly
|
||||
- Yearly
|
||||
type: string
|
||||
untilTime:
|
||||
description: UntilTime is the time of the final recurrence.
|
||||
If empty, the schedule recurs forever.
|
||||
format: date-time
|
||||
type: string
|
||||
type: object
|
||||
startTime:
|
||||
description: StartTime is the time at which the first override
|
||||
starts.
|
||||
format: date-time
|
||||
type: string
|
||||
required:
|
||||
- endTime
|
||||
- startTime
|
||||
type: object
|
||||
type: array
|
||||
type:
|
||||
description: Type is the type of metric to be used for autoscaling.
|
||||
The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
|
||||
|
|
@ -236,6 +185,56 @@ spec:
|
|||
type: object
|
||||
type: object
|
||||
type: array
|
||||
scheduledOverrides:
|
||||
description: ScheduledOverrides is the list of ScheduledOverride. It
|
||||
can be used to override a few fields of HorizontalRunnerAutoscalerSpec
|
||||
on schedule. The earlier a scheduled override is, the higher it is
|
||||
prioritized.
|
||||
items:
|
||||
description: ScheduledOverride can be used to override a few fields
|
||||
of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally
|
||||
be recurring, so that the correspoding override happens every day,
|
||||
week, month, or year.
|
||||
properties:
|
||||
endTime:
|
||||
description: EndTime is the time at which the first override ends.
|
||||
format: date-time
|
||||
type: string
|
||||
minReplicas:
|
||||
description: MinReplicas is the number of runners while overriding.
|
||||
If omitted, it doesn't override minReplicas.
|
||||
minimum: 0
|
||||
nullable: true
|
||||
type: integer
|
||||
recurrenceRule:
|
||||
properties:
|
||||
frequency:
|
||||
description: Frequency is the name of a predefined interval
|
||||
of each recurrence. The valid values are "Daily", "Weekly",
|
||||
"Monthly", and "Yearly". If empty, the corresponding override
|
||||
happens only once.
|
||||
enum:
|
||||
- Daily
|
||||
- Weekly
|
||||
- Monthly
|
||||
- Yearly
|
||||
type: string
|
||||
untilTime:
|
||||
description: UntilTime is the time of the final recurrence.
|
||||
If empty, the schedule recurs forever.
|
||||
format: date-time
|
||||
type: string
|
||||
type: object
|
||||
startTime:
|
||||
description: StartTime is the time at which the first override
|
||||
starts.
|
||||
format: date-time
|
||||
type: string
|
||||
required:
|
||||
- endTime
|
||||
- startTime
|
||||
type: object
|
||||
type: array
|
||||
type: object
|
||||
status:
|
||||
properties:
|
||||
|
|
|
|||
|
|
@ -185,12 +185,52 @@ func (r *HorizontalRunnerAutoscalerReconciler) SetupWithManager(mgr ctrl.Manager
|
|||
Complete(r)
|
||||
}
|
||||
|
||||
func (r *HorizontalRunnerAutoscalerReconciler) matchScheduledOverrides(log logr.Logger, now time.Time, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, *Period, *Period, error) {
|
||||
var minReplicas *int
|
||||
var active, upcoming *Period
|
||||
|
||||
for _, o := range hra.Spec.ScheduledOverrides {
|
||||
a, u, err := MatchSchedule(
|
||||
now, o.StartTime.Time, o.EndTime.Time,
|
||||
RecurrenceRule{
|
||||
Frequency: o.RecurrenceRule.Frequency,
|
||||
UntilTime: o.RecurrenceRule.UntilTime.Time,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return minReplicas, nil, nil, err
|
||||
}
|
||||
|
||||
// Use the first when there are two or more active scheduled overrides,
|
||||
// as the spec defines that the earlier scheduled override is prioritized higher than later ones.
|
||||
if active == nil {
|
||||
active = a
|
||||
|
||||
if o.MinReplicas != nil {
|
||||
minReplicas = o.MinReplicas
|
||||
}
|
||||
}
|
||||
|
||||
if upcoming == nil || (u != nil && u.StartTime.Before(upcoming.StartTime)) {
|
||||
upcoming = u
|
||||
}
|
||||
}
|
||||
|
||||
return minReplicas, active, upcoming, nil
|
||||
}
|
||||
|
||||
func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(log logr.Logger, now time.Time, rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (int, int, *int, error) {
|
||||
minReplicas := defaultReplicas
|
||||
if hra.Spec.MinReplicas != nil && *hra.Spec.MinReplicas >= 0 {
|
||||
minReplicas = *hra.Spec.MinReplicas
|
||||
}
|
||||
|
||||
if m, _, _, err := r.matchScheduledOverrides(log, now, hra); err != nil {
|
||||
return 0, 0, nil, err
|
||||
} else if m != nil {
|
||||
minReplicas = *m
|
||||
}
|
||||
|
||||
var suggestedReplicas int
|
||||
|
||||
suggestedReplicasFromCache := r.fetchSuggestedReplicasFromCache(hra)
|
||||
|
|
|
|||
Loading…
Reference in New Issue