Experimental support for ScheduledOverrides (#515)

This adds the initial version of ScheduledOverrides to HorizontalRunnerAutoscaler.
`MinReplicas` overriding should just work.
When there are two or more ScheduledOverrides, the earliest one that matched is activated. Each ScheduledOverride can be recurring or one-time. If you have two or more ScheduledOverrides, only one of them should be one-time. And the one-time override should be the earliest item in the list to make sense.

Tests will be added in another commit. Logging improvements and additional observability in HRA.Status will also be added in yet another commits.

Ref #484
This commit is contained in:
Yusuke Kuoka 2021-05-03 23:31:17 +09:00 committed by GitHub
parent b3cae25741
commit 0e0f385f72
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 153 additions and 115 deletions

View File

@ -54,6 +54,12 @@ type HorizontalRunnerAutoscalerSpec struct {
ScaleUpTriggers []ScaleUpTrigger `json:"scaleUpTriggers,omitempty"`
CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
// ScheduledOverrides is the list of ScheduledOverride.
// It can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule.
// The earlier a scheduled override is, the higher it is prioritized.
// +optional
ScheduledOverrides []ScheduledOverride `json:"scheduledOverrides,omitempty"`
}
type ScaleUpTrigger struct {
@ -142,12 +148,6 @@ type MetricSpec struct {
// You can only specify either ScaleDownFactor or ScaleDownAdjustment.
// +optional
ScaleDownAdjustment int `json:"scaleDownAdjustment,omitempty"`
// ScheduledOverrides is the list of ScheduledOverride.
// It can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule.
// The earlier a scheduled override is, the higher it is prioritized.
// +optional
ScheduledOverrides []ScheduledOverride `json:"scheduledOverrides,omitempty"`
}
// ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule.

View File

@ -212,6 +212,13 @@ func (in *HorizontalRunnerAutoscalerSpec) DeepCopyInto(out *HorizontalRunnerAuto
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.ScheduledOverrides != nil {
in, out := &in.ScheduledOverrides, &out.ScheduledOverrides
*out = make([]ScheduledOverride, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerSpec.
@ -263,13 +270,6 @@ func (in *MetricSpec) DeepCopyInto(out *MetricSpec) {
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.ScheduledOverrides != nil {
in, out := &in.ScheduledOverrides, &out.ScheduledOverrides
*out = make([]ScheduledOverride, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricSpec.

View File

@ -105,57 +105,6 @@ spec:
description: ScaleUpThreshold is the percentage of busy runners
greater than which will trigger the hpa to scale runners up.
type: string
scheduledOverrides:
description: ScheduledOverrides is the list of ScheduledOverride.
It can be used to override a few fields of HorizontalRunnerAutoscalerSpec
on schedule. The earlier a scheduled override is, the higher
it is prioritized.
items:
description: ScheduledOverride can be used to override a few
fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule
can optionally be recurring, so that the correspoding override
happens every day, week, month, or year.
properties:
endTime:
description: EndTime is the time at which the first override
ends.
format: date-time
type: string
minReplicas:
description: MinReplicas is the number of runners while
overriding. If omitted, it doesn't override minReplicas.
minimum: 0
nullable: true
type: integer
recurrenceRule:
properties:
frequency:
description: Frequency is the name of a predefined interval
of each recurrence. The valid values are "Daily",
"Weekly", "Monthly", and "Yearly". If empty, the corresponding
override happens only once.
enum:
- Daily
- Weekly
- Monthly
- Yearly
type: string
untilTime:
description: UntilTime is the time of the final recurrence.
If empty, the schedule recurs forever.
format: date-time
type: string
type: object
startTime:
description: StartTime is the time at which the first override
starts.
format: date-time
type: string
required:
- endTime
- startTime
type: object
type: array
type:
description: Type is the type of metric to be used for autoscaling.
The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
@ -236,6 +185,56 @@ spec:
type: object
type: object
type: array
scheduledOverrides:
description: ScheduledOverrides is the list of ScheduledOverride. It
can be used to override a few fields of HorizontalRunnerAutoscalerSpec
on schedule. The earlier a scheduled override is, the higher it is
prioritized.
items:
description: ScheduledOverride can be used to override a few fields
of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally
be recurring, so that the correspoding override happens every day,
week, month, or year.
properties:
endTime:
description: EndTime is the time at which the first override ends.
format: date-time
type: string
minReplicas:
description: MinReplicas is the number of runners while overriding.
If omitted, it doesn't override minReplicas.
minimum: 0
nullable: true
type: integer
recurrenceRule:
properties:
frequency:
description: Frequency is the name of a predefined interval
of each recurrence. The valid values are "Daily", "Weekly",
"Monthly", and "Yearly". If empty, the corresponding override
happens only once.
enum:
- Daily
- Weekly
- Monthly
- Yearly
type: string
untilTime:
description: UntilTime is the time of the final recurrence.
If empty, the schedule recurs forever.
format: date-time
type: string
type: object
startTime:
description: StartTime is the time at which the first override
starts.
format: date-time
type: string
required:
- endTime
- startTime
type: object
type: array
type: object
status:
properties:

View File

@ -105,57 +105,6 @@ spec:
description: ScaleUpThreshold is the percentage of busy runners
greater than which will trigger the hpa to scale runners up.
type: string
scheduledOverrides:
description: ScheduledOverrides is the list of ScheduledOverride.
It can be used to override a few fields of HorizontalRunnerAutoscalerSpec
on schedule. The earlier a scheduled override is, the higher
it is prioritized.
items:
description: ScheduledOverride can be used to override a few
fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule
can optionally be recurring, so that the correspoding override
happens every day, week, month, or year.
properties:
endTime:
description: EndTime is the time at which the first override
ends.
format: date-time
type: string
minReplicas:
description: MinReplicas is the number of runners while
overriding. If omitted, it doesn't override minReplicas.
minimum: 0
nullable: true
type: integer
recurrenceRule:
properties:
frequency:
description: Frequency is the name of a predefined interval
of each recurrence. The valid values are "Daily",
"Weekly", "Monthly", and "Yearly". If empty, the corresponding
override happens only once.
enum:
- Daily
- Weekly
- Monthly
- Yearly
type: string
untilTime:
description: UntilTime is the time of the final recurrence.
If empty, the schedule recurs forever.
format: date-time
type: string
type: object
startTime:
description: StartTime is the time at which the first override
starts.
format: date-time
type: string
required:
- endTime
- startTime
type: object
type: array
type:
description: Type is the type of metric to be used for autoscaling.
The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
@ -236,6 +185,56 @@ spec:
type: object
type: object
type: array
scheduledOverrides:
description: ScheduledOverrides is the list of ScheduledOverride. It
can be used to override a few fields of HorizontalRunnerAutoscalerSpec
on schedule. The earlier a scheduled override is, the higher it is
prioritized.
items:
description: ScheduledOverride can be used to override a few fields
of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally
be recurring, so that the correspoding override happens every day,
week, month, or year.
properties:
endTime:
description: EndTime is the time at which the first override ends.
format: date-time
type: string
minReplicas:
description: MinReplicas is the number of runners while overriding.
If omitted, it doesn't override minReplicas.
minimum: 0
nullable: true
type: integer
recurrenceRule:
properties:
frequency:
description: Frequency is the name of a predefined interval
of each recurrence. The valid values are "Daily", "Weekly",
"Monthly", and "Yearly". If empty, the corresponding override
happens only once.
enum:
- Daily
- Weekly
- Monthly
- Yearly
type: string
untilTime:
description: UntilTime is the time of the final recurrence.
If empty, the schedule recurs forever.
format: date-time
type: string
type: object
startTime:
description: StartTime is the time at which the first override
starts.
format: date-time
type: string
required:
- endTime
- startTime
type: object
type: array
type: object
status:
properties:

View File

@ -185,12 +185,52 @@ func (r *HorizontalRunnerAutoscalerReconciler) SetupWithManager(mgr ctrl.Manager
Complete(r)
}
func (r *HorizontalRunnerAutoscalerReconciler) matchScheduledOverrides(log logr.Logger, now time.Time, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, *Period, *Period, error) {
var minReplicas *int
var active, upcoming *Period
for _, o := range hra.Spec.ScheduledOverrides {
a, u, err := MatchSchedule(
now, o.StartTime.Time, o.EndTime.Time,
RecurrenceRule{
Frequency: o.RecurrenceRule.Frequency,
UntilTime: o.RecurrenceRule.UntilTime.Time,
},
)
if err != nil {
return minReplicas, nil, nil, err
}
// Use the first when there are two or more active scheduled overrides,
// as the spec defines that the earlier scheduled override is prioritized higher than later ones.
if active == nil {
active = a
if o.MinReplicas != nil {
minReplicas = o.MinReplicas
}
}
if upcoming == nil || (u != nil && u.StartTime.Before(upcoming.StartTime)) {
upcoming = u
}
}
return minReplicas, active, upcoming, nil
}
func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(log logr.Logger, now time.Time, rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (int, int, *int, error) {
minReplicas := defaultReplicas
if hra.Spec.MinReplicas != nil && *hra.Spec.MinReplicas >= 0 {
minReplicas = *hra.Spec.MinReplicas
}
if m, _, _, err := r.matchScheduledOverrides(log, now, hra); err != nil {
return 0, 0, nil, err
} else if m != nil {
minReplicas = *m
}
var suggestedReplicas int
suggestedReplicasFromCache := r.fetchSuggestedReplicasFromCache(hra)