Set listener qps and burst (#4558)

This commit is contained in:
Nikola Jokic
2026-09-08 15:49:58 +02:00
committed by GitHub
parent bc677d306c
commit 7c68e1d318
27 changed files with 1161 additions and 226 deletions
@@ -82,6 +82,9 @@ type AutoscalingListenerSpec struct {
// +optional // +optional
RoleBindingMetadata *ResourceMeta `json:"roleBindingMetadata,omitempty"` RoleBindingMetadata *ResourceMeta `json:"roleBindingMetadata,omitempty"`
// +optional
ListenerConfig *ListenerConfig `json:"listenerConfig,omitempty"`
} }
func (s *AutoscalingListenerSpec) Hash() string { func (s *AutoscalingListenerSpec) Hash() string {
@@ -121,6 +121,9 @@ type AutoscalingRunnerSetSpec struct {
// +optional // +optional
// +kubebuilder:validation:Minimum:=0 // +kubebuilder:validation:Minimum:=0
MinRunners *int `json:"minRunners,omitempty"` MinRunners *int `json:"minRunners,omitempty"`
// +optional
ListenerConfig *ListenerConfig `json:"listenerConfig,omitempty"`
} }
type TLSConfig struct { type TLSConfig struct {
@@ -0,0 +1,26 @@
package v1alpha1
// ListenerConfig holds configuration for the ghalistener pod.
type ListenerConfig struct {
// +optional
Scaler *ScalerConfig `json:"scaler,omitempty"`
}
// GetScaler returns the ScalerConfig, or nil if not set.
func (c *ListenerConfig) GetScaler() *ScalerConfig {
if c == nil {
return nil
}
return c.Scaler
}
// ScalerConfig configures the Kubernetes client used by the ghalistener scaler.
type ScalerConfig struct {
// +optional
// +kubebuilder:validation:Minimum:=1
QPS *int `json:"qps,omitempty"`
// +optional
// +kubebuilder:validation:Minimum:=1
Burst *int `json:"burst,omitempty"`
}
@@ -138,6 +138,11 @@ func (in *AutoscalingListenerSpec) DeepCopyInto(out *AutoscalingListenerSpec) {
*out = new(ResourceMeta) *out = new(ResourceMeta)
(*in).DeepCopyInto(*out) (*in).DeepCopyInto(*out)
} }
if in.ListenerConfig != nil {
in, out := &in.ListenerConfig, &out.ListenerConfig
*out = new(ListenerConfig)
(*in).DeepCopyInto(*out)
}
} }
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalingListenerSpec. // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalingListenerSpec.
@@ -308,6 +313,11 @@ func (in *AutoscalingRunnerSetSpec) DeepCopyInto(out *AutoscalingRunnerSetSpec)
*out = new(int) *out = new(int)
**out = **in **out = **in
} }
if in.ListenerConfig != nil {
in, out := &in.ListenerConfig, &out.ListenerConfig
*out = new(ListenerConfig)
(*in).DeepCopyInto(*out)
}
} }
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalingRunnerSetSpec. // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalingRunnerSetSpec.
@@ -627,6 +637,26 @@ func (in *HistogramMetric) DeepCopy() *HistogramMetric {
return out return out
} }
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ListenerConfig) DeepCopyInto(out *ListenerConfig) {
*out = *in
if in.Scaler != nil {
in, out := &in.Scaler, &out.Scaler
*out = new(ScalerConfig)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ListenerConfig.
func (in *ListenerConfig) DeepCopy() *ListenerConfig {
if in == nil {
return nil
}
out := new(ListenerConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *MetricsConfig) DeepCopyInto(out *MetricsConfig) { func (in *MetricsConfig) DeepCopyInto(out *MetricsConfig) {
*out = *in *out = *in
@@ -764,6 +794,31 @@ func (in *ResourceMeta) DeepCopy() *ResourceMeta {
return out return out
} }
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ScalerConfig) DeepCopyInto(out *ScalerConfig) {
*out = *in
if in.QPS != nil {
in, out := &in.QPS, &out.QPS
*out = new(int)
**out = **in
}
if in.Burst != nil {
in, out := &in.Burst, &out.Burst
*out = new(int)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScalerConfig.
func (in *ScalerConfig) DeepCopy() *ScalerConfig {
if in == nil {
return nil
}
out := new(ScalerConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TLSCertificateSource) DeepCopyInto(out *TLSCertificateSource) { func (in *TLSCertificateSource) DeepCopyInto(out *TLSCertificateSource) {
*out = *in *out = *in
@@ -123,6 +123,22 @@ spec:
type: object type: object
x-kubernetes-map-type: atomic x-kubernetes-map-type: atomic
type: array type: array
listenerConfig:
description: ListenerConfig holds configuration for the ghalistener
pod.
properties:
scaler:
description: ScalerConfig configures the Kubernetes client used
by the ghalistener scaler.
properties:
burst:
minimum: 1
type: integer
qps:
minimum: 1
type: integer
type: object
type: object
maxRunners: maxRunners:
minimum: 0 minimum: 0
type: integer type: integer
@@ -145,6 +145,20 @@ spec:
x-kubernetes-map-type: atomic x-kubernetes-map-type: atomic
type: object type: object
type: object type: object
listenerConfig:
description: ListenerConfig holds configuration for the ghalistener pod.
properties:
scaler:
description: ScalerConfig configures the Kubernetes client used by the ghalistener scaler.
properties:
burst:
minimum: 1
type: integer
qps:
minimum: 1
type: integer
type: object
type: object
listenerConfigSecretMetadata: listenerConfigSecretMetadata:
description: ResourceMeta carries metadata common to all internal resources description: ResourceMeta carries metadata common to all internal resources
properties: properties:
@@ -123,6 +123,22 @@ spec:
type: object type: object
x-kubernetes-map-type: atomic x-kubernetes-map-type: atomic
type: array type: array
listenerConfig:
description: ListenerConfig holds configuration for the ghalistener
pod.
properties:
scaler:
description: ScalerConfig configures the Kubernetes client used
by the ghalistener scaler.
properties:
burst:
minimum: 1
type: integer
qps:
minimum: 1
type: integer
type: object
type: object
maxRunners: maxRunners:
minimum: 0 minimum: 0
type: integer type: integer
@@ -145,6 +145,20 @@ spec:
x-kubernetes-map-type: atomic x-kubernetes-map-type: atomic
type: object type: object
type: object type: object
listenerConfig:
description: ListenerConfig holds configuration for the ghalistener pod.
properties:
scaler:
description: ScalerConfig configures the Kubernetes client used by the ghalistener scaler.
properties:
burst:
minimum: 1
type: integer
qps:
minimum: 1
type: integer
type: object
type: object
listenerConfigSecretMetadata: listenerConfigSecretMetadata:
description: ResourceMeta carries metadata common to all internal resources description: ResourceMeta carries metadata common to all internal resources
properties: properties:
@@ -1,8 +1,8 @@
{{- define "listener-template.pod" -}} {{- define "listener-template.pod" -}}
{{- $metadata := .Values.listenerPodTemplate.metadata | default dict -}} {{- $metadata := .metadata | default dict -}}
{{- $spec := .Values.listenerPodTemplate.spec | default dict -}} {{- $spec := .spec | default dict -}}
{{- if and (empty $metadata) (empty $spec) -}} {{- if and (empty $metadata) (empty $spec) -}}
{{- fail "listenerPodTemplate must have at least metadata or spec defined" -}} {{- fail ".Values.listener.podTemplate must have at least metadata or spec defined" -}}
{{- end -}} {{- end -}}
{{- with $metadata -}} {{- with $metadata -}}
metadata: metadata:
@@ -5,6 +5,55 @@
{{- $kubeDefaults := (index $kubeMode "default" | default true) }} {{- $kubeDefaults := (index $kubeMode "default" | default true) }}
{{- $kubeServiceAccountName := (index $kubeMode "serviceAccountName" | default "") }} {{- $kubeServiceAccountName := (index $kubeMode "serviceAccountName" | default "") }}
{{- $usesKubernetesSecrets := or (not .Values.secretResolution) (eq .Values.secretResolution.type "kubernetes") }} {{- $usesKubernetesSecrets := or (not .Values.secretResolution) (eq .Values.secretResolution.type "kubernetes") }}
{{- /* All listener configuration lives under .Values.listener */ -}}
{{- if hasKey .Values "listenerMetrics" }}
{{- fail ".Values.listenerMetrics has moved to .Values.listener.metrics" }}
{{- end }}
{{- if hasKey .Values "listenerPodTemplate" }}
{{- fail ".Values.listenerPodTemplate has moved to .Values.listener.podTemplate" }}
{{- end }}
{{- $listener := .Values.listener | default dict }}
{{- if not (kindIs "map" $listener) }}
{{- fail ".Values.listener must be an object" }}
{{- end }}
{{- $listenerKeys := list "metrics" "podTemplate" "scaler" }}
{{- range $name, $_ := $listener }}
{{- if not (has $name $listenerKeys) }}
{{- fail (printf ".Values.listener.%s is not a supported key; supported keys are: %s" $name (join ", " $listenerKeys)) }}
{{- end }}
{{- end }}
{{- range $name := list "metrics" "podTemplate" }}
{{- $value := index $listener $name | default dict }}
{{- if not (kindIs "map" $value) }}
{{- fail (printf ".Values.listener.%s must be an object" $name) }}
{{- end }}
{{- end }}
{{- $scaler := $listener.scaler | default dict }}
{{- if not (kindIs "map" $scaler) }}
{{- fail ".Values.listener.scaler must be an object" }}
{{- end }}
{{- $scalerKeys := list "burst" "qps" }}
{{- range $name, $_ := $scaler }}
{{- if not (has $name $scalerKeys) }}
{{- fail (printf ".Values.listener.scaler.%s is not a supported key; supported keys are: %s" $name (join ", " $scalerKeys)) }}
{{- end }}
{{- end }}
{{- range $name := $scalerKeys }}
{{- if hasKey $scaler $name }}
{{- $value := index $scaler $name }}
{{- if not (or (kindIs "int" $value) (kindIs "int64" $value) (kindIs "float64" $value)) }}
{{- fail (printf ".Values.listener.scaler.%s must be an integer greater than 0" $name) }}
{{- else }}
{{- $number := float64 $value }}
{{- if ne $number (floor $number) }}
{{- fail (printf ".Values.listener.scaler.%s must be an integer greater than 0" $name) }}
{{- else if lt $number (float64 1) }}
{{- fail (printf ".Values.listener.scaler.%s must be greater than 0" $name) }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- $listenerPodTemplate := $listener.podTemplate | default dict }}
{{- $runnerPod := (index $runner "pod" | default dict) -}} {{- $runnerPod := (index $runner "pod" | default dict) -}}
{{- if not (kindIs "map" $runnerPod) -}} {{- if not (kindIs "map" $runnerPod) -}}
@@ -165,12 +214,17 @@ spec:
minRunners: {{ .Values.scaleset.minRunners | int }} minRunners: {{ .Values.scaleset.minRunners | int }}
{{- end }} {{- end }}
{{- if and .Values.listenerPodTemplate (or .Values.listenerPodTemplate.metadata .Values.listenerPodTemplate.spec) }} {{- if or $listenerPodTemplate.metadata $listenerPodTemplate.spec }}
listenerTemplate: listenerTemplate:
{{- include "listener-template.pod" . | nindent 4}} {{- include "listener-template.pod" $listenerPodTemplate | nindent 4}}
{{- end }} {{- end }}
{{- with .Values.listenerMetrics }} {{- with $listener.scaler }}
listenerConfig:
scaler:
{{- toYaml . | nindent 6 }}
{{- end }}
{{- with $listener.metrics }}
listenerMetrics: listenerMetrics:
{{- toYaml . | nindent 4 }} {{- toYaml . | nindent 4 }}
{{- end }} {{- end }}
@@ -1,7 +1,111 @@
suite: "Test AutoscalingRunnerSet Listener Metrics" suite: "Test AutoscalingRunnerSet Listener"
templates: templates:
- autoscalingrunnserset.yaml - autoscalingrunnserset.yaml
tests: tests:
- it: should render default scaler values when listener.scaler is not overridden
set:
scaleset.name: "test"
auth.url: "https://github.com/org"
auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system"
release:
name: "test-name"
namespace: "test-namespace"
asserts:
- equal:
path: spec.listenerConfig.scaler.qps
value: 50
- equal:
path: spec.listenerConfig.scaler.burst
value: 100
- it: should render custom scaler values when overridden
set:
scaleset.name: "test"
auth.url: "https://github.com/org"
auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system"
listener:
scaler:
qps: 100
burst: 200
release:
name: "test-name"
namespace: "test-namespace"
asserts:
- equal:
path: spec.listenerConfig.scaler.qps
value: 100
- equal:
path: spec.listenerConfig.scaler.burst
value: 200
- it: should render qps override while keeping default burst
set:
scaleset.name: "test"
auth.url: "https://github.com/org"
auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system"
listener:
scaler:
qps: 75
release:
name: "test-name"
namespace: "test-namespace"
asserts:
- equal:
path: spec.listenerConfig.scaler.qps
value: 75
- equal:
path: spec.listenerConfig.scaler.burst
value: 100
- it: should render scaler and metrics together
set:
scaleset.name: "test"
auth.url: "https://github.com/org"
auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system"
listener:
scaler:
qps: 75
metrics:
counters:
gha_started_jobs_total:
labels:
- repository
release:
name: "test-name"
namespace: "test-namespace"
asserts:
- equal:
path: spec.listenerConfig.scaler.qps
value: 75
- exists:
path: spec.listenerMetrics
- equal:
path: spec.listenerMetrics.counters.gha_started_jobs_total.labels[0]
value: repository
- it: should not render listenerConfig when listener is null
set:
scaleset.name: "test"
auth.url: "https://github.com/org"
auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system"
listener: null
release:
name: "test-name"
namespace: "test-namespace"
asserts:
- notExists:
path: spec.listenerConfig
- it: should not render listenerMetrics when not configured - it: should not render listenerMetrics when not configured
set: set:
scaleset.name: "test" scaleset.name: "test"
@@ -23,18 +127,19 @@ tests:
auth.githubToken: "gh_token12345" auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc" controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system" controllerServiceAccount.namespace: "arc-system"
listenerMetrics: listener:
counters: metrics:
gha_started_jobs_total: counters:
labels: gha_started_jobs_total:
- repository labels:
- organization - repository
histograms: - organization
gha_job_startup_duration_seconds: histograms:
buckets: gha_job_startup_duration_seconds:
- 0.1 buckets:
- 1 - 0.1
- 2.5 - 1
- 2.5
release: release:
name: "test-name" name: "test-name"
namespace: "test-namespace" namespace: "test-namespace"
@@ -1,21 +1,22 @@
suite: "AutoscalingRunnerSet listenerPodTemplate" suite: "AutoscalingRunnerSet listener.podTemplate"
templates: templates:
- autoscalingrunnserset.yaml - autoscalingrunnserset.yaml
tests: tests:
- it: should render listenerTemplate from listenerPodTemplate - it: should render listenerTemplate from listener.podTemplate
set: set:
scaleset.name: "test" scaleset.name: "test"
auth.url: "https://github.com/org" auth.url: "https://github.com/org"
auth.githubToken: "gh_token12345" auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc" controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system" controllerServiceAccount.namespace: "arc-system"
listenerPodTemplate: listener:
spec: podTemplate:
containers: spec:
- name: listener containers:
image: "ghcr.io/actions/actions-runner-controller/actionsmetricsserver:latest" - name: listener
securityContext: image: "ghcr.io/actions/actions-runner-controller/actionsmetricsserver:latest"
runAsUser: 1000 securityContext:
runAsUser: 1000
release: release:
name: "test-name" name: "test-name"
namespace: "test-namespace" namespace: "test-namespace"
@@ -34,9 +35,10 @@ tests:
auth.githubToken: "gh_token12345" auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc" controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system" controllerServiceAccount.namespace: "arc-system"
listenerPodTemplate: listener:
spec: podTemplate:
restartPolicy: Always spec:
restartPolicy: Always
release: release:
name: "test-name" name: "test-name"
namespace: "test-namespace" namespace: "test-namespace"
@@ -44,3 +46,49 @@ tests:
- equal: - equal:
path: spec.listenerTemplate.spec.containers[0].name path: spec.listenerTemplate.spec.containers[0].name
value: listener value: listener
- it: should render podTemplate alongside scaler and metrics
set:
scaleset.name: "test"
auth.url: "https://github.com/org"
auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system"
listener:
scaler:
qps: 75
podTemplate:
spec:
restartPolicy: Always
metrics:
counters:
gha_started_jobs_total:
labels:
- repository
release:
name: "test-name"
namespace: "test-namespace"
asserts:
- equal:
path: spec.listenerConfig.scaler.qps
value: 75
- equal:
path: spec.listenerTemplate.spec.containers[0].name
value: listener
- equal:
path: spec.listenerMetrics.counters.gha_started_jobs_total.labels[0]
value: repository
- it: should not render listenerTemplate when podTemplate is empty
set:
scaleset.name: "test"
auth.url: "https://github.com/org"
auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system"
release:
name: "test-name"
namespace: "test-namespace"
asserts:
- notExists:
path: spec.listenerTemplate
@@ -0,0 +1,122 @@
suite: "Test AutoscalingRunnerSet Listener Validation"
templates:
- autoscalingrunnserset.yaml
set: &base
scaleset.name: "test"
auth.url: "https://github.com/org"
auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system"
tests:
- it: rejects zero qps
set:
<<: *base
listener.scaler.qps: 0
asserts:
- failedTemplate:
errorMessage: ".Values.listener.scaler.qps must be greater than 0"
- it: rejects negative burst
set:
<<: *base
listener.scaler.burst: -1
asserts:
- failedTemplate:
errorMessage: ".Values.listener.scaler.burst must be greater than 0"
- it: rejects fractional qps
set:
<<: *base
listener.scaler.qps: 1.5
asserts:
- failedTemplate:
errorMessage: ".Values.listener.scaler.qps must be an integer greater than 0"
- it: rejects a string burst
set:
<<: *base
listener.scaler.burst: "100"
asserts:
- failedTemplate:
errorMessage: ".Values.listener.scaler.burst must be an integer greater than 0"
- it: rejects a boolean qps
set:
<<: *base
listener.scaler.qps: true
asserts:
- failedTemplate:
errorMessage: ".Values.listener.scaler.qps must be an integer greater than 0"
- it: rejects a scaler that is not an object
set:
<<: *base
listener.scaler: "nope"
asserts:
- failedTemplate:
errorMessage: ".Values.listener.scaler must be an object"
- it: rejects a listener that is not an object
set:
<<: *base
listener: "nope"
asserts:
- failedTemplate:
errorMessage: ".Values.listener must be an object"
- it: rejects a metrics value that is not an object
set:
<<: *base
listener.metrics: "nope"
asserts:
- failedTemplate:
errorMessage: ".Values.listener.metrics must be an object"
- it: rejects a podTemplate that is not an object
set:
<<: *base
listener.podTemplate: "nope"
asserts:
- failedTemplate:
errorMessage: ".Values.listener.podTemplate must be an object"
- it: rejects a misspelled key under listener
set:
<<: *base
listener.metrcs.counters: {}
asserts:
- failedTemplate:
errorMessage: ".Values.listener.metrcs is not a supported key; supported keys are: metrics, podTemplate, scaler"
- it: rejects a misspelled key under listener.scaler
set:
<<: *base
listener.scaler.qbs: 10
asserts:
- failedTemplate:
errorMessage: ".Values.listener.scaler.qbs is not a supported key; supported keys are: burst, qps"
- it: rejects the legacy top-level listenerMetrics key
set:
<<: *base
listenerMetrics.counters: {}
asserts:
- failedTemplate:
errorMessage: ".Values.listenerMetrics has moved to .Values.listener.metrics"
- it: rejects the legacy top-level listenerPodTemplate key
set:
<<: *base
listenerPodTemplate.spec.restartPolicy: Always
asserts:
- failedTemplate:
errorMessage: ".Values.listenerPodTemplate has moved to .Values.listener.podTemplate"
- it: treats a podTemplate with neither metadata nor spec as a no-op
set:
<<: *base
listener.podTemplate.spec: {}
listener.podTemplate.metadata: {}
asserts:
- notExists:
path: spec.listenerTemplate
@@ -75,23 +75,6 @@ secretResolution:
# - example.com # - example.com
# - example.org # - example.org
## listenerTemplate is the PodSpec for each listener Pod
## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
# listenerPodTemplate:
# spec:
# containers:
# # Use this section to append additional configuration to the listener container.
# # If you change the name of the container, the configuration will not be applied to the listener,
# # and it will be treated as a side-car container.
# - name: listener
# securityContext:
# runAsUser: 1000
# # Use this section to add the configuration of a side-car container.
# # Comment it out or remove it if you don't need it.
# # Spec for this container will be applied as is without any modifications.
# - name: side-car
# image: example-sidecar
## Resource object allows modifying resources created by the chart itself ## Resource object allows modifying resources created by the chart itself
resource: resource:
# Specifies metadata that will be applied to all resources managed by ARC # Specifies metadata that will be applied to all resources managed by ARC
@@ -312,158 +295,183 @@ controllerServiceAccount:
namespace: "" namespace: ""
name: "" name: ""
## listenerMetrics are configurable metrics applied to the listener. # listener specific configuration. This configuration is applied to the listener component of the chart.
## In order to avoid helm merging these fields, we left the metrics commented out. listener:
## When configuring metrics, please uncomment the listenerMetrics object below. # scaler is config applied to the kubernetes client component of the listener.
## You can modify the configuration to remove the label or specify custom buckets for histogram. # Both values must be integers greater than 0.
## scaler:
## If the buckets field is not specified, the default buckets will be applied. Default buckets are # qps is the sustained rate of requests the listener may issue to the Kubernetes API server.
## provided here for documentation purposes qps: 50
# listenerMetrics: # burst is the number of requests the listener may issue to the Kubernetes API server in a burst.
# counters: burst: 100
# gha_started_jobs_total: ## podTemplate is the PodSpec for each listener Pod
# labels: ## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
# ["repository", "organization", "enterprise", "job_name", "event_name", "job_workflow_ref", "job_workflow_name", "job_workflow_target"] # podTemplate:
# gha_completed_jobs_total: # spec:
# labels: # containers:
# [ # # Use this section to append additional configuration to the listener container.
# "repository", # # If you change the name of the container, the configuration will not be applied to the listener,
# "organization", # # and it will be treated as a side-car container.
# "enterprise", # - name: listener
# "job_name", # securityContext:
# "event_name", # runAsUser: 1000
# "job_result", # # Use this section to add the configuration of a side-car container.
# "job_workflow_ref", # # Comment it out or remove it if you don't need it.
# "job_workflow_name", # # Spec for this container will be applied as is without any modifications.
# "job_workflow_target", # - name: side-car
# ] # image: example-sidecar
# gauges: ## metrics configuration for the listener.
# gha_assigned_jobs: ## In order to avoid helm merging these fields, we left the metrics commented out.
# labels: ["name", "namespace", "repository", "organization", "enterprise"] ## When configuring metrics, please uncomment the metrics object below.
# gha_running_jobs: ## You can modify the configuration to remove the label or specify custom buckets for histogram.
# labels: ["name", "namespace", "repository", "organization", "enterprise"] ##
# gha_registered_runners: ## If the buckets field is not specified, the default buckets will be applied. Default buckets are
# labels: ["name", "namespace", "repository", "organization", "enterprise"] ## provided here for documentation purposes
# gha_busy_runners: # metrics:
# labels: ["name", "namespace", "repository", "organization", "enterprise"] # counters:
# gha_min_runners: # gha_started_jobs_total:
# labels: ["name", "namespace", "repository", "organization", "enterprise"] # labels:
# gha_max_runners: # ["repository", "organization", "enterprise", "job_name", "event_name", "job_workflow_ref", "job_workflow_name", "job_workflow_target"]
# labels: ["name", "namespace", "repository", "organization", "enterprise"] # gha_completed_jobs_total:
# gha_desired_runners: # labels:
# labels: ["name", "namespace", "repository", "organization", "enterprise"] # [
# gha_idle_runners: # "repository",
# labels: ["name", "namespace", "repository", "organization", "enterprise"] # "organization",
# histograms: # "enterprise",
# gha_job_startup_duration_seconds: # "job_name",
# labels: # "event_name",
# ["repository", "organization", "enterprise", "job_name", "event_name","job_workflow_ref", "job_workflow_name", "job_workflow_target"] # "job_result",
# buckets: # "job_workflow_ref",
# [ # "job_workflow_name",
# 0.01, # "job_workflow_target",
# 0.05, # ]
# 0.1, # gauges:
# 0.5, # gha_assigned_jobs:
# 1.0, # labels: ["name", "namespace", "repository", "organization", "enterprise"]
# 2.0, # gha_running_jobs:
# 3.0, # labels: ["name", "namespace", "repository", "organization", "enterprise"]
# 4.0, # gha_registered_runners:
# 5.0, # labels: ["name", "namespace", "repository", "organization", "enterprise"]
# 6.0, # gha_busy_runners:
# 7.0, # labels: ["name", "namespace", "repository", "organization", "enterprise"]
# 8.0, # gha_min_runners:
# 9.0, # labels: ["name", "namespace", "repository", "organization", "enterprise"]
# 10.0, # gha_max_runners:
# 12.0, # labels: ["name", "namespace", "repository", "organization", "enterprise"]
# 15.0, # gha_desired_runners:
# 18.0, # labels: ["name", "namespace", "repository", "organization", "enterprise"]
# 20.0, # gha_idle_runners:
# 25.0, # labels: ["name", "namespace", "repository", "organization", "enterprise"]
# 30.0, # histograms:
# 40.0, # gha_job_startup_duration_seconds:
# 50.0, # labels:
# 60.0, # ["repository", "organization", "enterprise", "job_name", "event_name","job_workflow_ref", "job_workflow_name", "job_workflow_target"]
# 70.0, # buckets:
# 80.0, # [
# 90.0, # 0.01,
# 100.0, # 0.05,
# 110.0, # 0.1,
# 120.0, # 0.5,
# 150.0, # 1.0,
# 180.0, # 2.0,
# 210.0, # 3.0,
# 240.0, # 4.0,
# 300.0, # 5.0,
# 360.0, # 6.0,
# 420.0, # 7.0,
# 480.0, # 8.0,
# 540.0, # 9.0,
# 600.0, # 10.0,
# 900.0, # 12.0,
# 1200.0, # 15.0,
# 1800.0, # 18.0,
# 2400.0, # 20.0,
# 3000.0, # 25.0,
# 3600.0, # 30.0,
# ] # 40.0,
# gha_job_execution_duration_seconds: # 50.0,
# labels: # 60.0,
# [ # 70.0,
# "repository", # 80.0,
# "organization", # 90.0,
# "enterprise", # 100.0,
# "job_name", # 110.0,
# "event_name", # 120.0,
# "job_result", # 150.0,
# "job_workflow_ref", # 180.0,
# "job_workflow_name", # 210.0,
# "job_workflow_target" # 240.0,
# ] # 300.0,
# buckets: # 360.0,
# [ # 420.0,
# 0.01, # 480.0,
# 0.05, # 540.0,
# 0.1, # 600.0,
# 0.5, # 900.0,
# 1.0, # 1200.0,
# 2.0, # 1800.0,
# 3.0, # 2400.0,
# 4.0, # 3000.0,
# 5.0, # 3600.0,
# 6.0, # ]
# 7.0, # gha_job_execution_duration_seconds:
# 8.0, # labels:
# 9.0, # [
# 10.0, # "repository",
# 12.0, # "organization",
# 15.0, # "enterprise",
# 18.0, # "job_name",
# 20.0, # "event_name",
# 25.0, # "job_result",
# 30.0, # "job_workflow_ref",
# 40.0, # "job_workflow_name",
# 50.0, # "job_workflow_target"
# 60.0, # ]
# 70.0, # buckets:
# 80.0, # [
# 90.0, # 0.01,
# 100.0, # 0.05,
# 110.0, # 0.1,
# 120.0, # 0.5,
# 150.0, # 1.0,
# 180.0, # 2.0,
# 210.0, # 3.0,
# 240.0, # 4.0,
# 300.0, # 5.0,
# 360.0, # 6.0,
# 420.0, # 7.0,
# 480.0, # 8.0,
# 540.0, # 9.0,
# 600.0, # 10.0,
# 900.0, # 12.0,
# 1200.0, # 15.0,
# 1800.0, # 18.0,
# 2400.0, # 20.0,
# 3000.0, # 25.0,
# 3600.0, # 30.0,
# ] # 40.0,
# 50.0,
# 60.0,
# 70.0,
# 80.0,
# 90.0,
# 100.0,
# 110.0,
# 120.0,
# 150.0,
# 180.0,
# 210.0,
# 240.0,
# 300.0,
# 360.0,
# 420.0,
# 480.0,
# 540.0,
# 600.0,
# 900.0,
# 1200.0,
# 1800.0,
# 2400.0,
# 3000.0,
# 3600.0,
# ]
@@ -1,5 +1,6 @@
{{- $resourceMeta := default (dict) .Values.resourceMeta }} {{- $resourceMeta := default (dict) .Values.resourceMeta }}
{{- $hasCustomResourceMeta := (and .Values.resourceMeta .Values.resourceMeta.autoscalingRunnerSet) }} {{- $hasCustomResourceMeta := (and .Values.resourceMeta .Values.resourceMeta.autoscalingRunnerSet) }}
{{- /* .Values.listenerConfig is validated by values.schema.json */ -}}
apiVersion: actions.github.com/v1alpha1 apiVersion: actions.github.com/v1alpha1
kind: AutoscalingRunnerSet kind: AutoscalingRunnerSet
metadata: metadata:
@@ -167,6 +168,11 @@ spec:
{{- toYaml . | nindent 4 }} {{- toYaml . | nindent 4 }}
{{- end }} {{- end }}
{{- with .Values.listenerConfig }}
listenerConfig:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with (index $resourceMeta "autoscalingListener") }} {{- with (index $resourceMeta "autoscalingListener") }}
autoscalingListener: autoscalingListener:
{{- include "gha-runner-scale-set.resourceMetaSpec" . | nindent 4 }} {{- include "gha-runner-scale-set.resourceMetaSpec" . | nindent 4 }}
@@ -18,6 +18,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1" rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/utils/ptr"
) )
func TestTemplateRenderedGitHubSecretWithGitHubToken(t *testing.T) { func TestTemplateRenderedGitHubSecretWithGitHubToken(t *testing.T) {
@@ -141,6 +142,170 @@ func TestTemplateRenderedGitHubSecretErrorWithMissingAppInput(t *testing.T) {
assert.ErrorContains(t, err, "provide .Values.githubConfigSecret.github_app_installation_id and .Values.githubConfigSecret.github_app_private_key") assert.ErrorContains(t, err, "provide .Values.githubConfigSecret.github_app_installation_id and .Values.githubConfigSecret.github_app_private_key")
} }
func TestTemplateListenerScalerValidation(t *testing.T) {
t.Parallel()
helmChartPath, err := filepath.Abs("../../gha-runner-scale-set")
require.NoError(t, err)
baseValues := map[string]string{
"githubConfigUrl": "https://github.com/actions",
"githubConfigSecret.github_token": "gh_token12345",
"controllerServiceAccount.name": "arc",
"controllerServiceAccount.namespace": "arc-system",
}
tests := []struct {
name string
setValues map[string]string
setStrValues map[string]string
wantErrorText string
}{
{
name: "zero qps",
setValues: map[string]string{"listenerConfig.scaler.qps": "0"},
wantErrorText: "at '/listenerConfig/scaler/qps': minimum: got 0, want 1",
},
{
name: "negative burst",
setValues: map[string]string{"listenerConfig.scaler.burst": "-1"},
wantErrorText: "at '/listenerConfig/scaler/burst': minimum: got -1, want 1",
},
{
name: "fractional qps",
setValues: map[string]string{"listenerConfig.scaler.qps": "1.5"},
wantErrorText: "at '/listenerConfig/scaler/qps'",
},
{
name: "string burst",
setStrValues: map[string]string{"listenerConfig.scaler.burst": "100"},
wantErrorText: "at '/listenerConfig/scaler/burst': got string, want integer",
},
{
name: "boolean qps",
setValues: map[string]string{"listenerConfig.scaler.qps": "true"},
wantErrorText: "at '/listenerConfig/scaler/qps': got boolean, want integer",
},
{
name: "scaler is not an object",
setValues: map[string]string{"listenerConfig.scaler[0]": "a"},
wantErrorText: "at '/listenerConfig/scaler': got array, want null or object",
},
{
name: "listenerConfig is not an object",
setStrValues: map[string]string{"listenerConfig": "foo"},
wantErrorText: "at '/listenerConfig': got string, want null or object",
},
{
name: "misspelled scaler key",
setValues: map[string]string{"listenerConfig.scalar.qps": "5"},
wantErrorText: "at '/listenerConfig': additional properties 'scalar' not allowed",
},
{
name: "unknown key under scaler",
setStrValues: map[string]string{"listenerConfig.scaler.foo": "bar"},
wantErrorText: "at '/listenerConfig/scaler': additional properties 'foo' not allowed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
setValues := make(map[string]string, len(baseValues)+len(tt.setValues))
for key, value := range baseValues {
setValues[key] = value
}
for key, value := range tt.setValues {
setValues[key] = value
}
options := &helm.Options{
Logger: logger.Discard,
SetValues: setValues,
SetStrValues: tt.setStrValues,
KubectlOptions: k8s.NewKubectlOptions("", "", "test"),
}
_, err := helm.RenderTemplateContextE(t, t.Context(), options, helmChartPath, "test-runners", []string{"templates/autoscalingrunnerset.yaml"})
require.Error(t, err)
assert.ErrorContains(t, err, tt.wantErrorText)
})
}
}
func TestTemplateListenerScalerConfig(t *testing.T) {
t.Parallel()
helmChartPath, err := filepath.Abs("../../gha-runner-scale-set")
require.NoError(t, err)
baseValues := map[string]string{
"githubConfigUrl": "https://github.com/actions",
"githubConfigSecret.github_token": "gh_token12345",
"controllerServiceAccount.name": "arc",
"controllerServiceAccount.namespace": "arc-system",
}
tests := []struct {
name string
setValues map[string]string
wantQPS *int
wantBurst *int
}{
{
name: "defaults from values.yaml",
wantQPS: ptr.To(50),
wantBurst: ptr.To(100),
},
{
name: "both overridden",
setValues: map[string]string{"listenerConfig.scaler.qps": "100", "listenerConfig.scaler.burst": "200"},
wantQPS: ptr.To(100),
wantBurst: ptr.To(200),
},
{
name: "qps overridden keeps default burst",
setValues: map[string]string{"listenerConfig.scaler.qps": "75"},
wantQPS: ptr.To(75),
wantBurst: ptr.To(100),
},
{
name: "listenerConfig disabled",
setValues: map[string]string{"listenerConfig": "null"},
wantQPS: nil,
wantBurst: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
setValues := make(map[string]string, len(baseValues)+len(tt.setValues))
for key, value := range baseValues {
setValues[key] = value
}
for key, value := range tt.setValues {
setValues[key] = value
}
options := &helm.Options{
Logger: logger.Discard,
SetValues: setValues,
KubectlOptions: k8s.NewKubectlOptions("", "", "test"),
}
output := helm.RenderTemplateContext(t, t.Context(), options, helmChartPath, "test-runners", []string{"templates/autoscalingrunnerset.yaml"})
var ars v1alpha1.AutoscalingRunnerSet
helm.UnmarshalK8SYaml(t, output, &ars)
if tt.wantQPS == nil && tt.wantBurst == nil {
assert.Nil(t, ars.Spec.ListenerConfig.GetScaler())
return
}
scaler := ars.Spec.ListenerConfig.GetScaler()
require.NotNil(t, scaler)
assert.Equal(t, tt.wantQPS, scaler.QPS)
assert.Equal(t, tt.wantBurst, scaler.Burst)
})
}
}
func TestTemplateNotRenderedGitHubSecretWithPredefinedSecret(t *testing.T) { func TestTemplateNotRenderedGitHubSecretWithPredefinedSecret(t *testing.T) {
t.Parallel() t.Parallel()
@@ -0,0 +1,32 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "gha-runner-scale-set values",
"description": "Only values that require strict validation are described here. Any value not listed is intentionally unconstrained.",
"type": "object",
"properties": {
"listenerConfig": {
"description": "Configuration for the ghalistener pod.",
"type": ["object", "null"],
"additionalProperties": false,
"properties": {
"scaler": {
"description": "Configuration for the Kubernetes client used by the ghalistener scaler.",
"type": ["object", "null"],
"additionalProperties": false,
"properties": {
"qps": {
"description": "Queries per second the listener may issue to the Kubernetes API server.",
"type": "integer",
"minimum": 1
},
"burst": {
"description": "Burst of queries the listener may issue to the Kubernetes API server.",
"type": "integer",
"minimum": 1
}
}
}
}
}
}
}
+6
View File
@@ -146,6 +146,12 @@ githubConfigSecret:
# - name: side-car # - name: side-car
# image: example-sidecar # image: example-sidecar
## listenerConfig holds configuration for the ghalistener pod.
listenerConfig:
scaler:
qps: 50
burst: 100
## listenerMetrics are configurable metrics applied to the listener. ## listenerMetrics are configurable metrics applied to the listener.
## In order to avoid helm merging these fields, we left the metrics commented out. ## In order to avoid helm merging these fields, we left the metrics commented out.
## When configuring metrics, please uncomment the listenerMetrics object below. ## When configuring metrics, please uncomment the listenerMetrics object below.
+13 -12
View File
@@ -32,18 +32,19 @@ type Config struct {
// It is initially set to nil if VaultType is set. // It is initially set to nil if VaultType is set.
// Otherwise, it is populated with the GitHub App credentials from the GitHub secret. // Otherwise, it is populated with the GitHub App credentials from the GitHub secret.
*appconfig.AppConfig *appconfig.AppConfig
EphemeralRunnerSetNamespace string `json:"ephemeral_runner_set_namespace"` EphemeralRunnerSetNamespace string `json:"ephemeral_runner_set_namespace"`
EphemeralRunnerSetName string `json:"ephemeral_runner_set_name"` EphemeralRunnerSetName string `json:"ephemeral_runner_set_name"`
MaxRunners int `json:"max_runners"` MaxRunners int `json:"max_runners"`
MinRunners int `json:"min_runners"` MinRunners int `json:"min_runners"`
RunnerScaleSetID int `json:"runner_scale_set_id"` RunnerScaleSetID int `json:"runner_scale_set_id"`
RunnerScaleSetName string `json:"runner_scale_set_name"` RunnerScaleSetName string `json:"runner_scale_set_name"`
ServerRootCA string `json:"server_root_ca"` ServerRootCA string `json:"server_root_ca"`
LogLevel string `json:"log_level"` LogLevel string `json:"log_level"`
LogFormat string `json:"log_format"` LogFormat string `json:"log_format"`
MetricsAddr string `json:"metrics_addr"` MetricsAddr string `json:"metrics_addr"`
MetricsEndpoint string `json:"metrics_endpoint"` MetricsEndpoint string `json:"metrics_endpoint"`
Metrics *v1alpha1.MetricsConfig `json:"metrics"` Metrics *v1alpha1.MetricsConfig `json:"metrics"`
ListenerConfig *v1alpha1.ListenerConfig `json:"listener_config"`
} }
func Read(ctx context.Context, configPath string) (*Config, error) { func Read(ctx context.Context, configPath string) (*Config, error) {
+1
View File
@@ -120,6 +120,7 @@ func run(ctx context.Context, config *config.Config) error {
EphemeralRunnerSetName: config.EphemeralRunnerSetName, EphemeralRunnerSetName: config.EphemeralRunnerSetName,
MaxRunners: config.MaxRunners, MaxRunners: config.MaxRunners,
MinRunners: config.MinRunners, MinRunners: config.MinRunners,
ScalerConfig: config.ListenerConfig.GetScaler(),
}, },
scaler.WithLogger(logger.With("component", "worker")), scaler.WithLogger(logger.With("component", "worker")),
) )
+46 -10
View File
@@ -30,8 +30,14 @@ type Config struct {
EphemeralRunnerSetName string EphemeralRunnerSetName string
MaxRunners int MaxRunners int
MinRunners int MinRunners int
ScalerConfig *v1alpha1.ScalerConfig
} }
const (
defaultQPS = 50
defaultBurst = 100
)
// The Scaler's role is to process the messages it receives from the listener. // The Scaler's role is to process the messages it receives from the listener.
// It then initiates Kubernetes API requests to carry out the necessary actions. // It then initiates Kubernetes API requests to carry out the necessary actions.
type Scaler struct { type Scaler struct {
@@ -52,12 +58,22 @@ func New(config Config, options ...Option) (*Scaler, error) {
targetRunners: -1, targetRunners: -1,
patchSeq: -1, patchSeq: -1,
} }
for _, option := range options {
option(w)
}
if err := w.applyDefaults(); err != nil {
return nil, err
}
conf, err := rest.InClusterConfig() conf, err := rest.InClusterConfig()
if err != nil { if err != nil {
return nil, err return nil, err
} }
qps, burst := effectiveRateLimiterConfig(config.ScalerConfig, w.logger)
conf.QPS = float32(qps)
conf.Burst = burst
clientset, err := kubernetes.NewForConfig(conf) clientset, err := kubernetes.NewForConfig(conf)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -65,17 +81,36 @@ func New(config Config, options ...Option) (*Scaler, error) {
w.clientset = clientset w.clientset = clientset
for _, option := range options {
option(w)
}
if err := w.applyDefaults(); err != nil {
return nil, err
}
return w, nil return w, nil
} }
func effectiveRateLimiterConfig(config *v1alpha1.ScalerConfig, logger *slog.Logger) (int, int) {
if config == nil {
logger.Debug("Listener scaler configuration is missing; using defaults", "qps", defaultQPS, "burst", defaultBurst)
return defaultQPS, defaultBurst
}
qps := defaultQPS
if config.QPS == nil {
logger.Debug("Listener scaler qps is missing; using default", "default", defaultQPS)
} else if *config.QPS < 1 {
logger.Warn("Listener scaler qps must be greater than 0; using default", "configured", *config.QPS, "default", defaultQPS)
} else {
qps = *config.QPS
}
burst := defaultBurst
if config.Burst == nil {
logger.Debug("Listener scaler burst is missing; using default", "default", defaultBurst)
} else if *config.Burst < 1 {
logger.Warn("Listener scaler burst must be greater than 0; using default", "configured", *config.Burst, "default", defaultBurst)
} else {
burst = *config.Burst
}
return qps, burst
}
func (w *Scaler) applyDefaults() error { func (w *Scaler) applyDefaults() error {
if w.logger == nil { if w.logger == nil {
w.logger = slog.New(slog.DiscardHandler) w.logger = slog.New(slog.DiscardHandler)
@@ -213,10 +248,11 @@ func (w *Scaler) HandleDesiredRunnerCount(ctx context.Context, count int) (int,
Do(ctx). Do(ctx).
Into(patchedEphemeralRunnerSet) Into(patchedEphemeralRunnerSet)
if err != nil { if err != nil {
return 0, fmt.Errorf("could not patch ephemeral runner set , patch JSON: %s, error: %w", string(mergePatch), err) return 0, fmt.Errorf("could not patch ephemeral runner set, patch JSON: %s, error: %w", string(mergePatch), err)
} }
w.logger.Info("Ephemeral runner set scaled.", w.logger.Info(
"Ephemeral runner set scaled.",
"namespace", w.config.EphemeralRunnerSetNamespace, "namespace", w.config.EphemeralRunnerSetNamespace,
"name", w.config.EphemeralRunnerSetName, "name", w.config.EphemeralRunnerSetName,
"replicas", patchedEphemeralRunnerSet.Spec.Replicas, "replicas", patchedEphemeralRunnerSet.Spec.Replicas,
+111
View File
@@ -1,15 +1,126 @@
package scaler package scaler
import ( import (
"bytes"
"log/slog" "log/slog"
"math" "math"
"strconv"
"testing" "testing"
"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
var discardLogger = slog.New(slog.DiscardHandler) var discardLogger = slog.New(slog.DiscardHandler)
func TestEffectiveRateLimiterConfig(t *testing.T) {
qps := 75
burst := 150
zero := 0
negative := -1
tests := []struct {
name string
config *v1alpha1.ScalerConfig
wantQPS int
wantBurst int
wantLog string
wantLevel string
}{
{
name: "uses configured values",
config: &v1alpha1.ScalerConfig{
QPS: &qps,
Burst: &burst,
},
wantQPS: qps,
wantBurst: burst,
},
{
name: "defaults missing config",
wantQPS: defaultQPS,
wantBurst: defaultBurst,
wantLog: "Listener scaler configuration is missing; using defaults",
wantLevel: "DEBUG",
},
{
name: "defaults missing qps",
config: &v1alpha1.ScalerConfig{Burst: &burst},
wantQPS: defaultQPS,
wantBurst: burst,
wantLog: "Listener scaler qps is missing; using default",
wantLevel: "DEBUG",
},
{
name: "defaults missing burst",
config: &v1alpha1.ScalerConfig{QPS: &qps},
wantQPS: qps,
wantBurst: defaultBurst,
wantLog: "Listener scaler burst is missing; using default",
wantLevel: "DEBUG",
},
{
name: "defaults zero qps",
config: &v1alpha1.ScalerConfig{QPS: &zero, Burst: &burst},
wantQPS: defaultQPS,
wantBurst: burst,
wantLog: "Listener scaler qps must be greater than 0; using default",
wantLevel: "WARN",
},
{
name: "defaults negative burst",
config: &v1alpha1.ScalerConfig{QPS: &qps, Burst: &negative},
wantQPS: qps,
wantBurst: defaultBurst,
wantLog: "Listener scaler burst must be greater than 0; using default",
wantLevel: "WARN",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var logs bytes.Buffer
logger := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug}))
qps, burst := effectiveRateLimiterConfig(tt.config, logger)
assert.Equal(t, tt.wantQPS, qps)
assert.Equal(t, tt.wantBurst, burst)
if tt.wantLog == "" {
assert.Empty(t, logs.String())
return
}
assert.Contains(t, logs.String(), "msg="+strconv.Quote(tt.wantLog))
// Missing values are a normal configuration, so they must not be
// logged as warnings; only out-of-range values are.
assert.Contains(t, logs.String(), "level="+tt.wantLevel)
})
}
}
// TestEffectiveRateLimiterConfig_QuietAtInfoLevel asserts that a listener which
// does not configure the scaler produces no output at the default log level.
func TestEffectiveRateLimiterConfig_QuietAtInfoLevel(t *testing.T) {
for _, tt := range []struct {
name string
config *v1alpha1.ScalerConfig
}{
{name: "nil config"},
{name: "empty config", config: &v1alpha1.ScalerConfig{}},
} {
t.Run(tt.name, func(t *testing.T) {
var logs bytes.Buffer
logger := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelInfo}))
qps, burst := effectiveRateLimiterConfig(tt.config, logger)
assert.Equal(t, defaultQPS, qps)
assert.Equal(t, defaultBurst, burst)
assert.Empty(t, logs.String())
})
}
}
func TestSetDesiredWorkerState_MinMaxDefaults(t *testing.T) { func TestSetDesiredWorkerState_MinMaxDefaults(t *testing.T) {
newEmptyWorker := func() *Scaler { newEmptyWorker := func() *Scaler {
return &Scaler{ return &Scaler{
@@ -123,6 +123,22 @@ spec:
type: object type: object
x-kubernetes-map-type: atomic x-kubernetes-map-type: atomic
type: array type: array
listenerConfig:
description: ListenerConfig holds configuration for the ghalistener
pod.
properties:
scaler:
description: ScalerConfig configures the Kubernetes client used
by the ghalistener scaler.
properties:
burst:
minimum: 1
type: integer
qps:
minimum: 1
type: integer
type: object
type: object
maxRunners: maxRunners:
minimum: 0 minimum: 0
type: integer type: integer
@@ -145,6 +145,20 @@ spec:
x-kubernetes-map-type: atomic x-kubernetes-map-type: atomic
type: object type: object
type: object type: object
listenerConfig:
description: ListenerConfig holds configuration for the ghalistener pod.
properties:
scaler:
description: ScalerConfig configures the Kubernetes client used by the ghalistener scaler.
properties:
burst:
minimum: 1
type: integer
qps:
minimum: 1
type: integer
type: object
type: object
listenerConfigSecretMetadata: listenerConfigSecretMetadata:
description: ResourceMeta carries metadata common to all internal resources description: ResourceMeta carries metadata common to all internal resources
properties: properties:
@@ -394,8 +394,9 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl.
labelsModified := !maps.Equal(listenerConfigSecret.Labels, desiredLabels) labelsModified := !maps.Equal(listenerConfigSecret.Labels, desiredLabels)
desiredAnnotations := r.mergeAnnotations(listenerConfigSecret.Annotations, desiredSecret.Annotations) desiredAnnotations := r.mergeAnnotations(listenerConfigSecret.Annotations, desiredSecret.Annotations)
annotationsModified := !maps.Equal(listenerConfigSecret.Annotations, desiredAnnotations) annotationsModified := !maps.Equal(listenerConfigSecret.Annotations, desiredAnnotations)
dataModified := !reflect.DeepEqual(listenerConfigSecret.Data, desiredSecret.Data)
if labelsModified || annotationsModified { if labelsModified || annotationsModified || dataModified {
updatedSecret := listenerConfigSecret.DeepCopy() updatedSecret := listenerConfigSecret.DeepCopy()
if labelsModified { if labelsModified {
updatedSecret.Labels = desiredLabels updatedSecret.Labels = desiredLabels
@@ -403,6 +404,9 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl.
if annotationsModified { if annotationsModified {
updatedSecret.Annotations = desiredAnnotations updatedSecret.Annotations = desiredAnnotations
} }
if dataModified {
updatedSecret.Data = desiredSecret.Data
}
log.Info("Updating listener config secret", "namespace", updatedSecret.Namespace, "name", updatedSecret.Name) log.Info("Updating listener config secret", "namespace", updatedSecret.Namespace, "name", updatedSecret.Name)
if err := r.Patch(ctx, updatedSecret, client.MergeFrom(&listenerConfigSecret)); err != nil { if err := r.Patch(ctx, updatedSecret, client.MergeFrom(&listenerConfigSecret)); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to update listener config secret: %w", err) return ctrl.Result{}, fmt.Errorf("failed to update listener config secret: %w", err)
@@ -423,6 +423,63 @@ var _ = Describe("Test AutoScalingListener controller", func() {
).Should(BeEquivalentTo(rulesForListenerRole([]string{updated.Spec.EphemeralRunnerSetName})), "Role should be updated") ).Should(BeEquivalentTo(rulesForListenerRole([]string{updated.Spec.EphemeralRunnerSetName})), "Role should be updated")
}) })
It("updates listener scaler configuration and recreates the listener pod", func() {
pod := new(corev1.Pod)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, pod)
},
autoscalingListenerTestTimeout,
autoscalingListenerTestInterval,
).Should(Succeed(), "Listener pod should be created")
oldPodUID := pod.UID
current := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, current)
Expect(err).NotTo(HaveOccurred(), "failed to get AutoScalingListener")
qps := 75
burst := 150
updated := current.DeepCopy()
updated.Spec.ListenerConfig = &v1alpha1.ListenerConfig{
Scaler: &v1alpha1.ScalerConfig{
QPS: &qps,
Burst: &burst,
},
}
err = k8sClient.Patch(ctx, updated, client.MergeFrom(current))
Expect(err).NotTo(HaveOccurred(), "failed to update listener scaler configuration")
secret := new(corev1.Secret)
Eventually(
func(g Gomega) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerConfigName(autoscalingListener), Namespace: autoscalingListener.Namespace}, secret)
g.Expect(err).NotTo(HaveOccurred(), "failed to get listener config Secret")
var config ghalistenerconfig.Config
err = json.Unmarshal(secret.Data["config.json"], &config)
g.Expect(err).NotTo(HaveOccurred(), "failed to parse listener configuration file")
g.Expect(config.ListenerConfig.GetScaler()).NotTo(BeNil())
g.Expect(config.ListenerConfig.GetScaler().QPS).NotTo(BeNil())
g.Expect(config.ListenerConfig.GetScaler().Burst).NotTo(BeNil())
g.Expect(*config.ListenerConfig.GetScaler().QPS).To(Equal(qps))
g.Expect(*config.ListenerConfig.GetScaler().Burst).To(Equal(burst))
},
autoscalingListenerTestTimeout,
autoscalingListenerTestInterval,
).Should(Succeed(), "Listener config Secret should be updated")
Eventually(
func() (types.UID, error) {
pod := new(corev1.Pod)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, pod)
return pod.UID, err
},
autoscalingListenerTestTimeout,
autoscalingListenerTestInterval,
).ShouldNot(Equal(oldPodUID), "Listener pod should be recreated with the updated configuration")
})
It("propagates updated listener metadata to owned resources", func() { It("propagates updated listener metadata to owned resources", func() {
assertPropagatedMetadata := func(expected string) { assertPropagatedMetadata := func(expected string) {
Eventually( Eventually(
@@ -170,6 +170,7 @@ func (b *ResourceBuilder) newAutoscalingListener(autoscalingRunnerSet *v1alpha1.
RoleMetadata: autoscalingRunnerSet.Spec.ListenerRoleMetadata, RoleMetadata: autoscalingRunnerSet.Spec.ListenerRoleMetadata,
RoleBindingMetadata: autoscalingRunnerSet.Spec.ListenerRoleBindingMetadata, RoleBindingMetadata: autoscalingRunnerSet.Spec.ListenerRoleBindingMetadata,
ConfigSecretMetadata: autoscalingRunnerSet.Spec.ListenerConfigSecretMetadata, ConfigSecretMetadata: autoscalingRunnerSet.Spec.ListenerConfigSecretMetadata,
ListenerConfig: autoscalingRunnerSet.Spec.ListenerConfig,
} }
labels := b.filterAndMergeLabels(autoscalingRunnerSet.Labels, map[string]string{ labels := b.filterAndMergeLabels(autoscalingRunnerSet.Labels, map[string]string{
@@ -269,6 +270,7 @@ func (b *ResourceBuilder) newScaleSetListenerConfig(autoscalingListener *v1alpha
MetricsAddr: metricsAddr, MetricsAddr: metricsAddr,
MetricsEndpoint: metricsEndpoint, MetricsEndpoint: metricsEndpoint,
Metrics: autoscalingListener.Spec.Metrics, Metrics: autoscalingListener.Spec.Metrics,
ListenerConfig: autoscalingListener.Spec.ListenerConfig,
} }
vault := autoscalingListener.Spec.VaultConfig vault := autoscalingListener.Spec.VaultConfig