From a401686bd568315fa07768e9d3a10f0cf316d6ff Mon Sep 17 00:00:00 2001 From: Junya Okabe <86868255+Okabe-Junya@users.noreply.github.com> Date: Thu, 23 Apr 2026 06:26:39 +0900 Subject: [PATCH] Add option to disable workqueue bucket rate limiter (#4451) --- .../templates/deployment.yaml | 5 +++ .../values.yaml | 7 ++++ .../autoscalinglistener_controller.go | 20 +++++----- .../autoscalingrunnerset_controller.go | 38 ++++++++++--------- .../ephemeralrunnerset_controller.go | 14 ++++--- controllers/actions.github.com/options.go | 21 ++++++++++ main.go | 28 ++++++++++++-- 7 files changed, 96 insertions(+), 37 deletions(-) diff --git a/charts/gha-runner-scale-set-controller/templates/deployment.yaml b/charts/gha-runner-scale-set-controller/templates/deployment.yaml index 200cbe0f..75f762db 100644 --- a/charts/gha-runner-scale-set-controller/templates/deployment.yaml +++ b/charts/gha-runner-scale-set-controller/templates/deployment.yaml @@ -93,6 +93,11 @@ spec: {{- with .Values.flags.k8sClientRateLimiterBurst }} - "--k8s-client-rate-limiter-burst={{ . }}" {{- end }} + {{- with .Values.flags.rateLimiter }} + {{- with .name }} + - "--workqueue-rate-limiter={{ . }}" + {{- end }} + {{- end }} command: - "/manager" {{- with .Values.metrics }} diff --git a/charts/gha-runner-scale-set-controller/values.yaml b/charts/gha-runner-scale-set-controller/values.yaml index fb54ed99..a83bdb3a 100644 --- a/charts/gha-runner-scale-set-controller/values.yaml +++ b/charts/gha-runner-scale-set-controller/values.yaml @@ -136,6 +136,13 @@ flags: # excludeLabelPropagationPrefixes: # - "argocd.argoproj.io/instance" + ## Workqueue rate limiter configuration. + ## By default, controller-runtime uses a combined rate limiter with both a per-item + ## exponential backoff and an overall token bucket (10 QPS, 100 bucket size). + ## Valid names: "bucket_rate_limiter" (default), "typed_rate_limiter" (per-item only, no global token bucket). + # rateLimiter: + # name: "bucket_rate_limiter" + # Overrides the default `.Release.Namespace` for all resources in this chart. namespaceOverride: "" diff --git a/controllers/actions.github.com/autoscalinglistener_controller.go b/controllers/actions.github.com/autoscalinglistener_controller.go index 4c8ed34a..f83c760e 100644 --- a/controllers/actions.github.com/autoscalinglistener_controller.go +++ b/controllers/actions.github.com/autoscalinglistener_controller.go @@ -692,7 +692,7 @@ func (r *AutoscalingListenerReconciler) publishRunningListener(autoscalingListen } // SetupWithManager sets up the controller with the Manager. -func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error { +func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error { labelBasedWatchFunc := func(_ context.Context, obj client.Object) []reconcile.Request { var requests []reconcile.Request labels := obj.GetLabels() @@ -716,14 +716,16 @@ func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error return requests } - return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha1.AutoscalingListener{}). - Owns(&corev1.Pod{}). - Owns(&corev1.ServiceAccount{}). - Watches(&rbacv1.Role{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). - Watches(&rbacv1.RoleBinding{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). - WithEventFilter(predicate.ResourceVersionChangedPredicate{}). - Complete(r) + return builderWithOptions( + ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.AutoscalingListener{}). + Owns(&corev1.Pod{}). + Owns(&corev1.ServiceAccount{}). + Watches(&rbacv1.Role{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). + Watches(&rbacv1.RoleBinding{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). + WithEventFilter(predicate.ResourceVersionChangedPredicate{}), + opts, + ).Complete(r) } func listenerContainerStatus(pod *corev1.Pod) *corev1.ContainerStatus { diff --git a/controllers/actions.github.com/autoscalingrunnerset_controller.go b/controllers/actions.github.com/autoscalingrunnerset_controller.go index 3509de91..f31bfd39 100644 --- a/controllers/actions.github.com/autoscalingrunnerset_controller.go +++ b/controllers/actions.github.com/autoscalingrunnerset_controller.go @@ -762,25 +762,27 @@ func (r *AutoscalingRunnerSetReconciler) listEphemeralRunnerSets(ctx context.Con } // SetupWithManager sets up the controller with the Manager. -func (r *AutoscalingRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha1.AutoscalingRunnerSet{}). - Owns(&v1alpha1.EphemeralRunnerSet{}). - Watches(&v1alpha1.AutoscalingListener{}, handler.EnqueueRequestsFromMapFunc( - func(_ context.Context, o client.Object) []reconcile.Request { - autoscalingListener := o.(*v1alpha1.AutoscalingListener) - return []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, - Name: autoscalingListener.Spec.AutoscalingRunnerSetName, +func (r *AutoscalingRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error { + return builderWithOptions( + ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.AutoscalingRunnerSet{}). + Owns(&v1alpha1.EphemeralRunnerSet{}). + Watches(&v1alpha1.AutoscalingListener{}, handler.EnqueueRequestsFromMapFunc( + func(_ context.Context, o client.Object) []reconcile.Request { + autoscalingListener := o.(*v1alpha1.AutoscalingListener) + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, + Name: autoscalingListener.Spec.AutoscalingRunnerSetName, + }, }, - }, - } - }, - )). - WithEventFilter(predicate.ResourceVersionChangedPredicate{}). - Complete(r) + } + }, + )). + WithEventFilter(predicate.ResourceVersionChangedPredicate{}), + opts, + ).Complete(r) } type autoscalingRunnerSetFinalizerDependencyCleaner struct { diff --git a/controllers/actions.github.com/ephemeralrunnerset_controller.go b/controllers/actions.github.com/ephemeralrunnerset_controller.go index 7f81216d..acfcba57 100644 --- a/controllers/actions.github.com/ephemeralrunnerset_controller.go +++ b/controllers/actions.github.com/ephemeralrunnerset_controller.go @@ -522,12 +522,14 @@ func (r *EphemeralRunnerSetReconciler) deleteEphemeralRunnerWithActionsClient(ct } // SetupWithManager sets up the controller with the Manager. -func (r *EphemeralRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha1.EphemeralRunnerSet{}). - Owns(&v1alpha1.EphemeralRunner{}). - WithEventFilter(predicate.ResourceVersionChangedPredicate{}). - Complete(r) +func (r *EphemeralRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error { + return builderWithOptions( + ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.EphemeralRunnerSet{}). + Owns(&v1alpha1.EphemeralRunner{}). + WithEventFilter(predicate.ResourceVersionChangedPredicate{}), + opts, + ).Complete(r) } type ephemeralRunnerStepper struct { diff --git a/controllers/actions.github.com/options.go b/controllers/actions.github.com/options.go index 7c7c240e..bc9583de 100644 --- a/controllers/actions.github.com/options.go +++ b/controllers/actions.github.com/options.go @@ -1,8 +1,10 @@ package actionsgithubcom import ( + "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) // Options is the optional configuration for the controllers, which can be @@ -37,6 +39,25 @@ func WithMaxConcurrentReconciles(n int) Option { } } +// WithTypedRateLimiter sets the rate limiter for the controller's workqueue. +// +// By default, the controller-runtime uses +// workqueue.DefaultTypedControllerRateLimiter[reconcile.Request], which combines +// an exponential backoff per-item limiter with a token bucket overall limiter +// (10 QPS, 100 bucket size). In large-scale environments with many runner +// scale sets, the token bucket limiter can become a bottleneck for +// reconciliation throughput. +// +// Use this option to override the default rate limiter, for example, to use +// workqueue.DefaultTypedItemBasedRateLimiter[reconcile.Request], which removes +// the overall token bucket constraint while keeping the per-item exponential +// backoff. +func WithTypedRateLimiter(rateLimiter workqueue.TypedRateLimiter[reconcile.Request]) Option { + return func(b *controller.Options) { + b.RateLimiter = rateLimiter + } +} + // builderWithOptions applies the given options to the provided builder, if any. // This is a helper function to avoid the need to import the controller-runtime package in every reconciler source file // and the command package that creates the controller. diff --git a/main.go b/main.go index 116f2920..e1879dda 100644 --- a/main.go +++ b/main.go @@ -39,10 +39,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/webhook" // +kubebuilder:scaffold:imports ) @@ -110,6 +112,8 @@ func main() { k8sClientRateLimiterQPS int k8sClientRateLimiterBurst int + + workqueueRateLimiter string ) var c github.Config err = envconfig.Process("github", &c) @@ -155,6 +159,7 @@ func main() { flag.Var(&autoScalerImagePullSecrets, "auto-scaler-image-pull-secrets", "The default image-pull secret name for auto-scaler listener container.") flag.IntVar(&k8sClientRateLimiterQPS, "k8s-client-rate-limiter-qps", 20, "The QPS value of the K8s client rate limiter.") flag.IntVar(&k8sClientRateLimiterBurst, "k8s-client-rate-limiter-burst", 30, "The burst value of the K8s client rate limiter.") + flag.StringVar(&workqueueRateLimiter, "workqueue-rate-limiter", "", `The workqueue rate limiter to use. Valid values are "bucket_rate_limiter" (default) and "typed_rate_limiter" (per-item only, no global token bucket).`) flag.Parse() runnerPodDefaults.RunnerImagePullSecrets = runnerImagePullSecrets @@ -293,6 +298,20 @@ func main() { log.Info("Resource builder initializing") + var controllerOpts []actionsgithubcom.Option + switch workqueueRateLimiter { + case "typed_rate_limiter": + log.Info("Using typed rate limiter (per-item only, no global token bucket)") + controllerOpts = append(controllerOpts, + actionsgithubcom.WithTypedRateLimiter(workqueue.DefaultTypedItemBasedRateLimiter[reconcile.Request]()), + ) + case "bucket_rate_limiter", "": + log.Info("Using default bucket rate limiter") + default: + log.Error(fmt.Errorf("unknown workqueue rate limiter: %s", workqueueRateLimiter), "invalid --workqueue-rate-limiter value") + os.Exit(1) + } + if err = (&actionsgithubcom.AutoscalingRunnerSetReconciler{ Client: mgr.GetClient(), Log: log.WithName("AutoscalingRunnerSet").WithValues("version", build.Version), @@ -302,17 +321,18 @@ func main() { UpdateStrategy: actionsgithubcom.UpdateStrategy(updateStrategy), DefaultRunnerScaleSetListenerImagePullSecrets: autoScalerImagePullSecrets, ResourceBuilder: rb, - }).SetupWithManager(mgr); err != nil { + }).SetupWithManager(mgr, controllerOpts...); err != nil { log.Error(err, "unable to create controller", "controller", "AutoscalingRunnerSet") os.Exit(1) } + runnerOpts := append(controllerOpts, actionsgithubcom.WithMaxConcurrentReconciles(opts.RunnerMaxConcurrentReconciles)) if err = (&actionsgithubcom.EphemeralRunnerReconciler{ Client: mgr.GetClient(), Log: log.WithName("EphemeralRunner").WithValues("version", build.Version), Scheme: mgr.GetScheme(), ResourceBuilder: rb, - }).SetupWithManager(mgr, actionsgithubcom.WithMaxConcurrentReconciles(opts.RunnerMaxConcurrentReconciles)); err != nil { + }).SetupWithManager(mgr, runnerOpts...); err != nil { log.Error(err, "unable to create controller", "controller", "EphemeralRunner") os.Exit(1) } @@ -323,7 +343,7 @@ func main() { Scheme: mgr.GetScheme(), PublishMetrics: metricsAddr != "0", ResourceBuilder: rb, - }).SetupWithManager(mgr); err != nil { + }).SetupWithManager(mgr, controllerOpts...); err != nil { log.Error(err, "unable to create controller", "controller", "EphemeralRunnerSet") os.Exit(1) } @@ -335,7 +355,7 @@ func main() { ListenerMetricsAddr: listenerMetricsAddr, ListenerMetricsEndpoint: listenerMetricsEndpoint, ResourceBuilder: rb, - }).SetupWithManager(mgr); err != nil { + }).SetupWithManager(mgr, controllerOpts...); err != nil { log.Error(err, "unable to create controller", "controller", "AutoscalingListener") os.Exit(1) }