From 3998f6dee6d4ed9d316fd8e09c744f9f5cd61fcd Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Thu, 12 Dec 2024 05:19:43 +0900 Subject: [PATCH] Make EphemeralRunnerController MaxConcurrentReconciles configurable (#3832) Co-authored-by: Bassem Dghaidi <568794+Link-@users.noreply.github.com> --- .../templates/deployment.yaml | 3 + .../values.yaml | 5 ++ .../ephemeralrunner_controller.go | 14 +++-- controllers/actions.github.com/options.go | 56 +++++++++++++++++++ main.go | 7 ++- 5 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 controllers/actions.github.com/options.go diff --git a/charts/gha-runner-scale-set-controller/templates/deployment.yaml b/charts/gha-runner-scale-set-controller/templates/deployment.yaml index 66b9a4b5..877ab984 100644 --- a/charts/gha-runner-scale-set-controller/templates/deployment.yaml +++ b/charts/gha-runner-scale-set-controller/templates/deployment.yaml @@ -65,6 +65,9 @@ spec: {{- with .Values.flags.watchSingleNamespace }} - "--watch-single-namespace={{ . }}" {{- end }} + {{- with .Values.runnerMaxConcurrentReconciles }} + - "--runner-max-concurrent-reconciles={{ . }}" + {{- end }} {{- with .Values.flags.updateStrategy }} - "--update-strategy={{ . }}" {{- end }} diff --git a/charts/gha-runner-scale-set-controller/values.yaml b/charts/gha-runner-scale-set-controller/values.yaml index 8e74317e..e96ffdda 100644 --- a/charts/gha-runner-scale-set-controller/values.yaml +++ b/charts/gha-runner-scale-set-controller/values.yaml @@ -106,6 +106,11 @@ flags: ## Defaults to watch all namespaces when unset. # watchSingleNamespace: "" + ## The maximum number of concurrent reconciles which can be run by the EphemeralRunner controller. + # Increase this value to improve the throughput of the controller. + # It may also increase the load on the API server and the external service (e.g. GitHub API). + runnerMaxConcurrentReconciles: 2 + ## Defines how the controller should handle upgrades while having running jobs. ## ## The strategies available are: diff --git a/controllers/actions.github.com/ephemeralrunner_controller.go b/controllers/actions.github.com/ephemeralrunner_controller.go index 36ea1146..e3a01827 100644 --- a/controllers/actions.github.com/ephemeralrunner_controller.go +++ b/controllers/actions.github.com/ephemeralrunner_controller.go @@ -823,12 +823,14 @@ func (r *EphemeralRunnerReconciler) deleteRunnerFromService(ctx context.Context, } // SetupWithManager sets up the controller with the Manager. -func (r *EphemeralRunnerReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha1.EphemeralRunner{}). - Owns(&corev1.Pod{}). - WithEventFilter(predicate.ResourceVersionChangedPredicate{}). - Complete(r) +func (r *EphemeralRunnerReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error { + return builderWithOptions( + ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.EphemeralRunner{}). + Owns(&corev1.Pod{}). + WithEventFilter(predicate.ResourceVersionChangedPredicate{}), + opts, + ).Complete(r) } func runnerContainerStatus(pod *corev1.Pod) *corev1.ContainerStatus { diff --git a/controllers/actions.github.com/options.go b/controllers/actions.github.com/options.go new file mode 100644 index 00000000..7c7c240e --- /dev/null +++ b/controllers/actions.github.com/options.go @@ -0,0 +1,56 @@ +package actionsgithubcom + +import ( + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/controller" +) + +// Options is the optional configuration for the controllers, which can be +// set via command-line flags or environment variables. +type Options struct { + // RunnerMaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run + // by the EphemeralRunnerController. + RunnerMaxConcurrentReconciles int +} + +// OptionsWithDefault returns the default options. +// This is here to maintain the options and their default values in one place, +// rather than having to correlate those in multiple places. +func OptionsWithDefault() Options { + return Options{ + RunnerMaxConcurrentReconciles: 2, + } +} + +type Option func(*controller.Options) + +// WithMaxConcurrentReconciles sets the maximum number of concurrent Reconciles which can be run. +// +// This is useful to improve the throughput of the controller, but it may also increase the load on the API server and +// the external service (e.g. GitHub API). The default value is 1, as defined by the controller-runtime. +// +// See https://github.com/actions/actions-runner-controller/issues/3021 for more information +// on real-world use cases and the potential impact of this option. +func WithMaxConcurrentReconciles(n int) Option { + return func(b *controller.Options) { + b.MaxConcurrentReconciles = n + } +} + +// builderWithOptions applies the given options to the provided builder, if any. +// This is a helper function to avoid the need to import the controller-runtime package in every reconciler source file +// and the command package that creates the controller. +// This is also useful for reducing code duplication around setting controller options in +// multiple reconcilers. +func builderWithOptions(b *builder.Builder, opts []Option) *builder.Builder { + if len(opts) == 0 { + return b + } + + var controllerOpts controller.Options + for _, opt := range opts { + opt(&controllerOpts) + } + + return b.WithOptions(controllerOpts) +} diff --git a/main.go b/main.go index d7edea6c..a7da63a6 100644 --- a/main.go +++ b/main.go @@ -102,6 +102,8 @@ func main() { autoScalerImagePullSecrets stringSlice + opts = actionsgithubcom.OptionsWithDefault() + commonRunnerLabels commaSeparatedStringSlice ) var c github.Config @@ -136,6 +138,7 @@ func main() { flag.DurationVar(&defaultScaleDownDelay, "default-scale-down-delay", actionssummerwindnet.DefaultScaleDownDelay, "The approximate delay for a scale down followed by a scale up, used to prevent flapping (down->up->down->... loop)") flag.IntVar(&port, "port", 9443, "The port to which the admission webhook endpoint should bind") flag.DurationVar(&syncPeriod, "sync-period", 1*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled.") + flag.IntVar(&opts.RunnerMaxConcurrentReconciles, "runner-max-concurrent-reconciles", opts.RunnerMaxConcurrentReconciles, "The maximum number of concurrent reconciles which can be run by the EphemeralRunner controller. Increase this value to improve the throughput of the controller, but it may also increase the load on the API server and the external service (e.g. GitHub API).") flag.Var(&commonRunnerLabels, "common-runner-labels", "Runner labels in the K1=V1,K2=V2,... format that are inherited all the runners created by the controller. See https://github.com/actions/actions-runner-controller/issues/321 for more information") flag.StringVar(&namespace, "watch-namespace", "", "The namespace to watch for custom resources. Set to empty for letting it watch for all namespaces.") flag.StringVar(&watchSingleNamespace, "watch-single-namespace", "", "Restrict to watch for custom resources in a single namespace.") @@ -156,6 +159,8 @@ func main() { } c.Log = &log + log.Info("Using options", "runner-max-concurrent-reconciles", opts.RunnerMaxConcurrentReconciles) + if !autoScalingRunnerSetOnly { ghClient, err = c.NewClient() if err != nil { @@ -285,7 +290,7 @@ func main() { Scheme: mgr.GetScheme(), ActionsClient: actionsMultiClient, ResourceBuilder: rb, - }).SetupWithManager(mgr); err != nil { + }).SetupWithManager(mgr, actionsgithubcom.WithMaxConcurrentReconciles(opts.RunnerMaxConcurrentReconciles)); err != nil { log.Error(err, "unable to create controller", "controller", "EphemeralRunner") os.Exit(1) }