Set per-controller reconcile concurrency defaults and remove --default-max-concurrent-reconciles (#4670)

charts/gha-runner-scale-set-controller-experimental/templates/_controller_template.tpl

@@ -47,9 +47,6 @@ args:
 {{- with .Values.controller.manager.config.watchSingleNamespace }}
   - "--watch-single-namespace={{ . }}"
 {{- end }}
-{{- with .Values.controller.manager.config.defaultMaxConcurrentReconciles }}
-  - "--default-max-concurrent-reconciles={{ . }}"
-{{- end }}
 {{- with .Values.controller.manager.config.autoscalingRunnerSetMaxConcurrentReconciles }}
   - "--autoscaling-runner-set-max-concurrent-reconciles={{ . }}"
 {{- end }}

charts/gha-runner-scale-set-controller-experimental/tests/controller_deployment_args_test.yaml

@@ -74,33 +74,31 @@ tests:
           path: spec.template.spec.containers[0].args
           content: "--listener-metrics-endpoint=/metrics"
 
-  - it: should omit every max-concurrent-reconciles flag by default
+  - it: should render every max-concurrent-reconciles flag by default
     release:
       name: "test-arc"
       namespace: "test-ns"
     asserts:
-      - notContains:
+      # An exact match, so this pins both the values and the render order.
+      - equal:
           path: spec.template.spec.containers[0].args
-          content: "--ephemeral-runner-max-concurrent-reconciles=1"
-      - notContains:
-          path: spec.template.spec.containers[0].args
-          content: "--default-max-concurrent-reconciles=1"
-      - notContains:
-          path: spec.template.spec.containers[0].args
-          content: "--autoscaling-runner-set-max-concurrent-reconciles=1"
-      - notContains:
-          path: spec.template.spec.containers[0].args
-          content: "--autoscaling-listener-max-concurrent-reconciles=1"
-      - notContains:
-          path: spec.template.spec.containers[0].args
-          content: "--ephemeral-runner-set-max-concurrent-reconciles=1"
+          value:
+            - "--auto-scaling-runner-set-only"
+            - "--log-level=debug"
+            - "--log-format=text"
+            - "--autoscaling-runner-set-max-concurrent-reconciles=2"
+            - "--autoscaling-listener-max-concurrent-reconciles=2"
+            - "--ephemeral-runner-set-max-concurrent-reconciles=2"
+            - "--ephemeral-runner-max-concurrent-reconciles=4"
+            - "--listener-metrics-addr=0"
+            - "--listener-metrics-endpoint="
+            - "--metrics-addr=0"
 
   - it: should include every max-concurrent-reconciles flag when configured
     set:
       controller:
         manager:
           config:
-            defaultMaxConcurrentReconciles: 4
             autoscalingRunnerSetMaxConcurrentReconciles: 3
             autoscalingListenerMaxConcurrentReconciles: 5
             ephemeralRunnerSetMaxConcurrentReconciles: 6
@@ -109,9 +107,6 @@ tests:
       name: "test-arc"
       namespace: "test-ns"
     asserts:
-      - contains:
-          path: spec.template.spec.containers[0].args
-          content: "--default-max-concurrent-reconciles=4"
       - contains:
           path: spec.template.spec.containers[0].args
           content: "--autoscaling-runner-set-max-concurrent-reconciles=3"

charts/gha-runner-scale-set-controller-experimental/values.yaml

@@ -28,15 +28,16 @@ controller:
       # Defaults to watch all namespaces when unset.
       watchSingleNamespace: ""
 
-      # The maximum number of concurrent reconciles applied to every controller that does not
-      # set its own value below. Defaults to 1 when unset.
-      defaultMaxConcurrentReconciles: null
-
-      # Per-controller overrides. Each defaults to defaultMaxConcurrentReconciles when unset.
-      autoscalingRunnerSetMaxConcurrentReconciles: null
-      autoscalingListenerMaxConcurrentReconciles: null
-      ephemeralRunnerSetMaxConcurrentReconciles: null
-      ephemeralRunnerMaxConcurrentReconciles: null
+      # The maximum number of concurrent reconciles for each controller, set directly
+      # per controller.
+      autoscalingRunnerSetMaxConcurrentReconciles: 2
+      autoscalingListenerMaxConcurrentReconciles: 2
+      ephemeralRunnerSetMaxConcurrentReconciles: 2
+      # The EphemeralRunner controller is higher because it is the only one of the four that
+      # routinely has many distinct objects to reconcile at once, one per runner.
+      # controller-runtime serialises reconciles per object, so extra workers on the other
+      # three only pay off when many runner scale sets exist.
+      ephemeralRunnerMaxConcurrentReconciles: 4
 
       # List of label prefixes that should NOT be propagated to internal resources.
       excludeLabelPropagationPrefixes: []

charts/gha-runner-scale-set-controller/templates/deployment.yaml

@@ -67,9 +67,6 @@ spec:
         {{- with .Values.flags.watchSingleNamespace }}
         - "--watch-single-namespace={{ . }}"
         {{- end }}
-        {{- with .Values.flags.defaultMaxConcurrentReconciles }}
-        - "--default-max-concurrent-reconciles={{ . }}"
-        {{- end }}
         {{- with .Values.flags.autoscalingRunnerSetMaxConcurrentReconciles }}
         - "--autoscaling-runner-set-max-concurrent-reconciles={{ . }}"
         {{- end }}

charts/gha-runner-scale-set-controller/tests/template_test.go

@@ -363,6 +363,10 @@ func TestTemplate_ControllerDeployment_Defaults(t *testing.T) {
 		"--auto-scaling-runner-set-only",
 		"--log-level=debug",
 		"--log-format=text",
+		"--autoscaling-runner-set-max-concurrent-reconciles=2",
+		"--autoscaling-listener-max-concurrent-reconciles=2",
+		"--ephemeral-runner-set-max-concurrent-reconciles=2",
+		"--ephemeral-runner-max-concurrent-reconciles=4",
 		"--metrics-addr=0",
 		"--listener-metrics-addr=0",
 		"--listener-metrics-endpoint=",
@@ -514,6 +518,10 @@ func TestTemplate_ControllerDeployment_Customize(t *testing.T) {
 		"--auto-scaler-image-pull-secrets=dockerhub",
 		"--log-level=info",
 		"--log-format=json",
+		"--autoscaling-runner-set-max-concurrent-reconciles=2",
+		"--autoscaling-listener-max-concurrent-reconciles=2",
+		"--ephemeral-runner-set-max-concurrent-reconciles=2",
+		"--ephemeral-runner-max-concurrent-reconciles=4",
 		"--listener-metrics-addr=0",
 		"--listener-metrics-endpoint=",
 		"--metrics-addr=0",
@@ -641,6 +649,10 @@ func TestTemplate_EnableLeaderElection(t *testing.T) {
 		"--leader-election-id=test-arc-gha-rs-controller",
 		"--log-level=debug",
 		"--log-format=text",
+		"--autoscaling-runner-set-max-concurrent-reconciles=2",
+		"--autoscaling-listener-max-concurrent-reconciles=2",
+		"--ephemeral-runner-set-max-concurrent-reconciles=2",
+		"--ephemeral-runner-max-concurrent-reconciles=4",
 		"--listener-metrics-addr=0",
 		"--listener-metrics-endpoint=",
 		"--metrics-addr=0",
@@ -681,6 +693,10 @@ func TestTemplate_ControllerDeployment_ForwardImagePullSecrets(t *testing.T) {
 		"--auto-scaler-image-pull-secrets=ghcr",
 		"--log-level=debug",
 		"--log-format=text",
+		"--autoscaling-runner-set-max-concurrent-reconciles=2",
+		"--autoscaling-listener-max-concurrent-reconciles=2",
+		"--ephemeral-runner-set-max-concurrent-reconciles=2",
+		"--ephemeral-runner-max-concurrent-reconciles=4",
 		"--listener-metrics-addr=0",
 		"--listener-metrics-endpoint=",
 		"--metrics-addr=0",
@@ -770,6 +786,10 @@ func TestTemplate_ControllerDeployment_WatchSingleNamespace(t *testing.T) {
 		"--log-level=debug",
 		"--log-format=text",
 		"--watch-single-namespace=demo",
+		"--autoscaling-runner-set-max-concurrent-reconciles=2",
+		"--autoscaling-listener-max-concurrent-reconciles=2",
+		"--ephemeral-runner-set-max-concurrent-reconciles=2",
+		"--ephemeral-runner-max-concurrent-reconciles=4",
 		"--listener-metrics-addr=0",
 		"--listener-metrics-endpoint=",
 		"--metrics-addr=0",
@@ -801,30 +821,49 @@ func TestTemplate_ControllerDeployment_MaxConcurrentReconciles(t *testing.T) {
 	releaseName := "test-arc"
 	namespaceName := "test-" + strings.ToLower(random.UniqueID())
 
-	options := &helm.Options{
-		Logger: logger.Discard,
-		SetValues: map[string]string{
-			"flags.defaultMaxConcurrentReconciles":              "4",
-			"flags.autoscalingRunnerSetMaxConcurrentReconciles": "3",
-			"flags.autoscalingListenerMaxConcurrentReconciles":  "5",
-			"flags.ephemeralRunnerSetMaxConcurrentReconciles":   "6",
-			"flags.ephemeralRunnerMaxConcurrentReconciles":      "20",
-		},
-		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
-	}
+	t.Run("every flag renders by default", func(t *testing.T) {
+		options := &helm.Options{
+			Logger:         logger.Discard,
+			KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+		}
 
-	output := helm.RenderTemplateContext(t, t.Context(), options, helmChartPath, releaseName, []string{"templates/deployment.yaml"})
+		output := helm.RenderTemplateContext(t, t.Context(), options, helmChartPath, releaseName, []string{"templates/deployment.yaml"})
 
-	var deployment appsv1.Deployment
-	helm.UnmarshalK8SYaml(t, output, &deployment)
+		var deployment appsv1.Deployment
+		helm.UnmarshalK8SYaml(t, output, &deployment)
 
-	assert.Len(t, deployment.Spec.Template.Spec.Containers, 1)
-	args := deployment.Spec.Template.Spec.Containers[0].Args
-	assert.Contains(t, args, "--default-max-concurrent-reconciles=4")
-	assert.Contains(t, args, "--autoscaling-runner-set-max-concurrent-reconciles=3")
-	assert.Contains(t, args, "--autoscaling-listener-max-concurrent-reconciles=5")
-	assert.Contains(t, args, "--ephemeral-runner-set-max-concurrent-reconciles=6")
-	assert.Contains(t, args, "--ephemeral-runner-max-concurrent-reconciles=20")
+		require.Len(t, deployment.Spec.Template.Spec.Containers, 1)
+		args := deployment.Spec.Template.Spec.Containers[0].Args
+		assert.Contains(t, args, "--autoscaling-runner-set-max-concurrent-reconciles=2")
+		assert.Contains(t, args, "--autoscaling-listener-max-concurrent-reconciles=2")
+		assert.Contains(t, args, "--ephemeral-runner-set-max-concurrent-reconciles=2")
+		assert.Contains(t, args, "--ephemeral-runner-max-concurrent-reconciles=4")
+	})
+
+	t.Run("every flag renders when configured", func(t *testing.T) {
+		options := &helm.Options{
+			Logger: logger.Discard,
+			SetValues: map[string]string{
+				"flags.autoscalingRunnerSetMaxConcurrentReconciles": "3",
+				"flags.autoscalingListenerMaxConcurrentReconciles":  "5",
+				"flags.ephemeralRunnerSetMaxConcurrentReconciles":   "6",
+				"flags.ephemeralRunnerMaxConcurrentReconciles":      "20",
+			},
+			KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+		}
+
+		output := helm.RenderTemplateContext(t, t.Context(), options, helmChartPath, releaseName, []string{"templates/deployment.yaml"})
+
+		var deployment appsv1.Deployment
+		helm.UnmarshalK8SYaml(t, output, &deployment)
+
+		require.Len(t, deployment.Spec.Template.Spec.Containers, 1)
+		args := deployment.Spec.Template.Spec.Containers[0].Args
+		assert.Contains(t, args, "--autoscaling-runner-set-max-concurrent-reconciles=3")
+		assert.Contains(t, args, "--autoscaling-listener-max-concurrent-reconciles=5")
+		assert.Contains(t, args, "--ephemeral-runner-set-max-concurrent-reconciles=6")
+		assert.Contains(t, args, "--ephemeral-runner-max-concurrent-reconciles=20")
+	})
 }
 
 func TestTemplate_ControllerContainerEnvironmentVariables(t *testing.T) {

charts/gha-runner-scale-set-controller/values.yaml

@@ -114,16 +114,17 @@ flags:
   ## Defaults to watch all namespaces when unset.
   # watchSingleNamespace: ""
 
-  ## The maximum number of concurrent reconciles applied to every controller that does not set its own value below.
-  # Increase this value to improve the throughput of the controller when running many runner scale sets.
-  # It may also increase the load on the API server and the external service (e.g. GitHub API).
-  # defaultMaxConcurrentReconciles: 1
-
-  ## Per-controller overrides. Each defaults to defaultMaxConcurrentReconciles when unset.
-  # autoscalingRunnerSetMaxConcurrentReconciles: 1
-  # autoscalingListenerMaxConcurrentReconciles: 1
-  # ephemeralRunnerSetMaxConcurrentReconciles: 1
-  # ephemeralRunnerMaxConcurrentReconciles: 1
+  ## The maximum number of concurrent reconciles for each controller, set directly per controller.
+  # Increase these values to improve the throughput of the controller when running many runner scale sets.
+  # They may also increase the load on the API server and the external service (e.g. GitHub API).
+  autoscalingRunnerSetMaxConcurrentReconciles: 2
+  autoscalingListenerMaxConcurrentReconciles: 2
+  ephemeralRunnerSetMaxConcurrentReconciles: 2
+  ## The EphemeralRunner controller is higher because it is the only one of the four that routinely
+  ## has many distinct objects to reconcile at once, one per runner. controller-runtime serialises
+  ## reconciles per object, so extra workers on the other three only pay off when many runner scale
+  ## sets exist.
+  ephemeralRunnerMaxConcurrentReconciles: 4
 
   ## Defines a list of prefixes that should not be propagated to internal resources.
   ## This is useful when you have labels that are used for internal purposes and should not be propagated to internal resources.