Make controller terminationGracePeriodSeconds configurable and align it with graceful shutdown timeout (#4556)

charts/gha-runner-scale-set-controller-experimental/templates/deployment.yaml

@@ -40,7 +40,7 @@ spec:
         -
       {{- toYaml . | nindent 10 }}
         {{- end }}
-      terminationGracePeriodSeconds: {{ default 10 (index $podSpec "terminationGracePeriodSeconds") }}
+      terminationGracePeriodSeconds: {{ default 35 (index $podSpec "terminationGracePeriodSeconds") }}
       volumes:
         - name: tmp
           emptyDir: {}

charts/gha-runner-scale-set-controller-experimental/tests/controller_deployment_pod_extra_fields_test.yaml

@@ -36,6 +36,20 @@ tests:
           path: spec.template.spec.dnsPolicy
           value: "ClusterFirstWithHostNet"
 
+  - it: should allow overriding terminationGracePeriodSeconds via controller.pod
+    set:
+      controller:
+        pod:
+          spec:
+            terminationGracePeriodSeconds: 60
+    release:
+      name: "test-name"
+      namespace: "test-namespace"
+    asserts:
+      - equal:
+          path: spec.template.spec.terminationGracePeriodSeconds
+          value: 60
+
   - it: should not allow overriding serviceAccountName via controller.pod
     set:
       controller:

charts/gha-runner-scale-set-controller-experimental/tests/controller_deployment_smoke_test.yaml

@@ -25,3 +25,6 @@ tests:
       - contains:
           path: spec.template.spec.containers[0].command
           content: "/manager"
+      - equal:
+          path: spec.template.spec.terminationGracePeriodSeconds
+          value: 35

charts/gha-runner-scale-set-controller-experimental/values.yaml

@@ -96,7 +96,9 @@ controller:
       affinity: {}
       # Pod topology spread constraints.
       topologySpreadConstraints: []
-      # Pod termination grace period (overrides default 10s).
+      # Pod termination grace period. Overrides the default of 35s, which must
+      # stay above the manager's graceful shutdown timeout (30s, controller-runtime's
+      # default) so in-flight reconciles can finish before the kubelet sends SIGKILL.
       terminationGracePeriodSeconds: null
       # Additional volumes appended to the default ones.
       volumes: []