diff --git a/.gitignore b/.gitignore index 713cf538..fd3eb6a8 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,7 @@ bin /.tools + +*.tgz +internal +internal/ diff --git a/Makefile.sabre.mk b/Makefile.sabre.mk new file mode 100644 index 00000000..138b4560 --- /dev/null +++ b/Makefile.sabre.mk @@ -0,0 +1,56 @@ +VERSION ?= 0.11.0-sabre-1 + +DOCKER_IMAGE_NAME ?= sabre/gh/arc/actions-runner-controller +DOCKER_IMAGE_VERSION ?= 0.11.0-sabre-1 + +COMMIT_SHA = $(shell git rev-parse HEAD) + +ifeq (${PLATFORMS}, ) + export PLATFORMS="linux/amd64" +endif + +package-controller-chart: ## build controller chart + @echo "Building package-controller" + @cd charts/gha-runner-scale-set-controller && \ + rm -rf *tgz && \ + helm package . --version ${VERSION} --app-version ${VERSION} + +package-runnerset-chart: ## build runnerset chart + @echo "Building package-runnerset" + @cd charts/gha-runner-scale-set && \ + rm -rf *tgz && \ + helm package . --version ${VERSION} --app-version ${VERSION} + +upload-charts: package-controller-chart package-runnerset-chart ## upload charts + @echo "Uploading charts" + @ngp nexus raw upload charts/gha-runner-scale-set-controller/gha-runner-scale-set-controller-${VERSION}.tgz sabre/gh/forked-arc/charts/gha-runner-scale-set-controller-${VERSION}.tgz + @ngp nexus raw upload charts/gha-runner-scale-set/gha-runner-scale-set-${VERSION}.tgz sabre/gh/forked-arc/charts/gha-runner-scale-set-${VERSION}.tgz + +build-controller-image: ## build docker image, it will not be pushed to remote repository + @echo "Building image" + export DOCKER_CLI_EXPERIMENTAL=enabled ;\ + export DOCKER_BUILDKIT=1 + export PUSH_ARG="--load" + + @if ! docker buildx ls | grep -q container-builder; then\ + docker buildx create --platform ${PLATFORMS} --name container-builder --use;\ + fi + docker buildx build \ + --platform ${PLATFORMS} \ + --build-arg VERSION=${VERSION} \ + --build-arg COMMIT_SHA=${COMMIT_SHA} \ + -t "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_VERSION}" \ + -f Dockerfile \ + . --load + +upload-image: build-controller-image ## build and push docker image to repository + @echo "Uploading image" + @ngp nexus docker upload "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_VERSION}" gh/forked-arc/actions-runner-controller:${DOCKER_IMAGE_VERSION} + +upload-all: upload-charts upload-image ## upload all built charts and defaults to raw-staging repo + @echo "All charts and images uploaded to raw-staging" + +help: ## show usage and tasks (default) + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +.DEFAULT_GOAL := help \ No newline at end of file diff --git a/README-SABRE.md b/README-SABRE.md new file mode 100644 index 00000000..a3698d3f --- /dev/null +++ b/README-SABRE.md @@ -0,0 +1,25 @@ +# Local development and update procedure + +Versions are kept in separate branches: release/-custom. + +## Prerequisites + +1. Clone `sabre-internal/gh-core.github-sabre-arc` repository. +2. Link `gh-core.github-sabre-arc/sabreoss-arc-devenv` folder as `./internal`. +3. Enter the folder and then run `devenev shell`. +4. Run all commands in the shell. + +## Update procedure + +1. Upgrade the fork branch with the original GH ARC repo: + * Upgrade should be made to the branch `master`, which is clean from our changes +2. Create new custom target branch `release/-custom` from the updated `master` branch or specific commit (if default branch is ahead selected base version) +3. Create your feature branch e.g: `feature/update-to-` from previous `release/-custom` + and rebase it on `release/-custom` + * If there are any conflicts, resolve them testing the code execution +4. Update the `Makefile.sabre.mk` file, changing `VERSION` and `DOCKER_IMAGE_VERSION` to -sabre-1 + * All other changes to the fork should introduce update in the `Makefile.sabre.mk` file as well + (e.g. next change in the controller image should be `DOCKER_IMAGE_VERSION=-sabre-2`) +5. Create pull request to `release/-custom` and merge it. +6. Release charts and controller/listener image from branch `release/-custom`: + * Run `make -f Makefile.sabre.mk upload-all` or other targets to release only charts or image. diff --git a/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalinglisteners.yaml b/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalinglisteners.yaml index 184d8775..a15fee9e 100644 --- a/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalinglisteners.yaml +++ b/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalinglisteners.yaml @@ -12,6 +12,8 @@ spec: listKind: AutoscalingListenerList plural: autoscalinglisteners singular: autoscalinglistener + shortNames: + - gharclistener scope: Namespaced versions: - additionalPrinterColumns: diff --git a/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalingrunnersets.yaml b/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalingrunnersets.yaml index eecee1fb..32b36a26 100644 --- a/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalingrunnersets.yaml +++ b/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalingrunnersets.yaml @@ -12,6 +12,8 @@ spec: listKind: AutoscalingRunnerSetList plural: autoscalingrunnersets singular: autoscalingrunnerset + shortNames: + - gharcrunnerset scope: Namespaced versions: - additionalPrinterColumns: diff --git a/charts/gha-runner-scale-set-controller/crds/actions.github.com_ephemeralrunners.yaml b/charts/gha-runner-scale-set-controller/crds/actions.github.com_ephemeralrunners.yaml index ad9f097f..099f527c 100644 --- a/charts/gha-runner-scale-set-controller/crds/actions.github.com_ephemeralrunners.yaml +++ b/charts/gha-runner-scale-set-controller/crds/actions.github.com_ephemeralrunners.yaml @@ -12,6 +12,8 @@ spec: listKind: EphemeralRunnerList plural: ephemeralrunners singular: ephemeralrunner + shortNames: + - gharcephemeralrunner scope: Namespaced versions: - additionalPrinterColumns: diff --git a/charts/gha-runner-scale-set-controller/crds/actions.github.com_ephemeralrunnersets.yaml b/charts/gha-runner-scale-set-controller/crds/actions.github.com_ephemeralrunnersets.yaml index a2c8f787..9f65c8ac 100644 --- a/charts/gha-runner-scale-set-controller/crds/actions.github.com_ephemeralrunnersets.yaml +++ b/charts/gha-runner-scale-set-controller/crds/actions.github.com_ephemeralrunnersets.yaml @@ -12,6 +12,8 @@ spec: listKind: EphemeralRunnerSetList plural: ephemeralrunnersets singular: ephemeralrunnerset + shortNames: + - gharcephemeralrunnerset scope: Namespaced versions: - additionalPrinterColumns: diff --git a/charts/gha-runner-scale-set-controller/templates/deployment.yaml b/charts/gha-runner-scale-set-controller/templates/deployment.yaml index 200cbe0f..4a853198 100644 --- a/charts/gha-runner-scale-set-controller/templates/deployment.yaml +++ b/charts/gha-runner-scale-set-controller/templates/deployment.yaml @@ -108,6 +108,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + {{- if eq .Values.flags.skipRbacSetup true }} + - name: SKIP_RBAC_SETUP_FOR_CONTROLLER + value: "true" + {{- end }} + {{- if eq .Values.flags.skipListenerSaCreation true }} + - name: SKIP_LISTENERS_SA_CREATION + value: "true" + {{- end }} + {{- if eq .Values.flags.skipRbacSetupForListeners true }} + - name: SKIP_RBAC_SETUP_FOR_LISTENERS + value: "true" + {{- end }} {{- with .Values.env }} {{- if kindIs "slice" . }} {{- toYaml . | nindent 8 }} diff --git a/charts/gha-runner-scale-set-controller/templates/leader_election_role.yaml b/charts/gha-runner-scale-set-controller/templates/leader_election_role.yaml index 236a51fc..2b56c197 100644 --- a/charts/gha-runner-scale-set-controller/templates/leader_election_role.yaml +++ b/charts/gha-runner-scale-set-controller/templates/leader_election_role.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.flags.skipRbacSetup false }} {{- if gt (int (default 1 .Values.replicaCount)) 1 }} # permissions to do leader election. apiVersion: rbac.authorization.k8s.io/v1 @@ -13,3 +14,4 @@ rules: resources: ["events"] verbs: ["create", "patch"] {{- end }} +{{- end }} diff --git a/charts/gha-runner-scale-set-controller/templates/leader_election_role_binding.yaml b/charts/gha-runner-scale-set-controller/templates/leader_election_role_binding.yaml index 16d38388..c41b65ae 100644 --- a/charts/gha-runner-scale-set-controller/templates/leader_election_role_binding.yaml +++ b/charts/gha-runner-scale-set-controller/templates/leader_election_role_binding.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.flags.skipRbacSetup false }} {{- if gt (int (default 1 .Values.replicaCount)) 1 }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -12,4 +13,5 @@ subjects: - kind: ServiceAccount name: {{ include "gha-runner-scale-set-controller.serviceAccountName" . }} namespace: {{ include "gha-runner-scale-set-controller.namespace" . }} +{{- end }} {{- end }} \ No newline at end of file diff --git a/charts/gha-runner-scale-set-controller/templates/manager_cluster_role.yaml b/charts/gha-runner-scale-set-controller/templates/manager_cluster_role.yaml index cc58e3c2..837f9fcd 100644 --- a/charts/gha-runner-scale-set-controller/templates/manager_cluster_role.yaml +++ b/charts/gha-runner-scale-set-controller/templates/manager_cluster_role.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.flags.skipRbacSetup false }} {{- if empty .Values.flags.watchSingleNamespace }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -142,3 +143,4 @@ rules: - watch - patch {{- end }} +{{- end }} diff --git a/charts/gha-runner-scale-set-controller/templates/manager_cluster_role_binding.yaml b/charts/gha-runner-scale-set-controller/templates/manager_cluster_role_binding.yaml index 60291d21..e24eb53c 100644 --- a/charts/gha-runner-scale-set-controller/templates/manager_cluster_role_binding.yaml +++ b/charts/gha-runner-scale-set-controller/templates/manager_cluster_role_binding.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.flags.skipRbacSetup false }} {{- if empty .Values.flags.watchSingleNamespace }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -11,4 +12,5 @@ subjects: - kind: ServiceAccount name: {{ include "gha-runner-scale-set-controller.serviceAccountName" . }} namespace: {{ include "gha-runner-scale-set-controller.namespace" . }} +{{- end }} {{- end }} \ No newline at end of file diff --git a/charts/gha-runner-scale-set-controller/templates/manager_listener_role.yaml b/charts/gha-runner-scale-set-controller/templates/manager_listener_role.yaml index a238d5fc..e322607a 100644 --- a/charts/gha-runner-scale-set-controller/templates/manager_listener_role.yaml +++ b/charts/gha-runner-scale-set-controller/templates/manager_listener_role.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.flags.skipRbacSetup false }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -37,4 +38,5 @@ rules: - delete - get - patch - - update \ No newline at end of file + - update +{{- end }} \ No newline at end of file diff --git a/charts/gha-runner-scale-set-controller/templates/manager_listener_role_binding.yaml b/charts/gha-runner-scale-set-controller/templates/manager_listener_role_binding.yaml index efc66b57..6237fd25 100644 --- a/charts/gha-runner-scale-set-controller/templates/manager_listener_role_binding.yaml +++ b/charts/gha-runner-scale-set-controller/templates/manager_listener_role_binding.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.flags.skipRbacSetup false }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -10,4 +11,5 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "gha-runner-scale-set-controller.serviceAccountName" . }} - namespace: {{ include "gha-runner-scale-set-controller.namespace" . }} \ No newline at end of file + namespace: {{ include "gha-runner-scale-set-controller.namespace" . }} +{{- end }} \ No newline at end of file diff --git a/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role.yaml b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role.yaml index c486a79b..9f232caf 100644 --- a/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role.yaml +++ b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.flags.skipRbacSetup false }} {{- if .Values.flags.watchSingleNamespace }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -82,3 +83,4 @@ rules: - list - watch {{- end }} +{{- end }} diff --git a/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role_binding.yaml b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role_binding.yaml index 3bb5247e..fc32f0ec 100644 --- a/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role_binding.yaml +++ b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role_binding.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.flags.skipRbacSetup false }} {{- if .Values.flags.watchSingleNamespace }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -12,4 +13,5 @@ subjects: - kind: ServiceAccount name: {{ include "gha-runner-scale-set-controller.serviceAccountName" . }} namespace: {{ include "gha-runner-scale-set-controller.namespace" . }} +{{- end }} {{- end }} \ No newline at end of file diff --git a/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role.yaml b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role.yaml index ac5a2d93..3a94464e 100644 --- a/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role.yaml +++ b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.flags.skipRbacSetup false }} {{- if .Values.flags.watchSingleNamespace }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -123,3 +124,4 @@ rules: - watch - patch {{- end }} +{{- end }} diff --git a/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role_binding.yaml b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role_binding.yaml index c4810b7a..5d027142 100644 --- a/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role_binding.yaml +++ b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role_binding.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.flags.skipRbacSetup false }} {{- if .Values.flags.watchSingleNamespace }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -13,3 +14,4 @@ subjects: name: {{ include "gha-runner-scale-set-controller.serviceAccountName" . }} namespace: {{ include "gha-runner-scale-set-controller.namespace" . }} {{- end }} +{{- end }} diff --git a/charts/gha-runner-scale-set-controller/values.yaml b/charts/gha-runner-scale-set-controller/values.yaml index fb54ed99..5aee0f6a 100644 --- a/charts/gha-runner-scale-set-controller/values.yaml +++ b/charts/gha-runner-scale-set-controller/values.yaml @@ -136,9 +136,21 @@ flags: # excludeLabelPropagationPrefixes: # - "argocd.argoproj.io/instance" -# Overrides the default `.Release.Namespace` for all resources in this chart. -namespaceOverride: "" - ## Defines the K8s client rate limiter parameters. # k8sClientRateLimiterQPS: 20 # k8sClientRateLimiterBurst: 30 + +# Additional flags + # Skip any RBAC setup for the controller (includes roles and rolebindings) assuming all setup upfront + # This will be applied in both modes: watchSingleNamespace and global + skipRbacSetup: false + + # Require service account for the listener pods provided via Autoscaling Runner Set spec, + # If not set, the default service account will be created and used + skipListenerSaCreation: false + + # Skip setting up the Role and RoleBinding for the created/used listeners, assuming all is done upfront + skipRbacSetupForListeners: false + +# Overrides the default `.Release.Namespace` for all resources in this chart. +namespaceOverride: "" \ No newline at end of file diff --git a/charts/gha-runner-scale-set/templates/manager_role.yaml b/charts/gha-runner-scale-set/templates/manager_role.yaml index bbf92799..4092b045 100644 --- a/charts/gha-runner-scale-set/templates/manager_role.yaml +++ b/charts/gha-runner-scale-set/templates/manager_role.yaml @@ -1,4 +1,5 @@ {{- $hasCustomResourceMeta := (and .Values.resourceMeta .Values.resourceMeta.managerRole) }} +{{- if eq .Values.flags.skipRbacSetup false }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -98,3 +99,4 @@ rules: verbs: - get {{- end }} +{{- end }} diff --git a/charts/gha-runner-scale-set/templates/manager_role_binding.yaml b/charts/gha-runner-scale-set/templates/manager_role_binding.yaml index 108af61c..090e7975 100644 --- a/charts/gha-runner-scale-set/templates/manager_role_binding.yaml +++ b/charts/gha-runner-scale-set/templates/manager_role_binding.yaml @@ -1,4 +1,5 @@ {{- $hasCustomResourceMeta := (and .Values.resourceMeta .Values.resourceMeta.managerRoleBinding) }} +{{- if eq .Values.flags.skipRbacSetup false }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -41,3 +42,4 @@ subjects: - kind: ServiceAccount name: {{ include "gha-runner-scale-set.managerServiceAccountName" . | nindent 4 }} namespace: {{ include "gha-runner-scale-set.managerServiceAccountNamespace" . | nindent 4 }} +{{- end }} diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml index 4b4640cf..89ba5920 100644 --- a/charts/gha-runner-scale-set/values.yaml +++ b/charts/gha-runner-scale-set/values.yaml @@ -133,6 +133,9 @@ githubConfigSecret: ## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec # listenerTemplate: # spec: +# # Service account name will be used if controller is installed with 'flags.skipListenerSaCreation' set to true +# # In such case, provided SA must be present in the same namespace as the controller +# serviceAccountName: "" # containers: # # Use this section to append additional configuration to the listener container. # # If you change the name of the container, the configuration will not be applied to the listener, @@ -530,3 +533,6 @@ namespaceOverride: "" # key: value # annotations: # key: value +flags: + # If set to true, assume all RBAC for the controller SA is already in place + skipRbacSetup: false \ No newline at end of file diff --git a/controllers/actions.github.com/autoscalinglistener_controller.go b/controllers/actions.github.com/autoscalinglistener_controller.go index 4c8ed34a..0e18628d 100644 --- a/controllers/actions.github.com/autoscalinglistener_controller.go +++ b/controllers/actions.github.com/autoscalinglistener_controller.go @@ -18,6 +18,7 @@ package actionsgithubcom import ( "context" + "errors" "fmt" "time" @@ -145,53 +146,72 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl. // Make sure the runner scale set listener service account is created for the listener pod in the controller namespace serviceAccount := new(corev1.ServiceAccount) - if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Namespace, Name: autoscalingListener.Name}, serviceAccount); err != nil { - if !kerrors.IsNotFound(err) { - log.Error(err, "Unable to get listener service accounts", "namespace", autoscalingListener.Namespace, "name", autoscalingListener.Name) - return ctrl.Result{}, err - } + if !ShouldSkipListenerSaCreation() { + log.Info("Creating a service account for the listener pod") + if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Namespace, Name: autoscalingListener.Name}, serviceAccount); err != nil { + if !kerrors.IsNotFound(err) { + log.Error(err, "Unable to get listener service accounts", "namespace", autoscalingListener.Namespace, "name", autoscalingListener.Name) + return ctrl.Result{}, err + } - // Create a service account for the listener pod in the controller namespace - log.Info("Creating a service account for the listener pod") - return r.createServiceAccountForListener(ctx, autoscalingListener, log) - } + // Create a service account for the listener pod in the controller namespace + log.Info("Creating a service account for the listener pod") + return r.createServiceAccountForListener(ctx, autoscalingListener, log) + } + } else { + log.Info("Skipping listener service account creation, checking if provided one exists") + serviceAccountName := autoscalingListener.Spec.Template.Spec.ServiceAccountName + if serviceAccountName == "" { + err := errors.New("Service Account name required for listener, but was not provided in the runnerset") + log.Error(err, "Service Account name required for listener, but was not provided in the runnerset", "namespace", autoscalingListener.Namespace) + return ctrl.Result{}, err + } + if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Namespace, Name: serviceAccountName}, serviceAccount); err != nil { + log.Error(err, "Service Account not found in the namespace", "namespace", autoscalingListener.Namespace, "name", serviceAccountName) + return ctrl.Result{}, errors.New("Listener's service account not found") + } + } // TODO: make sure the service account is up to date // Make sure the runner scale set listener role is created in the AutoscalingRunnerSet namespace - listenerRole := new(rbacv1.Role) - if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Name}, listenerRole); err != nil { - if !kerrors.IsNotFound(err) { - log.Error(err, "Unable to get listener role", "namespace", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, "name", autoscalingListener.Name) - return ctrl.Result{}, err - } + if !ShouldSkipListenerRbacSetup() { + listenerRole := new(rbacv1.Role) + if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Name}, listenerRole); err != nil { + if !kerrors.IsNotFound(err) { + log.Error(err, "Unable to get listener role", "namespace", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, "name", autoscalingListener.Name) + return ctrl.Result{}, err + } - // Create a role for the listener pod in the AutoScalingRunnerSet namespace - log.Info("Creating a role for the listener pod") - return r.createRoleForListener(ctx, autoscalingListener, log) - } + // Create a role for the listener pod in the AutoScalingRunnerSet namespace + log.Info("Creating a role for the listener pod") + return r.createRoleForListener(ctx, autoscalingListener, log) + } - // Make sure the listener role has the up-to-date rules - existingRuleHash := listenerRole.Labels["role-policy-rules-hash"] - desiredRules := rulesForListenerRole([]string{autoscalingListener.Spec.EphemeralRunnerSetName}) - desiredRulesHash := hash.ComputeTemplateHash(&desiredRules) - if existingRuleHash != desiredRulesHash { - log.Info("Updating the listener role with the up-to-date rules") - return r.updateRoleForListener(ctx, listenerRole, desiredRules, desiredRulesHash, log) - } + // Make sure the listener role has the up-to-date rules + existingRuleHash := listenerRole.Labels["role-policy-rules-hash"] + desiredRules := rulesForListenerRole([]string{autoscalingListener.Spec.EphemeralRunnerSetName}) + desiredRulesHash := hash.ComputeTemplateHash(&desiredRules) + if existingRuleHash != desiredRulesHash { + log.Info("Updating the listener role with the up-to-date rules") + return r.updateRoleForListener(ctx, listenerRole, desiredRules, desiredRulesHash, log) + } - // Make sure the runner scale set listener role binding is created - listenerRoleBinding := new(rbacv1.RoleBinding) - if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Name}, listenerRoleBinding); err != nil { - if !kerrors.IsNotFound(err) { - log.Error(err, "Unable to get listener role binding", "namespace", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, "name", autoscalingListener.Name) - return ctrl.Result{}, err - } + // Make sure the runner scale set listener role binding is created + listenerRoleBinding := new(rbacv1.RoleBinding) + if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Name}, listenerRoleBinding); err != nil { + if !kerrors.IsNotFound(err) { + log.Error(err, "Unable to get listener role binding", "namespace", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, "name", autoscalingListener.Name) + return ctrl.Result{}, err + } - // Create a role binding for the listener pod in the AutoScalingRunnerSet namespace - log.Info("Creating a role binding for the service account and role") - return r.createRoleBindingForListener(ctx, autoscalingListener, listenerRole, serviceAccount, log) - } + // Create a role binding for the listener pod in the AutoScalingRunnerSet namespace + log.Info("Creating a role binding for the service account and role") + return r.createRoleBindingForListener(ctx, autoscalingListener, listenerRole, serviceAccount, log) + } + } else { + log.Info("Skipping listener role and role binding creation") + } // Create a secret containing proxy config if specified if autoscalingListener.Spec.Proxy != nil { @@ -356,54 +376,62 @@ func (r *AutoscalingListenerReconciler) cleanupResources(ctx context.Context, au logger.Info("Listener proxy secret is deleted") } - listenerRoleBinding := new(rbacv1.RoleBinding) - err = r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Name}, listenerRoleBinding) - switch { - case err == nil: - if listenerRoleBinding.DeletionTimestamp.IsZero() { - logger.Info("Deleting the listener role binding") - if err := r.Delete(ctx, listenerRoleBinding); err != nil { - return false, fmt.Errorf("failed to delete listener role binding: %w", err) - } - } - requeue = true - case !kerrors.IsNotFound(err): - return false, fmt.Errorf("failed to get listener role binding: %w", err) - } - logger.Info("Listener role binding is deleted") + if !ShouldSkipListenerRbacSetup() { + listenerRoleBinding := new(rbacv1.RoleBinding) + err = r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Name}, listenerRoleBinding) + switch { + case err == nil: + if listenerRoleBinding.DeletionTimestamp.IsZero() { + logger.Info("Deleting the listener role binding") + if err := r.Delete(ctx, listenerRoleBinding); err != nil { + return false, fmt.Errorf("failed to delete listener role binding: %w", err) + } + } + requeue = true + case !kerrors.IsNotFound(err): + return false, fmt.Errorf("failed to get listener role binding: %w", err) + } + logger.Info("Listener role binding is deleted") - listenerRole := new(rbacv1.Role) - err = r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Name}, listenerRole) - switch { - case err == nil: - if listenerRole.DeletionTimestamp.IsZero() { - logger.Info("Deleting the listener role") - if err := r.Delete(ctx, listenerRole); err != nil { - return false, fmt.Errorf("failed to delete listener role: %w", err) - } - } - requeue = true - case !kerrors.IsNotFound(err): - return false, fmt.Errorf("failed to get listener role: %w", err) - } - logger.Info("Listener role is deleted") + listenerRole := new(rbacv1.Role) + err = r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Name}, listenerRole) + switch { + case err == nil: + if listenerRole.DeletionTimestamp.IsZero() { + logger.Info("Deleting the listener role") + if err := r.Delete(ctx, listenerRole); err != nil { + return false, fmt.Errorf("failed to delete listener role: %w", err) + } + } + requeue = true + case !kerrors.IsNotFound(err): + return false, fmt.Errorf("failed to get listener role: %w", err) + } + logger.Info("Listener role is deleted") + } else { + logger.Info("Skipping listener role and role binding deletion") + } - logger.Info("Cleaning up the listener service account") - listenerSa := new(corev1.ServiceAccount) - err = r.Get(ctx, types.NamespacedName{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, listenerSa) - switch { - case err == nil: - if listenerSa.DeletionTimestamp.IsZero() { - logger.Info("Deleting the listener service account") - if err := r.Delete(ctx, listenerSa); err != nil { - return false, fmt.Errorf("failed to delete listener service account: %w", err) - } - } - requeue = true - case !kerrors.IsNotFound(err): - return false, fmt.Errorf("failed to get listener service account: %w", err) + if !ShouldSkipListenerSaCreation() { + logger.Info("Cleaning up the listener service account") + listenerSa := new(corev1.ServiceAccount) + err = r.Get(ctx, types.NamespacedName{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, listenerSa) + switch { + case err == nil: + if listenerSa.DeletionTimestamp.IsZero() { + logger.Info("Deleting the listener service account") + if err := r.Delete(ctx, listenerSa); err != nil { + return false, fmt.Errorf("failed to delete listener service account: %w", err) + } + } + requeue = true + case !kerrors.IsNotFound(err): + return false, fmt.Errorf("failed to get listener service account: %w", err) + } + logger.Info("Listener service account is deleted") + } else { + logger.Info("Skipping listener service account deletion") } - logger.Info("Listener service account is deleted") return requeue, nil } @@ -693,6 +721,8 @@ func (r *AutoscalingListenerReconciler) publishRunningListener(autoscalingListen // SetupWithManager sets up the controller with the Manager. func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error { + log := mgr.GetLogger() + labelBasedWatchFunc := func(_ context.Context, obj client.Object) []reconcile.Request { var requests []reconcile.Request labels := obj.GetLabels() @@ -716,12 +746,28 @@ func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error return requests } - return ctrl.NewControllerManagedBy(mgr). + controller := ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.AutoscalingListener{}). - Owns(&corev1.Pod{}). - Owns(&corev1.ServiceAccount{}). - Watches(&rbacv1.Role{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). - Watches(&rbacv1.RoleBinding{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). + Owns(&corev1.Pod{}) + + if !ShouldSkipListenerSaCreation() { + log.Info("Controller will own ServiceAccount for listeners") + controller = controller. + Owns(&corev1.ServiceAccount{}) + } else { + log.Info("Skipping Listener's ServiceAccount ownership") + } + + if !ShouldSkipListenerRbacSetup() { + log.Info("Controller will watch RBAC resources for listeners") + controller = controller. + Watches(&rbacv1.Role{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). + Watches(&rbacv1.RoleBinding{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)) + } else { + log.Info("Skipping Listener's RBAC watches") + } + + return controller. WithEventFilter(predicate.ResourceVersionChangedPredicate{}). Complete(r) } diff --git a/controllers/actions.github.com/autoscalingrunnerset_controller.go b/controllers/actions.github.com/autoscalingrunnerset_controller.go index 3509de91..51dd25ef 100644 --- a/controllers/actions.github.com/autoscalingrunnerset_controller.go +++ b/controllers/actions.github.com/autoscalingrunnerset_controller.go @@ -461,13 +461,25 @@ func (r *AutoscalingRunnerSetReconciler) removeFinalizersFromDependentResources( logger: logger, } - c.removeKubernetesModeRoleBindingFinalizer(ctx) - c.removeKubernetesModeRoleFinalizer(ctx) - c.removeKubernetesModeServiceAccountFinalizer(ctx) - c.removeNoPermissionServiceAccountFinalizer(ctx) + if !ShouldSkipListenerRbacSetup() { + c.removeKubernetesModeRoleBindingFinalizer(ctx) + c.removeKubernetesModeRoleFinalizer(ctx) + } else { + logger.Info("Skipping Kubernetes mode RBAC finalizers cleanup") + } + if !ShouldSkipListenerSaCreation() { + c.removeKubernetesModeServiceAccountFinalizer(ctx) + } else { + logger.Info("Skipping Kubernetes mode service account finalizer cleanup") + } c.removeGitHubSecretFinalizer(ctx) - c.removeManagerRoleBindingFinalizer(ctx) - c.removeManagerRoleFinalizer(ctx) + if !ShouldSkipRbac() { + c.removeNoPermissionServiceAccountFinalizer(ctx) + c.removeManagerRoleBindingFinalizer(ctx) + c.removeManagerRoleFinalizer(ctx) + } else { + logger.Info("Skipping RBAC finalizers cleanup") + } return c.Err() } diff --git a/controllers/actions.github.com/constants.go b/controllers/actions.github.com/constants.go index 588c95be..9a90f18b 100644 --- a/controllers/actions.github.com/constants.go +++ b/controllers/actions.github.com/constants.go @@ -82,4 +82,14 @@ const resourceOwnerKey = ".metadata.controller" const ( ReasonTooManyPodFailures = "TooManyPodFailures" ReasonInvalidPodFailure = "InvalidPod" + ReasonQuotaExceeded = "QuotaExceeded" +) + +// Rbac managing envs flags +const ( + SkipRbacSetupForController = "SKIP_RBAC_SETUP_FOR_CONTROLLER" + + RequireListenerSAProvided = "SKIP_LISTENERS_SA_CREATION" + + SkipRbacSetupForListeners = "SKIP_RBAC_SETUP_FOR_LISTENERS" ) diff --git a/controllers/actions.github.com/utils.go b/controllers/actions.github.com/utils.go index a77b24ba..3fa15cfd 100644 --- a/controllers/actions.github.com/utils.go +++ b/controllers/actions.github.com/utils.go @@ -2,6 +2,7 @@ package actionsgithubcom import ( "k8s.io/apimachinery/pkg/util/rand" + "os" ) func FilterLabels(labels map[string]string, filter string) map[string]string { @@ -25,3 +26,15 @@ func RandStringRunes(n int) string { } return string(b) } + +func ShouldSkipRbac() bool { + return os.Getenv(SkipRbacSetupForController) == "true" +} + +func ShouldSkipListenerRbacSetup() bool { + return os.Getenv(SkipRbacSetupForListeners) == "true" +} + +func ShouldSkipListenerSaCreation() bool { + return os.Getenv(RequireListenerSAProvided) == "true" +}