Refactor main.go and introduce make run-scaleset to be able to run manager locally (#2337)

2023-03-10 18:05:51 +01:00 · 2023-03-10 18:05:51 +01:00 · a5f98dea75
parent 1d24d3b00d
commit a5f98dea75
7 changed files with 106 additions and 119 deletions
--- a/9
+++ b/9
@ -92,9 +92,14 @@ manager: generate fmt vet
 run: generate fmt vet manifests
 	go run ./main.go

+run-scaleset: generate fmt vet
+	CONTROLLER_MANAGER_POD_NAMESPACE=default \
+	CONTROLLER_MANAGER_CONTAINER_IMAGE="${DOCKER_IMAGE_NAME}:${VERSION}" \
+	go run ./main.go --auto-scaling-runner-set-only
+
 # Install CRDs into a cluster
 install: manifests
-	kustomize build config/crd | kubectl apply -f -
+	kustomize build config/crd | kubectl apply --server-side -f -

 # Uninstall CRDs from a cluster
 uninstall: manifests
@ -103,7 +108,7 @@ uninstall: manifests
 # Deploy controller in the configured Kubernetes cluster in ~/.kube/config
 deploy: manifests
 	cd config/manager && kustomize edit set image controller=${DOCKER_IMAGE_NAME}:${VERSION}
-	kustomize build config/default | kubectl apply -f -
+	kustomize build config/default | kubectl apply --server-side -f -

 # Generate manifests e.g. CRD, RBAC etc.
 manifests: manifests-gen-crds chart-crds
--- a/charts/gha-runner-scale-set-controller/templates/deployment.yaml
+++ b/charts/gha-runner-scale-set-controller/templates/deployment.yaml
@ -54,10 +54,8 @@ spec:
        command:
        - "/manager"
        env:
-        - name: CONTROLLER_MANAGER_POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
+        - name: CONTROLLER_MANAGER_CONTAINER_IMAGE
+          value: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
        - name: CONTROLLER_MANAGER_POD_NAMESPACE
          valueFrom:
            fieldRef:
--- a/charts/gha-runner-scale-set-controller/tests/template_test.go
+++ b/charts/gha-runner-scale-set-controller/tests/template_test.go
@ -261,9 +261,11 @@ func TestTemplate_ControllerDeployment_Defaults(t *testing.T) {
 	assert.Nil(t, deployment.Spec.Template.Spec.Affinity)
 	assert.Len(t, deployment.Spec.Template.Spec.Tolerations, 0)

+	managerImage := "ghcr.io/actions/gha-runner-scale-set-controller:dev"
+
 	assert.Len(t, deployment.Spec.Template.Spec.Containers, 1)
 	assert.Equal(t, "manager", deployment.Spec.Template.Spec.Containers[0].Name)
-	assert.Equal(t, "ghcr.io/actions/gha-runner-scale-set-controller:dev", deployment.Spec.Template.Spec.Containers[0].Image)
+	assert.Equal(t, managerImage, deployment.Spec.Template.Spec.Containers[0].Image)
 	assert.Equal(t, corev1.PullIfNotPresent, deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy)

 	assert.Len(t, deployment.Spec.Template.Spec.Containers[0].Command, 1)
@ -274,8 +276,8 @@ func TestTemplate_ControllerDeployment_Defaults(t *testing.T) {
 	assert.Equal(t, "--log-level=debug", deployment.Spec.Template.Spec.Containers[0].Args[1])

 	assert.Len(t, deployment.Spec.Template.Spec.Containers[0].Env, 2)
-	assert.Equal(t, "CONTROLLER_MANAGER_POD_NAME", deployment.Spec.Template.Spec.Containers[0].Env[0].Name)
-	assert.Equal(t, "metadata.name", deployment.Spec.Template.Spec.Containers[0].Env[0].ValueFrom.FieldRef.FieldPath)
+	assert.Equal(t, "CONTROLLER_MANAGER_CONTAINER_IMAGE", deployment.Spec.Template.Spec.Containers[0].Env[0].Name)
+	assert.Equal(t, managerImage, deployment.Spec.Template.Spec.Containers[0].Env[0].Value)

 	assert.Equal(t, "CONTROLLER_MANAGER_POD_NAMESPACE", deployment.Spec.Template.Spec.Containers[0].Env[1].Name)
 	assert.Equal(t, "metadata.namespace", deployment.Spec.Template.Spec.Containers[0].Env[1].ValueFrom.FieldRef.FieldPath)
@ -375,9 +377,11 @@ func TestTemplate_ControllerDeployment_Customize(t *testing.T) {
 	assert.Len(t, deployment.Spec.Template.Spec.Tolerations, 1)
 	assert.Equal(t, "foo", deployment.Spec.Template.Spec.Tolerations[0].Key)

+	managerImage := "ghcr.io/actions/gha-runner-scale-set-controller:dev"
+
 	assert.Len(t, deployment.Spec.Template.Spec.Containers, 1)
 	assert.Equal(t, "manager", deployment.Spec.Template.Spec.Containers[0].Name)
-	assert.Equal(t, "ghcr.io/actions/gha-runner-scale-set-controller:dev", deployment.Spec.Template.Spec.Containers[0].Image)
+	assert.Equal(t, managerImage, deployment.Spec.Template.Spec.Containers[0].Image)
 	assert.Equal(t, corev1.PullAlways, deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy)

 	assert.Len(t, deployment.Spec.Template.Spec.Containers[0].Command, 1)
@ -389,8 +393,8 @@ func TestTemplate_ControllerDeployment_Customize(t *testing.T) {
 	assert.Equal(t, "--log-level=debug", deployment.Spec.Template.Spec.Containers[0].Args[2])

 	assert.Len(t, deployment.Spec.Template.Spec.Containers[0].Env, 2)
-	assert.Equal(t, "CONTROLLER_MANAGER_POD_NAME", deployment.Spec.Template.Spec.Containers[0].Env[0].Name)
-	assert.Equal(t, "metadata.name", deployment.Spec.Template.Spec.Containers[0].Env[0].ValueFrom.FieldRef.FieldPath)
+	assert.Equal(t, "CONTROLLER_MANAGER_CONTAINER_IMAGE", deployment.Spec.Template.Spec.Containers[0].Env[0].Name)
+	assert.Equal(t, managerImage, deployment.Spec.Template.Spec.Containers[0].Env[0].Value)

 	assert.Equal(t, "CONTROLLER_MANAGER_POD_NAMESPACE", deployment.Spec.Template.Spec.Containers[0].Env[1].Name)
 	assert.Equal(t, "metadata.namespace", deployment.Spec.Template.Spec.Containers[0].Env[1].ValueFrom.FieldRef.FieldPath)
--- a/config/manager/env-replacement.yaml
+++ b/config/manager/env-replacement.yaml
@ -0,0 +1,10 @@
+source:
+  kind: Deployment
+  name: controller-manager
+  fieldPath: spec.template.spec.containers.[name=manager].image
+targets:
+- select:
+    kind: Deployment
+    name: controller-manager
+  fieldPaths:
+  - spec.template.spec.containers.[name=manager].env.[name=CONTROLLER_MANAGER_CONTAINER_IMAGE].value
--- a/config/manager/kustomization.yaml
+++ b/config/manager/kustomization.yaml
@ -6,3 +6,6 @@ images:
 - name: controller
  newName: summerwind/actions-runner-controller
  newTag: dev
+
+replacements:
+- path: env-replacement.yaml
--- a/config/manager/manager.yaml
+++ b/config/manager/manager.yaml
@ -50,10 +50,8 @@ spec:
              optional: true
        - name: GITHUB_APP_PRIVATE_KEY
          value: /etc/actions-runner-controller/github_app_private_key
-        - name: CONTROLLER_MANAGER_POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
+        - name: CONTROLLER_MANAGER_CONTAINER_IMAGE
+          value: CONTROLLER_MANAGER_CONTAINER_IMAGE
        - name: CONTROLLER_MANAGER_POD_NAMESPACE
          valueFrom:
            fieldRef:
--- a/main.go
+++ b/main.go
@ -17,7 +17,6 @@ limitations under the License.
 package main

 import (
-	"context"
 	"flag"
 	"fmt"
 	"os"
@ -33,9 +32,7 @@ import (
 	"github.com/actions/actions-runner-controller/github/actions"
 	"github.com/actions/actions-runner-controller/logging"
 	"github.com/kelseyhightower/envconfig"
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	ctrl "sigs.k8s.io/controller-runtime"
@ -47,9 +44,7 @@ const (
 	defaultDockerImage = "docker:dind"
 )

-var (
-	scheme = runtime.NewScheme()
-)
+var scheme = runtime.NewScheme()

 func init() {
 	_ = clientgoscheme.AddToScheme(scheme)
@ -68,6 +63,7 @@ func (i *stringSlice) Set(value string) error {
 	*i = append(*i, value)
 	return nil
 }
+
 func main() {
 	var (
 		err      error
@ -170,17 +166,69 @@ func main() {
 		os.Exit(1)
 	}

-	multiClient := actionssummerwindnet.NewMultiGitHubClient(
-		mgr.GetClient(),
-		ghClient,
-	)
+	if autoScalingRunnerSetOnly {
+		managerImage := os.Getenv("CONTROLLER_MANAGER_CONTAINER_IMAGE")
+		if managerImage == "" {
+			log.Error(err, "unable to obtain listener image")
+			os.Exit(1)
+		}
+		managerNamespace := os.Getenv("CONTROLLER_MANAGER_POD_NAMESPACE")
+		if managerNamespace == "" {
+			log.Error(err, "unable to obtain manager pod namespace")
+			os.Exit(1)
+		}

 		actionsMultiClient := actions.NewMultiClient(
 			"actions-runner-controller/"+build.Version,
 			log.WithName("actions-clients"),
 		)

-	if !autoScalingRunnerSetOnly {
+		if err = (&actionsgithubcom.AutoscalingRunnerSetReconciler{
+			Client:                             mgr.GetClient(),
+			Log:                                log.WithName("AutoscalingRunnerSet"),
+			Scheme:                             mgr.GetScheme(),
+			ControllerNamespace:                managerNamespace,
+			DefaultRunnerScaleSetListenerImage: managerImage,
+			ActionsClient:                      actionsMultiClient,
+			DefaultRunnerScaleSetListenerImagePullSecrets: autoScalerImagePullSecrets,
+		}).SetupWithManager(mgr); err != nil {
+			log.Error(err, "unable to create controller", "controller", "AutoscalingRunnerSet")
+			os.Exit(1)
+		}
+
+		if err = (&actionsgithubcom.EphemeralRunnerReconciler{
+			Client:        mgr.GetClient(),
+			Log:           log.WithName("EphemeralRunner"),
+			Scheme:        mgr.GetScheme(),
+			ActionsClient: actionsMultiClient,
+		}).SetupWithManager(mgr); err != nil {
+			log.Error(err, "unable to create controller", "controller", "EphemeralRunner")
+			os.Exit(1)
+		}
+
+		if err = (&actionsgithubcom.EphemeralRunnerSetReconciler{
+			Client:        mgr.GetClient(),
+			Log:           log.WithName("EphemeralRunnerSet"),
+			Scheme:        mgr.GetScheme(),
+			ActionsClient: actionsMultiClient,
+		}).SetupWithManager(mgr); err != nil {
+			log.Error(err, "unable to create controller", "controller", "EphemeralRunnerSet")
+			os.Exit(1)
+		}
+		if err = (&actionsgithubcom.AutoscalingListenerReconciler{
+			Client: mgr.GetClient(),
+			Log:    log.WithName("AutoscalingListener"),
+			Scheme: mgr.GetScheme(),
+		}).SetupWithManager(mgr); err != nil {
+			log.Error(err, "unable to create controller", "controller", "AutoscalingListener")
+			os.Exit(1)
+		}
+	} else {
+		multiClient := actionssummerwindnet.NewMultiGitHubClient(
+			mgr.GetClient(),
+			ghClient,
+		)
+
 		runnerReconciler := &actionssummerwindnet.RunnerReconciler{
 			Client:                    mgr.GetClient(),
 			Log:                       log.WithName("runner"),
@ -314,86 +362,6 @@ func main() {
 				log.Error(err, "unable to create webhook", "webhook", "RunnerReplicaSet")
 				os.Exit(1)
 			}
-		}
-	}
-
-	// We use this environment avariable to turn on the ScaleSet related controllers.
-	// Otherwise ARC's legacy chart is unable to deploy a working ARC controller-manager pod,
-	// due to that the chart does not contain new actions.* CRDs while ARC requires those CRDs.
-	//
-	// We might have used a more explicitly named environment variable for this,
-	// e.g. "CONTROLLER_MANAGER_ENABLE_SCALE_SET" to explicitly enable the new controllers,
-	// or "CONTROLLER_MANAGER_DISABLE_SCALE_SET" to explicitly disable the new controllers.
-	// However, doing so would affect either private ARC testers or current ARC users
-	// who run ARC without those variabls.
-	mgrPodName := os.Getenv("CONTROLLER_MANAGER_POD_NAME")
-	if mgrPodName != "" {
-		mgrPodNamespace := os.Getenv("CONTROLLER_MANAGER_POD_NAMESPACE")
-		var mgrPod corev1.Pod
-		err = mgr.GetAPIReader().Get(context.Background(), types.NamespacedName{Namespace: mgrPodNamespace, Name: mgrPodName}, &mgrPod)
-		if err != nil {
-			log.Error(err, fmt.Sprintf("unable to obtain manager pod: %s (%s)", mgrPodName, mgrPodNamespace))
-			os.Exit(1)
-		}
-
-		var mgrContainer *corev1.Container
-		for _, container := range mgrPod.Spec.Containers {
-			if container.Name == "manager" {
-				mgrContainer = &container
-				break
-			}
-		}
-
-		if mgrContainer != nil {
-			log.Info("Detected manager container", "image", mgrContainer.Image)
-		} else {
-			log.Error(err, "unable to obtain manager container image")
-			os.Exit(1)
-		}
-		if err = (&actionsgithubcom.AutoscalingRunnerSetReconciler{
-			Client:                             mgr.GetClient(),
-			Log:                                log.WithName("AutoscalingRunnerSet"),
-			Scheme:                             mgr.GetScheme(),
-			ControllerNamespace:                mgrPodNamespace,
-			DefaultRunnerScaleSetListenerImage: mgrContainer.Image,
-			ActionsClient:                      actionsMultiClient,
-			DefaultRunnerScaleSetListenerImagePullSecrets: autoScalerImagePullSecrets,
-		}).SetupWithManager(mgr); err != nil {
-			log.Error(err, "unable to create controller", "controller", "AutoscalingRunnerSet")
-			os.Exit(1)
-		}
-
-		if err = (&actionsgithubcom.EphemeralRunnerReconciler{
-			Client:        mgr.GetClient(),
-			Log:           log.WithName("EphemeralRunner"),
-			Scheme:        mgr.GetScheme(),
-			ActionsClient: actionsMultiClient,
-		}).SetupWithManager(mgr); err != nil {
-			log.Error(err, "unable to create controller", "controller", "EphemeralRunner")
-			os.Exit(1)
-		}
-
-		if err = (&actionsgithubcom.EphemeralRunnerSetReconciler{
-			Client:        mgr.GetClient(),
-			Log:           log.WithName("EphemeralRunnerSet"),
-			Scheme:        mgr.GetScheme(),
-			ActionsClient: actionsMultiClient,
-		}).SetupWithManager(mgr); err != nil {
-			log.Error(err, "unable to create controller", "controller", "EphemeralRunnerSet")
-			os.Exit(1)
-		}
-		if err = (&actionsgithubcom.AutoscalingListenerReconciler{
-			Client: mgr.GetClient(),
-			Log:    log.WithName("AutoscalingListener"),
-			Scheme: mgr.GetScheme(),
-		}).SetupWithManager(mgr); err != nil {
-			log.Error(err, "unable to create controller", "controller", "AutoscalingListener")
-			os.Exit(1)
-		}
-		// +kubebuilder:scaffold:builder
-	}
-
-	if !disableAdmissionWebhook && !autoScalingRunnerSetOnly {
 			injector := &actionssummerwindnet.PodRunnerTokenInjector{
 				Client:       mgr.GetClient(),
 				GitHubClient: multiClient,
@ -404,6 +372,7 @@ func main() {
 				os.Exit(1)
 			}
 		}
+	}

 	log.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {