diff --git a/Makefile b/Makefile index e42d2ae7..19cef42e 100644 --- a/Makefile +++ b/Makefile @@ -92,9 +92,14 @@ manager: generate fmt vet run: generate fmt vet manifests go run ./main.go +run-scaleset: generate fmt vet + CONTROLLER_MANAGER_POD_NAMESPACE=default \ + CONTROLLER_MANAGER_CONTAINER_IMAGE="${DOCKER_IMAGE_NAME}:${VERSION}" \ + go run ./main.go --auto-scaling-runner-set-only + # Install CRDs into a cluster install: manifests - kustomize build config/crd | kubectl apply -f - + kustomize build config/crd | kubectl apply --server-side -f - # Uninstall CRDs from a cluster uninstall: manifests @@ -103,7 +108,7 @@ uninstall: manifests # Deploy controller in the configured Kubernetes cluster in ~/.kube/config deploy: manifests cd config/manager && kustomize edit set image controller=${DOCKER_IMAGE_NAME}:${VERSION} - kustomize build config/default | kubectl apply -f - + kustomize build config/default | kubectl apply --server-side -f - # Generate manifests e.g. CRD, RBAC etc. manifests: manifests-gen-crds chart-crds diff --git a/charts/gha-runner-scale-set-controller/templates/deployment.yaml b/charts/gha-runner-scale-set-controller/templates/deployment.yaml index a35dc784..cae43f4e 100644 --- a/charts/gha-runner-scale-set-controller/templates/deployment.yaml +++ b/charts/gha-runner-scale-set-controller/templates/deployment.yaml @@ -54,10 +54,8 @@ spec: command: - "/manager" env: - - name: CONTROLLER_MANAGER_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name + - name: CONTROLLER_MANAGER_CONTAINER_IMAGE + value: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - name: CONTROLLER_MANAGER_POD_NAMESPACE valueFrom: fieldRef: @@ -98,4 +96,4 @@ spec: {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/charts/gha-runner-scale-set-controller/tests/template_test.go b/charts/gha-runner-scale-set-controller/tests/template_test.go index 00ab04b5..fe4bf020 100644 --- a/charts/gha-runner-scale-set-controller/tests/template_test.go +++ b/charts/gha-runner-scale-set-controller/tests/template_test.go @@ -261,9 +261,11 @@ func TestTemplate_ControllerDeployment_Defaults(t *testing.T) { assert.Nil(t, deployment.Spec.Template.Spec.Affinity) assert.Len(t, deployment.Spec.Template.Spec.Tolerations, 0) + managerImage := "ghcr.io/actions/gha-runner-scale-set-controller:dev" + assert.Len(t, deployment.Spec.Template.Spec.Containers, 1) assert.Equal(t, "manager", deployment.Spec.Template.Spec.Containers[0].Name) - assert.Equal(t, "ghcr.io/actions/gha-runner-scale-set-controller:dev", deployment.Spec.Template.Spec.Containers[0].Image) + assert.Equal(t, managerImage, deployment.Spec.Template.Spec.Containers[0].Image) assert.Equal(t, corev1.PullIfNotPresent, deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy) assert.Len(t, deployment.Spec.Template.Spec.Containers[0].Command, 1) @@ -274,8 +276,8 @@ func TestTemplate_ControllerDeployment_Defaults(t *testing.T) { assert.Equal(t, "--log-level=debug", deployment.Spec.Template.Spec.Containers[0].Args[1]) assert.Len(t, deployment.Spec.Template.Spec.Containers[0].Env, 2) - assert.Equal(t, "CONTROLLER_MANAGER_POD_NAME", deployment.Spec.Template.Spec.Containers[0].Env[0].Name) - assert.Equal(t, "metadata.name", deployment.Spec.Template.Spec.Containers[0].Env[0].ValueFrom.FieldRef.FieldPath) + assert.Equal(t, "CONTROLLER_MANAGER_CONTAINER_IMAGE", deployment.Spec.Template.Spec.Containers[0].Env[0].Name) + assert.Equal(t, managerImage, deployment.Spec.Template.Spec.Containers[0].Env[0].Value) assert.Equal(t, "CONTROLLER_MANAGER_POD_NAMESPACE", deployment.Spec.Template.Spec.Containers[0].Env[1].Name) assert.Equal(t, "metadata.namespace", deployment.Spec.Template.Spec.Containers[0].Env[1].ValueFrom.FieldRef.FieldPath) @@ -375,9 +377,11 @@ func TestTemplate_ControllerDeployment_Customize(t *testing.T) { assert.Len(t, deployment.Spec.Template.Spec.Tolerations, 1) assert.Equal(t, "foo", deployment.Spec.Template.Spec.Tolerations[0].Key) + managerImage := "ghcr.io/actions/gha-runner-scale-set-controller:dev" + assert.Len(t, deployment.Spec.Template.Spec.Containers, 1) assert.Equal(t, "manager", deployment.Spec.Template.Spec.Containers[0].Name) - assert.Equal(t, "ghcr.io/actions/gha-runner-scale-set-controller:dev", deployment.Spec.Template.Spec.Containers[0].Image) + assert.Equal(t, managerImage, deployment.Spec.Template.Spec.Containers[0].Image) assert.Equal(t, corev1.PullAlways, deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy) assert.Len(t, deployment.Spec.Template.Spec.Containers[0].Command, 1) @@ -389,8 +393,8 @@ func TestTemplate_ControllerDeployment_Customize(t *testing.T) { assert.Equal(t, "--log-level=debug", deployment.Spec.Template.Spec.Containers[0].Args[2]) assert.Len(t, deployment.Spec.Template.Spec.Containers[0].Env, 2) - assert.Equal(t, "CONTROLLER_MANAGER_POD_NAME", deployment.Spec.Template.Spec.Containers[0].Env[0].Name) - assert.Equal(t, "metadata.name", deployment.Spec.Template.Spec.Containers[0].Env[0].ValueFrom.FieldRef.FieldPath) + assert.Equal(t, "CONTROLLER_MANAGER_CONTAINER_IMAGE", deployment.Spec.Template.Spec.Containers[0].Env[0].Name) + assert.Equal(t, managerImage, deployment.Spec.Template.Spec.Containers[0].Env[0].Value) assert.Equal(t, "CONTROLLER_MANAGER_POD_NAMESPACE", deployment.Spec.Template.Spec.Containers[0].Env[1].Name) assert.Equal(t, "metadata.namespace", deployment.Spec.Template.Spec.Containers[0].Env[1].ValueFrom.FieldRef.FieldPath) diff --git a/config/manager/env-replacement.yaml b/config/manager/env-replacement.yaml new file mode 100644 index 00000000..7caef2cf --- /dev/null +++ b/config/manager/env-replacement.yaml @@ -0,0 +1,10 @@ +source: + kind: Deployment + name: controller-manager + fieldPath: spec.template.spec.containers.[name=manager].image +targets: +- select: + kind: Deployment + name: controller-manager + fieldPaths: + - spec.template.spec.containers.[name=manager].env.[name=CONTROLLER_MANAGER_CONTAINER_IMAGE].value diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index e7063a8d..6a60f8b3 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -6,3 +6,6 @@ images: - name: controller newName: summerwind/actions-runner-controller newTag: dev + +replacements: +- path: env-replacement.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d16d1698..f90df347 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -50,10 +50,8 @@ spec: optional: true - name: GITHUB_APP_PRIVATE_KEY value: /etc/actions-runner-controller/github_app_private_key - - name: CONTROLLER_MANAGER_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name + - name: CONTROLLER_MANAGER_CONTAINER_IMAGE + value: CONTROLLER_MANAGER_CONTAINER_IMAGE - name: CONTROLLER_MANAGER_POD_NAMESPACE valueFrom: fieldRef: diff --git a/main.go b/main.go index 6cdd4ef7..fbf8b3ff 100644 --- a/main.go +++ b/main.go @@ -17,7 +17,6 @@ limitations under the License. package main import ( - "context" "flag" "fmt" "os" @@ -33,9 +32,7 @@ import ( "github.com/actions/actions-runner-controller/github/actions" "github.com/actions/actions-runner-controller/logging" "github.com/kelseyhightower/envconfig" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" @@ -47,9 +44,7 @@ const ( defaultDockerImage = "docker:dind" ) -var ( - scheme = runtime.NewScheme() -) +var scheme = runtime.NewScheme() func init() { _ = clientgoscheme.AddToScheme(scheme) @@ -68,6 +63,7 @@ func (i *stringSlice) Set(value string) error { *i = append(*i, value) return nil } + func main() { var ( err error @@ -170,17 +166,69 @@ func main() { os.Exit(1) } - multiClient := actionssummerwindnet.NewMultiGitHubClient( - mgr.GetClient(), - ghClient, - ) + if autoScalingRunnerSetOnly { + managerImage := os.Getenv("CONTROLLER_MANAGER_CONTAINER_IMAGE") + if managerImage == "" { + log.Error(err, "unable to obtain listener image") + os.Exit(1) + } + managerNamespace := os.Getenv("CONTROLLER_MANAGER_POD_NAMESPACE") + if managerNamespace == "" { + log.Error(err, "unable to obtain manager pod namespace") + os.Exit(1) + } - actionsMultiClient := actions.NewMultiClient( - "actions-runner-controller/"+build.Version, - log.WithName("actions-clients"), - ) + actionsMultiClient := actions.NewMultiClient( + "actions-runner-controller/"+build.Version, + log.WithName("actions-clients"), + ) + + if err = (&actionsgithubcom.AutoscalingRunnerSetReconciler{ + Client: mgr.GetClient(), + Log: log.WithName("AutoscalingRunnerSet"), + Scheme: mgr.GetScheme(), + ControllerNamespace: managerNamespace, + DefaultRunnerScaleSetListenerImage: managerImage, + ActionsClient: actionsMultiClient, + DefaultRunnerScaleSetListenerImagePullSecrets: autoScalerImagePullSecrets, + }).SetupWithManager(mgr); err != nil { + log.Error(err, "unable to create controller", "controller", "AutoscalingRunnerSet") + os.Exit(1) + } + + if err = (&actionsgithubcom.EphemeralRunnerReconciler{ + Client: mgr.GetClient(), + Log: log.WithName("EphemeralRunner"), + Scheme: mgr.GetScheme(), + ActionsClient: actionsMultiClient, + }).SetupWithManager(mgr); err != nil { + log.Error(err, "unable to create controller", "controller", "EphemeralRunner") + os.Exit(1) + } + + if err = (&actionsgithubcom.EphemeralRunnerSetReconciler{ + Client: mgr.GetClient(), + Log: log.WithName("EphemeralRunnerSet"), + Scheme: mgr.GetScheme(), + ActionsClient: actionsMultiClient, + }).SetupWithManager(mgr); err != nil { + log.Error(err, "unable to create controller", "controller", "EphemeralRunnerSet") + os.Exit(1) + } + if err = (&actionsgithubcom.AutoscalingListenerReconciler{ + Client: mgr.GetClient(), + Log: log.WithName("AutoscalingListener"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + log.Error(err, "unable to create controller", "controller", "AutoscalingListener") + os.Exit(1) + } + } else { + multiClient := actionssummerwindnet.NewMultiGitHubClient( + mgr.GetClient(), + ghClient, + ) - if !autoScalingRunnerSetOnly { runnerReconciler := &actionssummerwindnet.RunnerReconciler{ Client: mgr.GetClient(), Log: log.WithName("runner"), @@ -314,94 +362,15 @@ func main() { log.Error(err, "unable to create webhook", "webhook", "RunnerReplicaSet") os.Exit(1) } - } - } - - // We use this environment avariable to turn on the ScaleSet related controllers. - // Otherwise ARC's legacy chart is unable to deploy a working ARC controller-manager pod, - // due to that the chart does not contain new actions.* CRDs while ARC requires those CRDs. - // - // We might have used a more explicitly named environment variable for this, - // e.g. "CONTROLLER_MANAGER_ENABLE_SCALE_SET" to explicitly enable the new controllers, - // or "CONTROLLER_MANAGER_DISABLE_SCALE_SET" to explicitly disable the new controllers. - // However, doing so would affect either private ARC testers or current ARC users - // who run ARC without those variabls. - mgrPodName := os.Getenv("CONTROLLER_MANAGER_POD_NAME") - if mgrPodName != "" { - mgrPodNamespace := os.Getenv("CONTROLLER_MANAGER_POD_NAMESPACE") - var mgrPod corev1.Pod - err = mgr.GetAPIReader().Get(context.Background(), types.NamespacedName{Namespace: mgrPodNamespace, Name: mgrPodName}, &mgrPod) - if err != nil { - log.Error(err, fmt.Sprintf("unable to obtain manager pod: %s (%s)", mgrPodName, mgrPodNamespace)) - os.Exit(1) - } - - var mgrContainer *corev1.Container - for _, container := range mgrPod.Spec.Containers { - if container.Name == "manager" { - mgrContainer = &container - break + injector := &actionssummerwindnet.PodRunnerTokenInjector{ + Client: mgr.GetClient(), + GitHubClient: multiClient, + Log: ctrl.Log.WithName("webhook").WithName("PodRunnerTokenInjector"), + } + if err = injector.SetupWithManager(mgr); err != nil { + log.Error(err, "unable to create webhook server", "webhook", "PodRunnerTokenInjector") + os.Exit(1) } - } - - if mgrContainer != nil { - log.Info("Detected manager container", "image", mgrContainer.Image) - } else { - log.Error(err, "unable to obtain manager container image") - os.Exit(1) - } - if err = (&actionsgithubcom.AutoscalingRunnerSetReconciler{ - Client: mgr.GetClient(), - Log: log.WithName("AutoscalingRunnerSet"), - Scheme: mgr.GetScheme(), - ControllerNamespace: mgrPodNamespace, - DefaultRunnerScaleSetListenerImage: mgrContainer.Image, - ActionsClient: actionsMultiClient, - DefaultRunnerScaleSetListenerImagePullSecrets: autoScalerImagePullSecrets, - }).SetupWithManager(mgr); err != nil { - log.Error(err, "unable to create controller", "controller", "AutoscalingRunnerSet") - os.Exit(1) - } - - if err = (&actionsgithubcom.EphemeralRunnerReconciler{ - Client: mgr.GetClient(), - Log: log.WithName("EphemeralRunner"), - Scheme: mgr.GetScheme(), - ActionsClient: actionsMultiClient, - }).SetupWithManager(mgr); err != nil { - log.Error(err, "unable to create controller", "controller", "EphemeralRunner") - os.Exit(1) - } - - if err = (&actionsgithubcom.EphemeralRunnerSetReconciler{ - Client: mgr.GetClient(), - Log: log.WithName("EphemeralRunnerSet"), - Scheme: mgr.GetScheme(), - ActionsClient: actionsMultiClient, - }).SetupWithManager(mgr); err != nil { - log.Error(err, "unable to create controller", "controller", "EphemeralRunnerSet") - os.Exit(1) - } - if err = (&actionsgithubcom.AutoscalingListenerReconciler{ - Client: mgr.GetClient(), - Log: log.WithName("AutoscalingListener"), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - log.Error(err, "unable to create controller", "controller", "AutoscalingListener") - os.Exit(1) - } - // +kubebuilder:scaffold:builder - } - - if !disableAdmissionWebhook && !autoScalingRunnerSetOnly { - injector := &actionssummerwindnet.PodRunnerTokenInjector{ - Client: mgr.GetClient(), - GitHubClient: multiClient, - Log: ctrl.Log.WithName("webhook").WithName("PodRunnerTokenInjector"), - } - if err = injector.SetupWithManager(mgr); err != nil { - log.Error(err, "unable to create webhook server", "webhook", "PodRunnerTokenInjector") - os.Exit(1) } }