Helm chart react changes for the new runner image. (#2348)

adrs/2022-10-17-runner-image.md

@@ -3,6 +3,19 @@
 
 **Status**: Done
 
+# Breaking Changes
+
+We aim to provide an similar experience (as close as possible) between self-hosted and GitHub-hosted runners. To achieve this, we are making the following changes to align our self-hosted runner container image with the Ubuntu runners managed by GitHub.
+Here are the changes:
+- We created a USER `runner(1001)` and a GROUP `docker(123)`
+- `sudo` has been on the image and the `runner` will be a passwordless sudoer.
+- The runner binary was placed placed under `/home/runner/` and launched using `/home/runner/run.sh`
+- The runner's work directory is `/home/runner/_work`
+- `$HOME` will point to `/home/runner`
+- The container image user will be the `runner(1001)`
+
+The latest Dockerfile can be found at: https://github.com/actions/runner/blob/main/images/Dockerfile
+
 # Context
 
 user can bring their own runner images, the contract we have are:

charts/gha-runner-scale-set/templates/_helpers.tpl

@@ -83,10 +83,10 @@ imagePullSecrets:
   {{ $val.imagePullSecrets | toYaml -}}
 {{- end }}
 command: ["cp"]
-args: ["-r", "-v", "/actions-runner/externals/.", "/actions-runner/tmpDir/"]
+args: ["-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"]
 volumeMounts:
   - name: dind-externals
-    mountPath: /actions-runner/tmpDir
+    mountPath: /home/runner/tmpDir
 {{- end }}
 {{- end }}
 {{- end }}
@@ -97,11 +97,11 @@ securityContext:
   privileged: true
 volumeMounts:
   - name: work
-    mountPath: /actions-runner/_work
+    mountPath: /home/runner/_work
   - name: dind-cert
     mountPath: /certs/client
   - name: dind-externals
-    mountPath: /actions-runner/externals
+    mountPath: /home/runner/externals
 {{- end }}
 
 {{- define "gha-runner-scale-set.dind-volume" -}}
@@ -125,12 +125,7 @@ volumeMounts:
   {{- range $i, $volume := .Values.template.spec.volumes }}
     {{- if eq $volume.name "work" }}
       {{- $createWorkVolume = 0 -}}
-- name: work
+- {{ $volume | toYaml | nindent 2 }}
-      {{- range $key, $val := $volume }}
-        {{- if ne $key "name" }}
-  {{ $key }}: {{ $val }}
-        {{- end }}
-      {{- end }}
     {{- end }}
   {{- end }}
   {{- if eq $createWorkVolume 1 }}
@@ -144,12 +139,7 @@ volumeMounts:
   {{- range $i, $volume := .Values.template.spec.volumes }}
     {{- if eq $volume.name "work" }}
       {{- $createWorkVolume = 0 -}}
-- name: work
+- {{ $volume | toYaml | nindent 2 }}
-      {{- range $key, $val := $volume }}
-        {{- if ne $key "name" }}
-  {{ $key }}: {{ $val }}
-        {{- end }}
-      {{- end }}
     {{- end }}
   {{- end }}
   {{- if eq $createWorkVolume 1 }}
@@ -282,7 +272,7 @@ volumeMounts:
     {{- end }}
     {{- if $mountWork }}
   - name: work
-    mountPath: /actions-runner/_work
+    mountPath: /home/runner/_work
     {{- end }}
     {{- if $mountDindCert }}
   - name: dind-cert
@@ -344,7 +334,7 @@ env:
     {{- end }}
     {{- if $setContainerHooks }}
   - name: ACTIONS_RUNNER_CONTAINER_HOOKS
-    value: /actions-runner/k8s/index.js
+    value: /home/runner/k8s/index.js
     {{- end }}
     {{- if $setPodName }}
   - name: ACTIONS_RUNNER_POD_NAME
@@ -388,7 +378,7 @@ volumeMounts:
     {{- end }}
     {{- if $mountWork }}
   - name: work
-    mountPath: /actions-runner/_work
+    mountPath: /home/runner/_work
     {{- end }}
     {{- if $mountGitHubServerTLS }}
   - name: github-server-tls-cert

charts/gha-runner-scale-set/templates/autoscalingrunnerset.yaml

@@ -124,8 +124,13 @@ spec:
         {{- if eq .Values.containerMode.type "dind" }}
           {{- include "gha-runner-scale-set.dind-volume" . | nindent 6 }}
           {{- include "gha-runner-scale-set.dind-work-volume" . | nindent 6 }}
+          {{- include "gha-runner-scale-set.non-work-volumes" . | nindent 6 }}
         {{- else if eq .Values.containerMode.type "kubernetes" }}
           {{- include "gha-runner-scale-set.kubernetes-mode-work-volume" . | nindent 6 }}
+          {{- include "gha-runner-scale-set.non-work-volumes" . | nindent 6 }}
+        {{- else }}
+          {{- with .Values.template.spec.volumes }}
+        {{- toYaml . | nindent 6 }}
+          {{- end }}
         {{- end }}
-        {{- include "gha-runner-scale-set.non-work-volumes" . | nindent 6 }}
       {{- end }}

charts/gha-runner-scale-set/tests/template_test.go

@@ -591,6 +591,98 @@ func TestTemplateRenderedAutoScalingRunnerSet_MinMaxRunners_FromValuesFile(t *te
 	assert.Equal(t, 10, *ars.Spec.MaxRunners, "MaxRunners should be 10")
 }
 
+func TestTemplateRenderedAutoScalingRunnerSet_ExtraVolumes(t *testing.T) {
+	t.Parallel()
+
+	// Path to the helm chart we will test
+	helmChartPath, err := filepath.Abs("../../gha-runner-scale-set")
+	require.NoError(t, err)
+
+	testValuesPath, err := filepath.Abs("../tests/values_extra_volumes.yaml")
+	require.NoError(t, err)
+
+	releaseName := "test-runners"
+	namespaceName := "test-" + strings.ToLower(random.UniqueId())
+
+	options := &helm.Options{
+		ValuesFiles:    []string{testValuesPath},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+	}
+
+	output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/autoscalingrunnerset.yaml"})
+
+	var ars v1alpha1.AutoscalingRunnerSet
+	helm.UnmarshalK8SYaml(t, output, &ars)
+
+	assert.Len(t, ars.Spec.Template.Spec.Volumes, 3, "Volumes should be 3")
+	assert.Equal(t, "foo", ars.Spec.Template.Spec.Volumes[0].Name, "Volume name should be foo")
+	assert.Equal(t, "bar", ars.Spec.Template.Spec.Volumes[1].Name, "Volume name should be bar")
+	assert.Equal(t, "work", ars.Spec.Template.Spec.Volumes[2].Name, "Volume name should be work")
+	assert.Equal(t, "/data", ars.Spec.Template.Spec.Volumes[2].HostPath.Path, "Volume host path should be /data")
+}
+
+func TestTemplateRenderedAutoScalingRunnerSet_DinD_ExtraVolumes(t *testing.T) {
+	t.Parallel()
+
+	// Path to the helm chart we will test
+	helmChartPath, err := filepath.Abs("../../gha-runner-scale-set")
+	require.NoError(t, err)
+
+	testValuesPath, err := filepath.Abs("../tests/values_dind_extra_volumes.yaml")
+	require.NoError(t, err)
+
+	releaseName := "test-runners"
+	namespaceName := "test-" + strings.ToLower(random.UniqueId())
+
+	options := &helm.Options{
+		ValuesFiles:    []string{testValuesPath},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+	}
+
+	output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/autoscalingrunnerset.yaml"})
+
+	var ars v1alpha1.AutoscalingRunnerSet
+	helm.UnmarshalK8SYaml(t, output, &ars)
+
+	assert.Len(t, ars.Spec.Template.Spec.Volumes, 5, "Volumes should be 5")
+	assert.Equal(t, "dind-cert", ars.Spec.Template.Spec.Volumes[0].Name, "Volume name should be dind-cert")
+	assert.Equal(t, "dind-externals", ars.Spec.Template.Spec.Volumes[1].Name, "Volume name should be dind-externals")
+	assert.Equal(t, "work", ars.Spec.Template.Spec.Volumes[2].Name, "Volume name should be work")
+	assert.Equal(t, "/data", ars.Spec.Template.Spec.Volumes[2].HostPath.Path, "Volume host path should be /data")
+	assert.Equal(t, "foo", ars.Spec.Template.Spec.Volumes[3].Name, "Volume name should be foo")
+	assert.Equal(t, "bar", ars.Spec.Template.Spec.Volumes[4].Name, "Volume name should be bar")
+}
+
+func TestTemplateRenderedAutoScalingRunnerSet_K8S_ExtraVolumes(t *testing.T) {
+	t.Parallel()
+
+	// Path to the helm chart we will test
+	helmChartPath, err := filepath.Abs("../../gha-runner-scale-set")
+	require.NoError(t, err)
+
+	testValuesPath, err := filepath.Abs("../tests/values_k8s_extra_volumes.yaml")
+	require.NoError(t, err)
+
+	releaseName := "test-runners"
+	namespaceName := "test-" + strings.ToLower(random.UniqueId())
+
+	options := &helm.Options{
+		ValuesFiles:    []string{testValuesPath},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+	}
+
+	output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/autoscalingrunnerset.yaml"})
+
+	var ars v1alpha1.AutoscalingRunnerSet
+	helm.UnmarshalK8SYaml(t, output, &ars)
+
+	assert.Len(t, ars.Spec.Template.Spec.Volumes, 3, "Volumes should be 3")
+	assert.Equal(t, "work", ars.Spec.Template.Spec.Volumes[0].Name, "Volume name should be work")
+	assert.Equal(t, "/data", ars.Spec.Template.Spec.Volumes[0].HostPath.Path, "Volume host path should be /data")
+	assert.Equal(t, "foo", ars.Spec.Template.Spec.Volumes[1].Name, "Volume name should be foo")
+	assert.Equal(t, "bar", ars.Spec.Template.Spec.Volumes[2].Name, "Volume name should be bar")
+}
+
 func TestTemplateRenderedAutoScalingRunnerSet_EnableDinD(t *testing.T) {
 	t.Parallel()
 
@@ -636,7 +728,7 @@ func TestTemplateRenderedAutoScalingRunnerSet_EnableDinD(t *testing.T) {
 	assert.Equal(t, "init-dind-externals", ars.Spec.Template.Spec.InitContainers[0].Name)
 	assert.Equal(t, "ghcr.io/actions/actions-runner:latest", ars.Spec.Template.Spec.InitContainers[0].Image)
 	assert.Equal(t, "cp", ars.Spec.Template.Spec.InitContainers[0].Command[0])
-	assert.Equal(t, "-r -v /actions-runner/externals/. /actions-runner/tmpDir/", strings.Join(ars.Spec.Template.Spec.InitContainers[0].Args, " "))
+	assert.Equal(t, "-r -v /home/runner/externals/. /home/runner/tmpDir/", strings.Join(ars.Spec.Template.Spec.InitContainers[0].Args, " "))
 
 	assert.Len(t, ars.Spec.Template.Spec.Containers, 2, "Template.Spec should have 2 container")
 	assert.Equal(t, "runner", ars.Spec.Template.Spec.Containers[0].Name)
@@ -653,7 +745,7 @@ func TestTemplateRenderedAutoScalingRunnerSet_EnableDinD(t *testing.T) {
 
 	assert.Len(t, ars.Spec.Template.Spec.Containers[0].VolumeMounts, 2, "The runner container should have 2 volume mounts, dind-cert and work")
 	assert.Equal(t, "work", ars.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name)
-	assert.Equal(t, "/actions-runner/_work", ars.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath)
+	assert.Equal(t, "/home/runner/_work", ars.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath)
 	assert.False(t, ars.Spec.Template.Spec.Containers[0].VolumeMounts[0].ReadOnly)
 
 	assert.Equal(t, "dind-cert", ars.Spec.Template.Spec.Containers[0].VolumeMounts[1].Name)
@@ -665,13 +757,19 @@ func TestTemplateRenderedAutoScalingRunnerSet_EnableDinD(t *testing.T) {
 	assert.True(t, *ars.Spec.Template.Spec.Containers[1].SecurityContext.Privileged)
 	assert.Len(t, ars.Spec.Template.Spec.Containers[1].VolumeMounts, 3, "The dind container should have 3 volume mounts, dind-cert, work and externals")
 	assert.Equal(t, "work", ars.Spec.Template.Spec.Containers[1].VolumeMounts[0].Name)
-	assert.Equal(t, "/actions-runner/_work", ars.Spec.Template.Spec.Containers[1].VolumeMounts[0].MountPath)
+	assert.Equal(t, "/home/runner/_work", ars.Spec.Template.Spec.Containers[1].VolumeMounts[0].MountPath)
 
 	assert.Equal(t, "dind-cert", ars.Spec.Template.Spec.Containers[1].VolumeMounts[1].Name)
 	assert.Equal(t, "/certs/client", ars.Spec.Template.Spec.Containers[1].VolumeMounts[1].MountPath)
 
 	assert.Equal(t, "dind-externals", ars.Spec.Template.Spec.Containers[1].VolumeMounts[2].Name)
-	assert.Equal(t, "/actions-runner/externals", ars.Spec.Template.Spec.Containers[1].VolumeMounts[2].MountPath)
+	assert.Equal(t, "/home/runner/externals", ars.Spec.Template.Spec.Containers[1].VolumeMounts[2].MountPath)
+
+	assert.Len(t, ars.Spec.Template.Spec.Volumes, 3, "Volumes should be 3")
+	assert.Equal(t, "dind-cert", ars.Spec.Template.Spec.Volumes[0].Name, "Volume name should be dind-cert")
+	assert.Equal(t, "dind-externals", ars.Spec.Template.Spec.Volumes[1].Name, "Volume name should be dind-externals")
+	assert.Equal(t, "work", ars.Spec.Template.Spec.Volumes[2].Name, "Volume name should be work")
+	assert.NotNil(t, ars.Spec.Template.Spec.Volumes[2].EmptyDir, "Volume work should be an emptyDir")
 }
 
 func TestTemplateRenderedAutoScalingRunnerSet_EnableKubernetesMode(t *testing.T) {
@@ -719,7 +817,7 @@ func TestTemplateRenderedAutoScalingRunnerSet_EnableKubernetesMode(t *testing.T)
 	assert.Equal(t, "ghcr.io/actions/actions-runner:latest", ars.Spec.Template.Spec.Containers[0].Image)
 
 	assert.Equal(t, "ACTIONS_RUNNER_CONTAINER_HOOKS", ars.Spec.Template.Spec.Containers[0].Env[0].Name)
-	assert.Equal(t, "/actions-runner/k8s/index.js", ars.Spec.Template.Spec.Containers[0].Env[0].Value)
+	assert.Equal(t, "/home/runner/k8s/index.js", ars.Spec.Template.Spec.Containers[0].Env[0].Value)
 	assert.Equal(t, "ACTIONS_RUNNER_POD_NAME", ars.Spec.Template.Spec.Containers[0].Env[1].Name)
 	assert.Equal(t, "ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER", ars.Spec.Template.Spec.Containers[0].Env[2].Name)
 	assert.Equal(t, "true", ars.Spec.Template.Spec.Containers[0].Env[2].Value)

charts/gha-runner-scale-set/tests/values_dind_extra_volumes.yaml

@@ -0,0 +1,19 @@
+githubConfigUrl: https://github.com/actions/actions-runner-controller
+githubConfigSecret:
+  github_token: test
+template:
+  spec:
+    containers:
+    - name: other
+      image: other-image:latest
+    volumes:
+    - name: foo
+      emptyDir: {}
+    - name: bar
+      emptyDir: {}
+    - name: work
+      hostPath:
+        path: /data
+        type: Directory
+containerMode:
+  type: dind

charts/gha-runner-scale-set/tests/values_extra_volumes.yaml

@@ -0,0 +1,17 @@
+githubConfigUrl: https://github.com/actions/actions-runner-controller
+githubConfigSecret:
+  github_token: test
+template:
+  spec:
+    containers:
+    - name: other
+      image: other-image:latest
+    volumes:
+    - name: foo
+      emptyDir: {}
+    - name: bar
+      emptyDir: {}
+    - name: work
+      hostPath:
+        path: /data
+        type: Directory

charts/gha-runner-scale-set/tests/values_k8s_extra_volumes.yaml

@@ -0,0 +1,19 @@
+githubConfigUrl: https://github.com/actions/actions-runner-controller
+githubConfigSecret:
+  github_token: test
+template:
+  spec:
+    containers:
+    - name: other
+      image: other-image:latest
+    volumes:
+    - name: foo
+      emptyDir: {}
+    - name: bar
+      emptyDir: {}
+    - name: work
+      hostPath:
+        path: /data
+        type: Directory
+containerMode:
+  type: kubernetes

charts/gha-runner-scale-set/values.yaml

@@ -74,7 +74,7 @@ template:
     containers:
     - name: runner
       image: ghcr.io/actions/actions-runner:latest
-      command: ["/actions-runner/run.sh"]
+      command: ["/home/runner/run.sh"]
 
 containerMode:
   type: ""  ## type can be set to dind or kubernetes
@@ -84,10 +84,10 @@ containerMode:
   ##     initContainers:
   ##     - name: initExternalsInternalVolume
   ##       image: ghcr.io/actions/actions-runner:latest
-  ##       command: ["cp", "-r", "-v", "/actions-runner/externals/.", "/actions-runner/tmpDir/"]
+  ##       command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"]
   ##       volumeMounts:
   ##         - name: externalsInternal
-  ##           mountPath: /actions-runner/tmpDir
+  ##           mountPath: /home/runner/tmpDir
   ##     containers:
   ##     - name: runner
   ##       image: ghcr.io/actions/actions-runner:latest
@@ -100,7 +100,7 @@ containerMode:
   ##           value: /certs/client
   ##       volumeMounts:
   ##         - name: workingDirectoryInternal
-  ##           mountPath: /actions-runner/_work
+  ##           mountPath: /home/runner/_work
   ##         - name: dinDInternal
   ##           mountPath: /certs/client
   ##           readOnly: true
@@ -111,9 +111,9 @@ containerMode:
   ##       volumeMounts:
   ##         - mountPath: /certs/client
   ##           name: dinDInternal
-  ##         - mountPath: /actions-runner/_work
+  ##         - mountPath: /home/runner/_work
   ##           name: workingDirectoryInternal
-  ##         - mountPath: /actions-runner/externals
+  ##         - mountPath: /home/runner/externals
   ##           name: externalsInternal
   ##     volumes:
   ##     - name: dinDInternal
@@ -131,7 +131,7 @@ containerMode:
   ##       image: ghcr.io/actions/actions-runner:latest
   ##       env:
   ##         - name: ACTIONS_RUNNER_CONTAINER_HOOKS
-  ##           value: /actions-runner/k8s/index.js
+  ##           value: /home/runner/k8s/index.js
   ##         - name: ACTIONS_RUNNER_POD_NAME
   ##           valueFrom:
   ##             fieldRef:
@@ -140,7 +140,7 @@ containerMode:
   ##           value: "true"
   ##       volumeMounts:
   ##         - name: work
-  ##           mountPath: /actions-runner/_work
+  ##           mountPath: /home/runner/_work
   ##     volumes:
   ##       - name: work
   ##         ephemeral:

controllers/actions.github.com/ephemeralrunner_controller_test.go

@@ -63,7 +63,7 @@ func newExampleRunner(name, namespace, configSecretName string) *v1alpha1.Epheme
 						{
 							Name:    "setup",
 							Image:   runnerImage,
-							Command: []string{"sh", "-c", "cp -r /actions-runner/* /runner/"},
+							Command: []string{"sh", "-c", "cp -r /home/runner/* /runner/"},
 							VolumeMounts: []corev1.VolumeMount{
 								{
 									Name:      "runner",