diff --git a/.github/workflows/e2e-test-linux-vm.yaml b/.github/workflows/e2e-test-linux-vm.yaml index 88ba0dd1..d0c71c21 100644 --- a/.github/workflows/e2e-test-linux-vm.yaml +++ b/.github/workflows/e2e-test-linux-vm.yaml @@ -120,6 +120,108 @@ jobs: helm uninstall ${{ steps.install_arc.outputs.ARC_NAME }} --namespace arc-runners kubectl wait --timeout=10s --for=delete AutoScalingRunnerSet -n demo -l app.kubernetes.io/instance=${{ steps.install_arc.outputs.ARC_NAME }} + - name: Dump gha-runner-scale-set-controller logs + if: always() && steps.install_arc_controller.outcome == 'success' + run: | + kubectl logs deployment/arc-gha-runner-scale-set-controller -n arc-systems + + single-namespace-setup: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Resolve inputs + id: resolved_inputs + run: | + TARGET_ORG="${{env.TARGET_ORG}}" + TARGET_REPO="${{env.TARGET_REPO}}" + if [ ! -z "${{inputs.target_org}}" ]; then + TARGET_ORG="${{inputs.target_org}}" + fi + if [ ! -z "${{inputs.target_repo}}" ]; then + TARGET_REPO="${{inputs.target_repo}}" + fi + echo "TARGET_ORG=$TARGET_ORG" >> $GITHUB_OUTPUT + echo "TARGET_REPO=$TARGET_REPO" >> $GITHUB_OUTPUT + + - uses: ./.github/actions/setup-arc-e2e + id: setup + with: + github-app-id: ${{secrets.ACTIONS_ACCESS_APP_ID}} + github-app-pk: ${{secrets.ACTIONS_ACCESS_PK}} + github-app-org: ${{steps.resolved_inputs.outputs.TARGET_ORG}} + docker-image-name: ${{env.IMAGE_NAME}} + docker-image-tag: ${{env.IMAGE_VERSION}} + + - name: Install gha-runner-scale-set-controller + id: install_arc_controller + run: | + kubectl create namespace arc-runners + helm install arc \ + --namespace "arc-systems" \ + --create-namespace \ + --set image.repository=${{ env.IMAGE_NAME }} \ + --set image.tag=${{ env.IMAGE_VERSION }} \ + --set flags.watchSingleNamespace=arc-runners \ + ./charts/gha-runner-scale-set-controller \ + --debug + count=0 + while true; do + POD_NAME=$(kubectl get pods -n arc-systems -l app.kubernetes.io/name=gha-runner-scale-set-controller -o name) + if [ -n "$POD_NAME" ]; then + echo "Pod found: $POD_NAME" + break + fi + if [ "$count" -ge 10 ]; then + echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-runner-scale-set-controller" + exit 1 + fi + sleep 1 + done + kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l app.kubernetes.io/name=gha-runner-scale-set-controller + kubectl get pod -n arc-systems + kubectl describe deployment arc-gha-runner-scale-set-controller -n arc-systems + + - name: Install gha-runner-scale-set + id: install_arc + run: | + ARC_NAME=arc-runner-${{github.job}}-$(date +'%M-%S')-$(($RANDOM % 100 + 1)) + helm install "$ARC_NAME" \ + --namespace "arc-runners" \ + --create-namespace \ + --set githubConfigUrl="https://github.com/${{ steps.resolved_inputs.outputs.TARGET_ORG }}/${{steps.resolved_inputs.outputs.TARGET_REPO}}" \ + --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \ + ./charts/gha-runner-scale-set \ + --debug + echo "ARC_NAME=$ARC_NAME" >> $GITHUB_OUTPUT + count=0 + while true; do + POD_NAME=$(kubectl get pods -n arc-systems -l auto-scaling-runner-set-name=$ARC_NAME -o name) + if [ -n "$POD_NAME" ]; then + echo "Pod found: $POD_NAME" + break + fi + if [ "$count" -ge 10 ]; then + echo "Timeout waiting for listener pod with label auto-scaling-runner-set-name=$ARC_NAME" + exit 1 + fi + sleep 1 + done + kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l auto-scaling-runner-set-name=$ARC_NAME + kubectl get pod -n arc-systems + + - name: Test ARC scales pods up and down + run: | + export GITHUB_TOKEN="${{ steps.setup.outputs.token }}" + export ARC_NAME="${{ steps.install_arc.outputs.ARC_NAME }}" + go test ./test_e2e_arc -v + + - name: Uninstall gha-runner-scale-set + if: always() && steps.install_arc.outcome == 'success' + run: | + helm uninstall ${{ steps.install_arc.outputs.ARC_NAME }} --namespace arc-runners + kubectl wait --timeout=10s --for=delete AutoScalingRunnerSet -n demo -l app.kubernetes.io/instance=${{ steps.install_arc.outputs.ARC_NAME }} + - name: Dump gha-runner-scale-set-controller logs if: always() && steps.install_arc_controller.outcome == 'success' run: | diff --git a/charts/gha-runner-scale-set-controller/templates/_helpers.tpl b/charts/gha-runner-scale-set-controller/templates/_helpers.tpl index ebef4a9b..eb37c21f 100644 --- a/charts/gha-runner-scale-set-controller/templates/_helpers.tpl +++ b/charts/gha-runner-scale-set-controller/templates/_helpers.tpl @@ -80,6 +80,14 @@ Create the name of the service account to use {{- include "gha-runner-scale-set-controller.fullname" . }}-manager-cluster-rolebinding {{- end }} +{{- define "gha-runner-scale-set-controller.managerSingleNamespaceRoleName" -}} +{{- include "gha-runner-scale-set-controller.fullname" . }}-manager-single-namespace-role +{{- end }} + +{{- define "gha-runner-scale-set-controller.managerSingleNamespaceRoleBinding" -}} +{{- include "gha-runner-scale-set-controller.fullname" . }}-manager-single-namespace-rolebinding +{{- end }} + {{- define "gha-runner-scale-set-controller.managerListenerRoleName" -}} {{- include "gha-runner-scale-set-controller.fullname" . }}-manager-listener-role {{- end }} diff --git a/charts/gha-runner-scale-set-controller/templates/deployment.yaml b/charts/gha-runner-scale-set-controller/templates/deployment.yaml index f509aa67..a8f02e62 100644 --- a/charts/gha-runner-scale-set-controller/templates/deployment.yaml +++ b/charts/gha-runner-scale-set-controller/templates/deployment.yaml @@ -7,6 +7,9 @@ metadata: {{- include "gha-runner-scale-set-controller.labels" . | nindent 4 }} actions.github.com/controller-service-account-namespace: {{ .Release.Namespace }} actions.github.com/controller-service-account-name: {{ include "gha-runner-scale-set-controller.serviceAccountName" . }} + {{- if .Values.flags.watchSingleNamespace }} + actions.github.com/controller-watch-single-namespace: {{ .Values.flags.watchSingleNamespace }} + {{- end }} spec: replicas: {{ default 1 .Values.replicaCount }} selector: @@ -53,6 +56,9 @@ spec: {{- with .Values.flags.logLevel }} - "--log-level={{ . }}" {{- end }} + {{- with .Values.flags.watchSingleNamespace }} + - "--watch-single-namespace={{ . }}" + {{- end }} command: - "/manager" env: diff --git a/charts/gha-runner-scale-set-controller/templates/manager_cluster_role.yaml b/charts/gha-runner-scale-set-controller/templates/manager_cluster_role.yaml index deb3c999..3ea31279 100644 --- a/charts/gha-runner-scale-set-controller/templates/manager_cluster_role.yaml +++ b/charts/gha-runner-scale-set-controller/templates/manager_cluster_role.yaml @@ -1,3 +1,4 @@ +{{- if empty .Values.flags.watchSingleNamespace }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -20,6 +21,7 @@ rules: resources: - autoscalingrunnersets/finalizers verbs: + - patch - update - apiGroups: - actions.github.com @@ -54,6 +56,7 @@ rules: resources: - autoscalinglisteners/finalizers verbs: + - patch - update - apiGroups: - actions.github.com @@ -92,13 +95,8 @@ rules: resources: - ephemeralrunners/finalizers verbs: - - create - - delete - - get - - list - patch - update - - watch - apiGroups: - actions.github.com resources: @@ -135,3 +133,4 @@ rules: verbs: - list - watch +{{- end }} diff --git a/charts/gha-runner-scale-set-controller/templates/manager_cluster_role_binding.yaml b/charts/gha-runner-scale-set-controller/templates/manager_cluster_role_binding.yaml index 4ce8f9b8..041d73a9 100644 --- a/charts/gha-runner-scale-set-controller/templates/manager_cluster_role_binding.yaml +++ b/charts/gha-runner-scale-set-controller/templates/manager_cluster_role_binding.yaml @@ -1,3 +1,4 @@ +{{- if empty .Values.flags.watchSingleNamespace }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -9,4 +10,5 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "gha-runner-scale-set-controller.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} \ No newline at end of file + namespace: {{ .Release.Namespace }} +{{- end }} \ No newline at end of file diff --git a/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role.yaml b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role.yaml new file mode 100644 index 00000000..a72dc738 --- /dev/null +++ b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role.yaml @@ -0,0 +1,84 @@ +{{- if .Values.flags.watchSingleNamespace }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "gha-runner-scale-set-controller.managerSingleNamespaceRoleName" . }} + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: + - actions.github.com + resources: + - autoscalinglisteners + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - actions.github.com + resources: + - autoscalinglisteners/status + verbs: + - get + - patch + - update +- apiGroups: + - actions.github.com + resources: + - autoscalinglisteners/finalizers + verbs: + - patch + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - list + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + verbs: + - list + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + verbs: + - list + - watch +- apiGroups: + - actions.github.com + resources: + - autoscalingrunnersets + verbs: + - list + - watch +- apiGroups: + - actions.github.com + resources: + - ephemeralrunnersets + verbs: + - list + - watch +- apiGroups: + - actions.github.com + resources: + - ephemeralrunners + verbs: + - list + - watch +{{- end }} \ No newline at end of file diff --git a/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role_binding.yaml b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role_binding.yaml new file mode 100644 index 00000000..3423b9dd --- /dev/null +++ b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_controller_role_binding.yaml @@ -0,0 +1,15 @@ +{{- if .Values.flags.watchSingleNamespace }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "gha-runner-scale-set-controller.managerSingleNamespaceRoleBinding" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "gha-runner-scale-set-controller.managerSingleNamespaceRoleName" . }} +subjects: +- kind: ServiceAccount + name: {{ include "gha-runner-scale-set-controller.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} \ No newline at end of file diff --git a/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role.yaml b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role.yaml new file mode 100644 index 00000000..bf840bcf --- /dev/null +++ b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role.yaml @@ -0,0 +1,117 @@ +{{- if .Values.flags.watchSingleNamespace }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "gha-runner-scale-set-controller.managerSingleNamespaceRoleName" . }} + namespace: {{ .Values.flags.watchSingleNamespace }} +rules: +- apiGroups: + - actions.github.com + resources: + - autoscalingrunnersets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - actions.github.com + resources: + - autoscalingrunnersets/finalizers + verbs: + - patch + - update +- apiGroups: + - actions.github.com + resources: + - autoscalingrunnersets/status + verbs: + - get + - patch + - update +- apiGroups: + - actions.github.com + resources: + - ephemeralrunnersets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - actions.github.com + resources: + - ephemeralrunnersets/status + verbs: + - get + - patch + - update +- apiGroups: + - actions.github.com + resources: + - ephemeralrunners + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - actions.github.com + resources: + - ephemeralrunners/finalizers + verbs: + - patch + - update +- apiGroups: + - actions.github.com + resources: + - ephemeralrunners/status + verbs: + - get + - patch + - update +- apiGroups: + - actions.github.com + resources: + - autoscalinglisteners + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - list + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + verbs: + - list + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + verbs: + - list + - watch +{{- end }} \ No newline at end of file diff --git a/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role_binding.yaml b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role_binding.yaml new file mode 100644 index 00000000..3edd0c61 --- /dev/null +++ b/charts/gha-runner-scale-set-controller/templates/manager_single_namespace_watch_role_binding.yaml @@ -0,0 +1,15 @@ +{{- if .Values.flags.watchSingleNamespace }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "gha-runner-scale-set-controller.managerSingleNamespaceRoleBinding" . }} + namespace: {{ .Values.flags.watchSingleNamespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "gha-runner-scale-set-controller.managerSingleNamespaceRoleName" . }} +subjects: +- kind: ServiceAccount + name: {{ include "gha-runner-scale-set-controller.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} \ No newline at end of file diff --git a/charts/gha-runner-scale-set-controller/tests/template_test.go b/charts/gha-runner-scale-set-controller/tests/template_test.go index b954867e..2a00370a 100644 --- a/charts/gha-runner-scale-set-controller/tests/template_test.go +++ b/charts/gha-runner-scale-set-controller/tests/template_test.go @@ -170,6 +170,12 @@ func TestTemplate_CreateManagerClusterRole(t *testing.T) { assert.Empty(t, managerClusterRole.Namespace, "ClusterRole should not have a namespace") assert.Equal(t, "test-arc-gha-runner-scale-set-controller-manager-cluster-role", managerClusterRole.Name) assert.Equal(t, 15, len(managerClusterRole.Rules)) + + _, err = helm.RenderTemplateE(t, options, helmChartPath, releaseName, []string{"templates/manager_single_namespace_controller_role.yaml"}) + assert.ErrorContains(t, err, "could not find template templates/manager_single_namespace_controller_role.yaml in chart", "We should get an error because the template should be skipped") + + _, err = helm.RenderTemplateE(t, options, helmChartPath, releaseName, []string{"templates/manager_single_namespace_watch_role.yaml"}) + assert.ErrorContains(t, err, "could not find template templates/manager_single_namespace_watch_role.yaml in chart", "We should get an error because the template should be skipped") } func TestTemplate_ManagerClusterRoleBinding(t *testing.T) { @@ -199,6 +205,12 @@ func TestTemplate_ManagerClusterRoleBinding(t *testing.T) { assert.Equal(t, "test-arc-gha-runner-scale-set-controller-manager-cluster-role", managerClusterRoleBinding.RoleRef.Name) assert.Equal(t, "test-arc-gha-runner-scale-set-controller", managerClusterRoleBinding.Subjects[0].Name) assert.Equal(t, namespaceName, managerClusterRoleBinding.Subjects[0].Namespace) + + _, err = helm.RenderTemplateE(t, options, helmChartPath, releaseName, []string{"templates/manager_single_namespace_controller_role_binding.yaml"}) + assert.ErrorContains(t, err, "could not find template templates/manager_single_namespace_controller_role_binding.yaml in chart", "We should get an error because the template should be skipped") + + _, err = helm.RenderTemplateE(t, options, helmChartPath, releaseName, []string{"templates/manager_single_namespace_watch_role_binding.yaml"}) + assert.ErrorContains(t, err, "could not find template templates/manager_single_namespace_watch_role_binding.yaml in chart", "We should get an error because the template should be skipped") } func TestTemplate_CreateManagerListenerRole(t *testing.T) { @@ -297,6 +309,7 @@ func TestTemplate_ControllerDeployment_Defaults(t *testing.T) { assert.Equal(t, "Helm", deployment.Labels["app.kubernetes.io/managed-by"]) assert.Equal(t, namespaceName, deployment.Labels["actions.github.com/controller-service-account-namespace"]) assert.Equal(t, "test-arc-gha-runner-scale-set-controller", deployment.Labels["actions.github.com/controller-service-account-name"]) + assert.NotContains(t, deployment.Labels, "actions.github.com/controller-watch-single-namespace") assert.Equal(t, int32(1), *deployment.Spec.Replicas) @@ -595,3 +608,215 @@ func TestTemplate_ControllerDeployment_ForwardImagePullSecrets(t *testing.T) { assert.Equal(t, "--auto-scaler-image-pull-secrets=dockerhub,ghcr", deployment.Spec.Template.Spec.Containers[0].Args[1]) assert.Equal(t, "--log-level=debug", deployment.Spec.Template.Spec.Containers[0].Args[2]) } + +func TestTemplate_ControllerDeployment_WatchSingleNamespace(t *testing.T) { + t.Parallel() + + // Path to the helm chart we will test + helmChartPath, err := filepath.Abs("../../gha-runner-scale-set-controller") + require.NoError(t, err) + + chartContent, err := os.ReadFile(filepath.Join(helmChartPath, "Chart.yaml")) + require.NoError(t, err) + + chart := new(Chart) + err = yaml.Unmarshal(chartContent, chart) + require.NoError(t, err) + + releaseName := "test-arc" + namespaceName := "test-" + strings.ToLower(random.UniqueId()) + + options := &helm.Options{ + SetValues: map[string]string{ + "image.tag": "dev", + "flags.watchSingleNamespace": "demo", + }, + KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName), + } + + output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/deployment.yaml"}) + + var deployment appsv1.Deployment + helm.UnmarshalK8SYaml(t, output, &deployment) + + assert.Equal(t, namespaceName, deployment.Namespace) + assert.Equal(t, "test-arc-gha-runner-scale-set-controller", deployment.Name) + assert.Equal(t, "gha-runner-scale-set-controller-"+chart.Version, deployment.Labels["helm.sh/chart"]) + assert.Equal(t, "gha-runner-scale-set-controller", deployment.Labels["app.kubernetes.io/name"]) + assert.Equal(t, "test-arc", deployment.Labels["app.kubernetes.io/instance"]) + assert.Equal(t, chart.AppVersion, deployment.Labels["app.kubernetes.io/version"]) + assert.Equal(t, "Helm", deployment.Labels["app.kubernetes.io/managed-by"]) + assert.Equal(t, namespaceName, deployment.Labels["actions.github.com/controller-service-account-namespace"]) + assert.Equal(t, "test-arc-gha-runner-scale-set-controller", deployment.Labels["actions.github.com/controller-service-account-name"]) + assert.Equal(t, "demo", deployment.Labels["actions.github.com/controller-watch-single-namespace"]) + + assert.Equal(t, int32(1), *deployment.Spec.Replicas) + + assert.Equal(t, "gha-runner-scale-set-controller", deployment.Spec.Selector.MatchLabels["app.kubernetes.io/name"]) + assert.Equal(t, "test-arc", deployment.Spec.Selector.MatchLabels["app.kubernetes.io/instance"]) + + assert.Equal(t, "gha-runner-scale-set-controller", deployment.Spec.Template.Labels["app.kubernetes.io/name"]) + assert.Equal(t, "test-arc", deployment.Spec.Template.Labels["app.kubernetes.io/instance"]) + + assert.Equal(t, "manager", deployment.Spec.Template.Annotations["kubectl.kubernetes.io/default-container"]) + + assert.Len(t, deployment.Spec.Template.Spec.ImagePullSecrets, 0) + assert.Equal(t, "test-arc-gha-runner-scale-set-controller", deployment.Spec.Template.Spec.ServiceAccountName) + assert.Nil(t, deployment.Spec.Template.Spec.SecurityContext) + assert.Empty(t, deployment.Spec.Template.Spec.PriorityClassName) + assert.Equal(t, int64(10), *deployment.Spec.Template.Spec.TerminationGracePeriodSeconds) + assert.Len(t, deployment.Spec.Template.Spec.Volumes, 1) + assert.Equal(t, "tmp", deployment.Spec.Template.Spec.Volumes[0].Name) + assert.NotNil(t, 10, deployment.Spec.Template.Spec.Volumes[0].EmptyDir) + + assert.Len(t, deployment.Spec.Template.Spec.NodeSelector, 0) + assert.Nil(t, deployment.Spec.Template.Spec.Affinity) + assert.Len(t, deployment.Spec.Template.Spec.Tolerations, 0) + + managerImage := "ghcr.io/actions/gha-runner-scale-set-controller:dev" + + assert.Len(t, deployment.Spec.Template.Spec.Containers, 1) + assert.Equal(t, "manager", deployment.Spec.Template.Spec.Containers[0].Name) + assert.Equal(t, "ghcr.io/actions/gha-runner-scale-set-controller:dev", deployment.Spec.Template.Spec.Containers[0].Image) + assert.Equal(t, corev1.PullIfNotPresent, deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy) + + assert.Len(t, deployment.Spec.Template.Spec.Containers[0].Command, 1) + assert.Equal(t, "/manager", deployment.Spec.Template.Spec.Containers[0].Command[0]) + + assert.Len(t, deployment.Spec.Template.Spec.Containers[0].Args, 3) + assert.Equal(t, "--auto-scaling-runner-set-only", deployment.Spec.Template.Spec.Containers[0].Args[0]) + assert.Equal(t, "--log-level=debug", deployment.Spec.Template.Spec.Containers[0].Args[1]) + assert.Equal(t, "--watch-single-namespace=demo", deployment.Spec.Template.Spec.Containers[0].Args[2]) + + assert.Len(t, deployment.Spec.Template.Spec.Containers[0].Env, 2) + assert.Equal(t, "CONTROLLER_MANAGER_CONTAINER_IMAGE", deployment.Spec.Template.Spec.Containers[0].Env[0].Name) + assert.Equal(t, managerImage, deployment.Spec.Template.Spec.Containers[0].Env[0].Value) + + assert.Equal(t, "CONTROLLER_MANAGER_POD_NAMESPACE", deployment.Spec.Template.Spec.Containers[0].Env[1].Name) + assert.Equal(t, "metadata.namespace", deployment.Spec.Template.Spec.Containers[0].Env[1].ValueFrom.FieldRef.FieldPath) + + assert.Empty(t, deployment.Spec.Template.Spec.Containers[0].Resources) + assert.Nil(t, deployment.Spec.Template.Spec.Containers[0].SecurityContext) + assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 1) + assert.Equal(t, "tmp", deployment.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name) + assert.Equal(t, "/tmp", deployment.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath) +} + +func TestTemplate_WatchSingleNamespace_NotCreateManagerClusterRole(t *testing.T) { + t.Parallel() + + // Path to the helm chart we will test + helmChartPath, err := filepath.Abs("../../gha-runner-scale-set-controller") + require.NoError(t, err) + + releaseName := "test-arc" + namespaceName := "test-" + strings.ToLower(random.UniqueId()) + + options := &helm.Options{ + SetValues: map[string]string{ + "flags.watchSingleNamespace": "demo", + }, + KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName), + } + + _, err = helm.RenderTemplateE(t, options, helmChartPath, releaseName, []string{"templates/manager_cluster_role.yaml"}) + assert.ErrorContains(t, err, "could not find template templates/manager_cluster_role.yaml in chart", "We should get an error because the template should be skipped") +} + +func TestTemplate_WatchSingleNamespace_NotManagerClusterRoleBinding(t *testing.T) { + t.Parallel() + + // Path to the helm chart we will test + helmChartPath, err := filepath.Abs("../../gha-runner-scale-set-controller") + require.NoError(t, err) + + releaseName := "test-arc" + namespaceName := "test-" + strings.ToLower(random.UniqueId()) + + options := &helm.Options{ + SetValues: map[string]string{ + "serviceAccount.create": "true", + "flags.watchSingleNamespace": "demo", + }, + KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName), + } + + _, err = helm.RenderTemplateE(t, options, helmChartPath, releaseName, []string{"templates/manager_cluster_role_binding.yaml"}) + assert.ErrorContains(t, err, "could not find template templates/manager_cluster_role_binding.yaml in chart", "We should get an error because the template should be skipped") +} + +func TestTemplate_CreateManagerSingleNamespaceRole(t *testing.T) { + t.Parallel() + + // Path to the helm chart we will test + helmChartPath, err := filepath.Abs("../../gha-runner-scale-set-controller") + require.NoError(t, err) + + releaseName := "test-arc" + namespaceName := "test-" + strings.ToLower(random.UniqueId()) + + options := &helm.Options{ + SetValues: map[string]string{ + "flags.watchSingleNamespace": "demo", + }, + KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName), + } + + output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/manager_single_namespace_controller_role.yaml"}) + + var managerSingleNamespaceControllerRole rbacv1.Role + helm.UnmarshalK8SYaml(t, output, &managerSingleNamespaceControllerRole) + + assert.Equal(t, "test-arc-gha-runner-scale-set-controller-manager-single-namespace-role", managerSingleNamespaceControllerRole.Name) + assert.Equal(t, namespaceName, managerSingleNamespaceControllerRole.Namespace) + assert.Equal(t, 10, len(managerSingleNamespaceControllerRole.Rules)) + + output = helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/manager_single_namespace_watch_role.yaml"}) + + var managerSingleNamespaceWatchRole rbacv1.Role + helm.UnmarshalK8SYaml(t, output, &managerSingleNamespaceWatchRole) + + assert.Equal(t, "test-arc-gha-runner-scale-set-controller-manager-single-namespace-role", managerSingleNamespaceWatchRole.Name) + assert.Equal(t, "demo", managerSingleNamespaceWatchRole.Namespace) + assert.Equal(t, 13, len(managerSingleNamespaceWatchRole.Rules)) +} + +func TestTemplate_ManagerSingleNamespaceRoleBinding(t *testing.T) { + t.Parallel() + + // Path to the helm chart we will test + helmChartPath, err := filepath.Abs("../../gha-runner-scale-set-controller") + require.NoError(t, err) + + releaseName := "test-arc" + namespaceName := "test-" + strings.ToLower(random.UniqueId()) + + options := &helm.Options{ + SetValues: map[string]string{ + "flags.watchSingleNamespace": "demo", + }, + KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName), + } + + output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/manager_single_namespace_controller_role_binding.yaml"}) + + var managerSingleNamespaceControllerRoleBinding rbacv1.RoleBinding + helm.UnmarshalK8SYaml(t, output, &managerSingleNamespaceControllerRoleBinding) + + assert.Equal(t, "test-arc-gha-runner-scale-set-controller-manager-single-namespace-rolebinding", managerSingleNamespaceControllerRoleBinding.Name) + assert.Equal(t, namespaceName, managerSingleNamespaceControllerRoleBinding.Namespace) + assert.Equal(t, "test-arc-gha-runner-scale-set-controller-manager-single-namespace-role", managerSingleNamespaceControllerRoleBinding.RoleRef.Name) + assert.Equal(t, "test-arc-gha-runner-scale-set-controller", managerSingleNamespaceControllerRoleBinding.Subjects[0].Name) + assert.Equal(t, namespaceName, managerSingleNamespaceControllerRoleBinding.Subjects[0].Namespace) + + output = helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/manager_single_namespace_watch_role_binding.yaml"}) + + var managerSingleNamespaceWatchRoleBinding rbacv1.RoleBinding + helm.UnmarshalK8SYaml(t, output, &managerSingleNamespaceWatchRoleBinding) + + assert.Equal(t, "test-arc-gha-runner-scale-set-controller-manager-single-namespace-rolebinding", managerSingleNamespaceWatchRoleBinding.Name) + assert.Equal(t, "demo", managerSingleNamespaceWatchRoleBinding.Namespace) + assert.Equal(t, "test-arc-gha-runner-scale-set-controller-manager-single-namespace-role", managerSingleNamespaceWatchRoleBinding.RoleRef.Name) + assert.Equal(t, "test-arc-gha-runner-scale-set-controller", managerSingleNamespaceWatchRoleBinding.Subjects[0].Name) + assert.Equal(t, namespaceName, managerSingleNamespaceWatchRoleBinding.Subjects[0].Namespace) +} diff --git a/charts/gha-runner-scale-set-controller/values.yaml b/charts/gha-runner-scale-set-controller/values.yaml index 4e232944..055d68e9 100644 --- a/charts/gha-runner-scale-set-controller/values.yaml +++ b/charts/gha-runner-scale-set-controller/values.yaml @@ -68,3 +68,7 @@ flags: # Log level can be set here with one of the following values: "debug", "info", "warn", "error". # Defaults to "debug". logLevel: "debug" + + # Restricts the controller to only watch resources in the desired namespace. + # Defaults to watch all namespaces when unset. + # watchSingleNamespace: "" \ No newline at end of file diff --git a/charts/gha-runner-scale-set/templates/_helpers.tpl b/charts/gha-runner-scale-set/templates/_helpers.tpl index 208e42ea..013ca73e 100644 --- a/charts/gha-runner-scale-set/templates/_helpers.tpl +++ b/charts/gha-runner-scale-set/templates/_helpers.tpl @@ -476,25 +476,46 @@ volumeMounts: {{- end }} {{- end }} {{- if eq $searchControllerDeployment 1 }} - {{- $counter := 0 }} + {{- $multiNamespacesCounter := 0 }} + {{- $singleNamespaceCounter := 0 }} {{- $controllerDeployment := dict }} + {{- $singleNamespaceControllerDeployments := dict }} {{- $managerServiceAccountName := "" }} {{- range $index, $deployment := (lookup "apps/v1" "Deployment" "" "").items }} - {{- range $key, $val := $deployment.metadata.labels }} - {{- if and (eq $key "app.kubernetes.io/part-of") (eq $val "gha-runner-scale-set-controller") }} - {{- $counter = add $counter 1 }} - {{- $controllerDeployment = $deployment }} + {{- if kindIs "map" $deployment.metadata.labels }} + {{- if eq (get $deployment.metadata.labels "app.kubernetes.io/part-of") "gha-runner-scale-set-controller" }} + {{- if hasKey $deployment.metadata.labels "actions.github.com/controller-watch-single-namespace" }} + {{- $singleNamespaceCounter = add $singleNamespaceCounter 1 }} + {{- $_ := set $singleNamespaceControllerDeployments (get $deployment.metadata.labels "actions.github.com/controller-watch-single-namespace") $deployment}} + {{- else }} + {{- $multiNamespacesCounter = add $multiNamespacesCounter 1 }} + {{- $controllerDeployment = $deployment }} + {{- end }} {{- end }} {{- end }} {{- end }} - {{- if lt $counter 1 }} - {{- fail "No gha-runner-scale-set-controller deployment found using label (app.kubernetes.io/part-of=gha-runner-scale-set-controller), consider setting controllerServiceAccount.name in values.yaml to be explicit if you think the discovery is wrong." }} + {{- if and (eq $multiNamespacesCounter 0) (eq $singleNamespaceCounter 0) }} + {{- fail "No gha-runner-scale-set-controller deployment found using label (app.kubernetes.io/part-of=gha-runner-scale-set-controller). Consider setting controllerServiceAccount.name in values.yaml to be explicit if you think the discovery is wrong." }} {{- end }} - {{- if gt $counter 1 }} - {{- fail "More than one gha-runner-scale-set-controller deployment found using label (app.kubernetes.io/part-of=gha-runner-scale-set-controller), consider setting controllerServiceAccount.name in values.yaml to be explicit if you think the discovery is wrong." }} + {{- if and (gt $multiNamespacesCounter 0) (gt $singleNamespaceCounter 0) }} + {{- fail "Found both gha-runner-scale-set-controller installed with flags.watchSingleNamespace set and unset in cluster, this is not supported. Consider setting controllerServiceAccount.name in values.yaml to be explicit if you think the discovery is wrong." }} {{- end }} - {{- with $controllerDeployment.metadata }} - {{- $managerServiceAccountName = (get $controllerDeployment.metadata.labels "actions.github.com/controller-service-account-name") }} + {{- if gt $multiNamespacesCounter 1 }} + {{- fail "More than one gha-runner-scale-set-controller deployment found using label (app.kubernetes.io/part-of=gha-runner-scale-set-controller). Consider setting controllerServiceAccount.name in values.yaml to be explicit if you think the discovery is wrong." }} + {{- end }} + {{- if eq $multiNamespacesCounter 1 }} + {{- with $controllerDeployment.metadata }} + {{- $managerServiceAccountName = (get $controllerDeployment.metadata.labels "actions.github.com/controller-service-account-name") }} + {{- end }} + {{- else if gt $singleNamespaceCounter 0 }} + {{- if hasKey $singleNamespaceControllerDeployments .Release.Namespace }} + {{- $controllerDeployment = get $singleNamespaceControllerDeployments .Release.Namespace }} + {{- with $controllerDeployment.metadata }} + {{- $managerServiceAccountName = (get $controllerDeployment.metadata.labels "actions.github.com/controller-service-account-name") }} + {{- end }} + {{- else }} + {{- fail "No gha-runner-scale-set-controller deployment that watch this namespace found using label (actions.github.com/controller-watch-single-namespace). Consider setting controllerServiceAccount.name in values.yaml to be explicit if you think the discovery is wrong." }} + {{- end }} {{- end }} {{- if eq $managerServiceAccountName "" }} {{- fail "No service account name found for gha-runner-scale-set-controller deployment using label (actions.github.com/controller-service-account-name), consider setting controllerServiceAccount.name in values.yaml to be explicit if you think the discovery is wrong." }} @@ -512,28 +533,49 @@ volumeMounts: {{- end }} {{- end }} {{- if eq $searchControllerDeployment 1 }} - {{- $counter := 0 }} + {{- $multiNamespacesCounter := 0 }} + {{- $singleNamespaceCounter := 0 }} {{- $controllerDeployment := dict }} + {{- $singleNamespaceControllerDeployments := dict }} {{- $managerServiceAccountNamespace := "" }} {{- range $index, $deployment := (lookup "apps/v1" "Deployment" "" "").items }} - {{- range $key, $val := $deployment.metadata.labels }} - {{- if and (eq $key "app.kubernetes.io/part-of") (eq $val "gha-runner-scale-set-controller") }} - {{- $counter = add $counter 1 }} - {{- $controllerDeployment = $deployment }} + {{- if kindIs "map" $deployment.metadata.labels }} + {{- if eq (get $deployment.metadata.labels "app.kubernetes.io/part-of") "gha-runner-scale-set-controller" }} + {{- if hasKey $deployment.metadata.labels "actions.github.com/controller-watch-single-namespace" }} + {{- $singleNamespaceCounter = add $singleNamespaceCounter 1 }} + {{- $_ := set $singleNamespaceControllerDeployments (get $deployment.metadata.labels "actions.github.com/controller-watch-single-namespace") $deployment}} + {{- else }} + {{- $multiNamespacesCounter = add $multiNamespacesCounter 1 }} + {{- $controllerDeployment = $deployment }} + {{- end }} {{- end }} {{- end }} {{- end }} - {{- if lt $counter 1 }} - {{- fail "No gha-runner-scale-set-controller deployment found using label (app.kubernetes.io/part-of=gha-runner-scale-set-controller), consider setting controllerServiceAccount.name to be explicit if you think the discovery is wrong." }} + {{- if and (eq $multiNamespacesCounter 0) (eq $singleNamespaceCounter 0) }} + {{- fail "No gha-runner-scale-set-controller deployment found using label (app.kubernetes.io/part-of=gha-runner-scale-set-controller). Consider setting controllerServiceAccount.name in values.yaml to be explicit if you think the discovery is wrong." }} {{- end }} - {{- if gt $counter 1 }} - {{- fail "More than one gha-runner-scale-set-controller deployment found using label (app.kubernetes.io/part-of=gha-runner-scale-set-controller), consider setting controllerServiceAccount.name to be explicit if you think the discovery is wrong." }} + {{- if and (gt $multiNamespacesCounter 0) (gt $singleNamespaceCounter 0) }} + {{- fail "Found both gha-runner-scale-set-controller installed with flags.watchSingleNamespace set and unset in cluster, this is not supported. Consider setting controllerServiceAccount.name in values.yaml to be explicit if you think the discovery is wrong." }} {{- end }} - {{- with $controllerDeployment.metadata }} - {{- $managerServiceAccountNamespace = (get $controllerDeployment.metadata.labels "actions.github.com/controller-service-account-namespace") }} + {{- if gt $multiNamespacesCounter 1 }} + {{- fail "More than one gha-runner-scale-set-controller deployment found using label (app.kubernetes.io/part-of=gha-runner-scale-set-controller). Consider setting controllerServiceAccount.name in values.yaml to be explicit if you think the discovery is wrong." }} + {{- end }} + {{- if eq $multiNamespacesCounter 1 }} + {{- with $controllerDeployment.metadata }} + {{- $managerServiceAccountNamespace = (get $controllerDeployment.metadata.labels "actions.github.com/controller-service-account-namespace") }} + {{- end }} + {{- else if gt $singleNamespaceCounter 0 }} + {{- if hasKey $singleNamespaceControllerDeployments .Release.Namespace }} + {{- $controllerDeployment = get $singleNamespaceControllerDeployments .Release.Namespace }} + {{- with $controllerDeployment.metadata }} + {{- $managerServiceAccountNamespace = (get $controllerDeployment.metadata.labels "actions.github.com/controller-service-account-namespace") }} + {{- end }} + {{- else }} + {{- fail "No gha-runner-scale-set-controller deployment that watch this namespace found using label (actions.github.com/controller-watch-single-namespace). Consider setting controllerServiceAccount.name in values.yaml to be explicit if you think the discovery is wrong." }} + {{- end }} {{- end }} {{- if eq $managerServiceAccountNamespace "" }} - {{- fail "No service account namespace found for gha-runner-scale-set-controller deployment using label (actions.github.com/controller-service-account-namespace), consider setting controllerServiceAccount.name to be explicit if you think the discovery is wrong." }} + {{- fail "No service account namespace found for gha-runner-scale-set-controller deployment using label (actions.github.com/controller-service-account-namespace), consider setting controllerServiceAccount.name in values.yaml to be explicit if you think the discovery is wrong." }} {{- end }} {{- $managerServiceAccountNamespace }} {{- end }} diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml index 97bb1aa7..7cb190f7 100644 --- a/charts/gha-runner-scale-set/values.yaml +++ b/charts/gha-runner-scale-set/values.yaml @@ -155,7 +155,7 @@ containerMode: ## the following is required when containerMode.type=kubernetes kubernetesModeWorkVolumeClaim: accessModes: ["ReadWriteOnce"] - # For testing, use https://github.com/rancher/local-path-provisioner to provide dynamic provision volume + # For local testing, use https://github.com/openebs/dynamic-localpv-provisioner/blob/develop/docs/quickstart.md to provide dynamic provision volume with storageClassName: openebs-hostpath # TODO: remove before release storageClassName: "dynamic-blob-storage" resources: diff --git a/main.go b/main.go index d91ada77..ac9a79c5 100644 --- a/main.go +++ b/main.go @@ -37,6 +37,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" // +kubebuilder:scaffold:imports ) @@ -90,6 +91,7 @@ func main() { namespace string logLevel string logFormat string + watchSingleNamespace string autoScalerImagePullSecrets stringSlice @@ -126,6 +128,7 @@ func main() { flag.DurationVar(&syncPeriod, "sync-period", 1*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled.") flag.Var(&commonRunnerLabels, "common-runner-labels", "Runner labels in the K1=V1,K2=V2,... format that are inherited all the runners created by the controller. See https://github.com/actions/actions-runner-controller/issues/321 for more information") flag.StringVar(&namespace, "watch-namespace", "", "The namespace to watch for custom resources. Set to empty for letting it watch for all namespaces.") + flag.StringVar(&watchSingleNamespace, "watch-single-namespace", "", "Restrict to watch for custom resources in a single namespace.") flag.StringVar(&logLevel, "log-level", logging.LogLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`) flag.StringVar(&logFormat, "log-format", "text", `The log format. Valid options are "text" and "json". Defaults to "text"`) flag.BoolVar(&autoScalingRunnerSetOnly, "auto-scaling-runner-set-only", false, "Make controller only reconcile AutoRunnerScaleSet object.") @@ -149,13 +152,27 @@ func main() { ctrl.SetLogger(log) + managerNamespace := "" + var newCache cache.NewCacheFunc + if autoScalingRunnerSetOnly { // We don't support metrics for AutoRunnerScaleSet for now metricsAddr = "0" + + managerNamespace = os.Getenv("CONTROLLER_MANAGER_POD_NAMESPACE") + if managerNamespace == "" { + log.Error(err, "unable to obtain manager pod namespace") + os.Exit(1) + } + + if len(watchSingleNamespace) > 0 { + newCache = cache.MultiNamespacedCacheBuilder([]string{managerNamespace, watchSingleNamespace}) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, + NewCache: newCache, MetricsBindAddress: metricsAddr, LeaderElection: enableLeaderElection, LeaderElectionID: leaderElectionId, @@ -178,11 +195,6 @@ func main() { log.Error(err, "unable to obtain listener image") os.Exit(1) } - managerNamespace := os.Getenv("CONTROLLER_MANAGER_POD_NAMESPACE") - if managerNamespace == "" { - log.Error(err, "unable to obtain manager pod namespace") - os.Exit(1) - } actionsMultiClient := actions.NewMultiClient( "actions-runner-controller/"+build.Version,