Update E2E test workflow. (#2395)

.github/actions/e2e-arc-test/action.yaml

@@ -1,45 +0,0 @@
-name: 'E2E ARC Test Action'
-description: 'Includes common arc installation, setup and test file run'
-
-inputs:
-  github-token:
-    description: 'JWT generated with Github App inputs'
-    required: true
-  config-url:
-    description: "URL of the repo, org or enterprise where the runner scale sets will be registered"
-    required: true
-  docker-image-repo:
-    description: "Local docker image repo for testing"
-    required: true
-  docker-image-tag:
-    description: "Tag of ARC Docker image for testing"
-    required: true
-
-runs:
-  using: "composite"
-  steps:
-    - name: Install ARC
-      run: helm install arc --namespace "arc-systems" --create-namespace --set image.tag=${{ inputs.docker-image-tag }}  --set image.repository=${{ inputs.docker-image-repo }} ./charts/gha-runner-scale-set-controller
-      shell: bash
-    - name: Get datetime
-      # We are using this value further in the runner installation to avoid runner name collision that are a risk with hard coded values.
-      # A datetime including the 3 nanoseconds are a good option for this and also adds to readability and runner sorting if needed.
-      run: echo "DATE_TIME=$(date +'%Y-%m-%d-%H-%M-%S-%3N')" >> $GITHUB_ENV
-      shell: bash
-    - name: Install runners
-      run: |
-          helm install "arc-runner-${{ env.DATE_TIME }}" \
-          --namespace "arc-runners" \
-          --create-namespace \
-          --set githubConfigUrl="${{ inputs.config-url }}" \
-          --set githubConfigSecret.github_token="${{ inputs.github-token }}" \
-          ./charts/gha-runner-scale-set \
-          --debug
-          kubectl get pods -A
-      shell: bash
-    - name: Test ARC scales pods up and down
-      run: |
-          export GITHUB_TOKEN="${{ inputs.github-token }}"
-          export DATE_TIME="${{ env.DATE_TIME }}"
-          go test ./test_e2e_arc -v
-      shell: bash

.github/actions/setup-arc-e2e/action.yaml

@@ -0,0 +1,65 @@
+name: 'Setup ARC E2E Test Action'
+description: 'Build controller image, create kind cluster, load the image, and exchange ARC configure token.'
+
+inputs:
+  github-app-id:
+    description: 'GitHub App Id for exchange access token'
+    required: true
+  github-app-pk:
+    description: "GitHub App private key for exchange access token"
+    required: true
+  github-app-org:
+    description: 'The organization the GitHub App has installed on'
+    required: true
+  docker-image-name:
+    description: "Local docker image name for building"
+    required: true
+  docker-image-tag:
+    description: "Tag of ARC Docker image for building"
+    required: true
+
+outputs:
+  token:
+    description: 'Token to use for configure ARC'
+    value: ${{steps.config-token.outputs.token}}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+      with:
+          # Pinning v0.9.1 for Buildx and BuildKit v0.10.6
+          # BuildKit v0.11 which has a bug causing intermittent 
+          # failures pushing images to GHCR
+          version: v0.9.1
+          driver-opts: image=moby/buildkit:v0.10.6
+
+    - name: Build controller image
+      uses: docker/build-push-action@v3
+      with:
+        file: Dockerfile
+        platforms: linux/amd64
+        load: true
+        build-args: |
+          DOCKER_IMAGE_NAME=${{inputs.docker-image-name}}
+          VERSION=${{inputs.docker-image-tag}} 
+        tags: |
+          ${{inputs.docker-image-name}}:${{inputs.docker-image-tag}}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+
+    - name: Create Kind cluster and load image
+      shell: bash
+      run: |
+        PATH=$(go env GOPATH)/bin:$PATH
+        kind create cluster --name arc-e2e
+        kind load docker-image ${{inputs.docker-image-name}}:${{inputs.docker-image-tag}} --name arc-e2e
+
+    - name: Get configure token
+      id: config-token
+      uses: peter-murray/workflow-application-token-action@8e1ba3bf1619726336414f1014e37f17fbadf1db
+      with:
+        application_id: ${{ inputs.github-app-id }}
+        application_private_key: ${{ inputs.github-app-pk }}
+        organization: ${{ inputs.github-app-org }}

.github/workflows/e2e-test-linux-vm.yaml

@@ -5,47 +5,122 @@ on:
     branches:
       - master
   pull_request:
+      branches:
+      - master
   workflow_dispatch:
+    inputs:
+      target_org:
+        description: The org of the test repository.
+        required: true
+        default: actions-runner-controller
+      target_repo:
+        description: The repository to install the ARC.
+        required: true
+        default: arc_e2e_test_dummy
 
 env:
   TARGET_ORG: actions-runner-controller
-  CLUSTER_NAME: e2e-test
+  TARGET_REPO: arc_e2e_test_dummy
-  RUNNER_VERSION: 2.302.1
+  IMAGE_NAME: "arc-test-image"
-  IMAGE_REPO: "test/test-image"
+  IMAGE_VERSION: "dev"
   
 jobs:
-  setup-steps:
+  default-setup:
-    runs-on: [ubuntu-latest]
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Add env variables
+
+      - name: Resolve inputs
+        id: resolved_inputs
         run: |
-            TAG=$(echo "0.0.$GITHUB_SHA")
+          TARGET_ORG="${{env.TARGET_ORG}}"
-            echo "TAG=$TAG" >> $GITHUB_ENV
+          TARGET_REPO="${{env.TARGET_REPO}}"
-            echo "IMAGE=$IMAGE_REPO:$TAG" >> $GITHUB_ENV
+          if [ ! -z "${{inputs.target_org}}" ]; then
-      - name: Set up Docker Buildx
+            TARGET_ORG="${{inputs.target_org}}"
-        uses: docker/setup-buildx-action@v2
+          fi
+          if [ ! -z "${{inputs.target_repo}}" ]; then
+            TARGET_REPO="${{inputs.target_repo}}"
+          fi
+          echo "TARGET_ORG=$TARGET_ORG" >> $GITHUB_OUTPUT
+          echo "TARGET_REPO=$TARGET_REPO" >> $GITHUB_OUTPUT
+
+      - uses: ./.github/actions/setup-arc-e2e
+        id: setup
         with:
-          version: latest
+          github-app-id: ${{secrets.ACTIONS_ACCESS_APP_ID}}
-      - name: Docker Build Test Image
+          github-app-pk: ${{secrets.ACTIONS_ACCESS_PK}}
+          github-app-org: ${{steps.resolved_inputs.outputs.TARGET_ORG}}
+          docker-image-name: ${{env.IMAGE_NAME}}
+          docker-image-tag: ${{env.IMAGE_VERSION}}
+
+      - name: Install gha-runner-scale-set-controller
+        id: install_arc_controller
         run: |
-            DOCKER_CLI_EXPERIMENTAL=enabled DOCKER_BUILDKIT=1 docker buildx build --build-arg RUNNER_VERSION=$RUNNER_VERSION --build-arg TAG=$TAG -t $IMAGE . --load
+          helm install arc \
-      - name: Create Kind cluster
+          --namespace "arc-systems" \
+          --create-namespace \
+          --set image.repository=${{ env.IMAGE_NAME }} \
+          --set image.tag=${{ env.IMAGE_VERSION }} \
+          ./charts/gha-runner-scale-set-controller \
+          --debug
+          count=0
+          while true; do
+            POD_NAME=$(kubectl get pods -n arc-systems -l app.kubernetes.io/name=gha-runner-scale-set-controller -o name)
+            if [ -n "$POD_NAME" ]; then
+              echo "Pod found: $POD_NAME"
+              break
+            fi
+            if [ "$count" -ge 10 ]; then
+              echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-runner-scale-set-controller"
+              exit 1
+            fi
+            sleep 1
+          done
+          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l app.kubernetes.io/name=gha-runner-scale-set-controller
+          kubectl get pod -n arc-systems
+          kubectl describe deployment arc-gha-runner-scale-set-controller -n arc-systems
+
+      - name: Install gha-runner-scale-set
+        id: install_arc
         run: |
-          PATH=$(go env GOPATH)/bin:$PATH
+          ARC_NAME=arc-runner-${{github.job}}-$(date +'%M-%S')-$(($RANDOM % 100 + 1))
-          kind create cluster --name $CLUSTER_NAME
+          helm install "$ARC_NAME" \
-      - name: Load Image to Kind Cluster
+          --namespace "arc-runners" \
-        run: kind load docker-image $IMAGE --name $CLUSTER_NAME
+          --create-namespace \
-      - name: Get Token
+          --set githubConfigUrl="https://github.com/${{ steps.resolved_inputs.outputs.TARGET_ORG }}/${{steps.resolved_inputs.outputs.TARGET_REPO}}" \
-        id: get_workflow_token
+          --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \
-        uses: peter-murray/workflow-application-token-action@8e1ba3bf1619726336414f1014e37f17fbadf1db
+          ./charts/gha-runner-scale-set \
-        with:
+          --debug
-          application_id: ${{ secrets.ACTIONS_ACCESS_APP_ID }}
+          echo "ARC_NAME=$ARC_NAME" >> $GITHUB_OUTPUT
-          application_private_key: ${{ secrets.ACTIONS_ACCESS_PK }}
+          count=0
-          organization: ${{ env.TARGET_ORG }}
+          while true; do
-      - uses: ./.github/actions/e2e-arc-test
+            POD_NAME=$(kubectl get pods -n arc-systems -l auto-scaling-runner-set-name=$ARC_NAME -o name)
-        with:
+            if [ -n "$POD_NAME" ]; then
-          github-token: ${{ steps.get_workflow_token.outputs.token }}
+              echo "Pod found: $POD_NAME"
-          config-url: "https://github.com/actions-runner-controller/arc_e2e_test_dummy"
+              break
-          docker-image-repo: $IMAGE_REPO
+            fi
-          docker-image-tag: $TAG
+            if [ "$count" -ge 10 ]; then
+              echo "Timeout waiting for listener pod with label auto-scaling-runner-set-name=$ARC_NAME"
+              exit 1
+            fi
+            sleep 1
+          done
+          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l auto-scaling-runner-set-name=$ARC_NAME
+          kubectl get pod -n arc-systems
+
+      - name: Test ARC scales pods up and down
+        run: |
+          export GITHUB_TOKEN="${{ steps.setup.outputs.token }}"
+          export ARC_NAME="${{ steps.install_arc.outputs.ARC_NAME }}"
+          go test ./test_e2e_arc -v
+
+      - name: Uninstall gha-runner-scale-set
+        if: always() && steps.install_arc.outcome == 'success'
+        run: |
+          helm uninstall ${{ steps.install_arc.outputs.ARC_NAME }} --namespace arc-runners
+          kubectl wait --timeout=10s --for=delete AutoScalingRunnerSet -n demo -l app.kubernetes.io/instance=${{ steps.install_arc.outputs.ARC_NAME }}
+
+      - name: Dump gha-runner-scale-set-controller logs
+        if: always() && steps.install_arc_controller.outcome == 'success'
+        run: |
+          kubectl logs deployment/arc-gha-runner-scale-set-controller -n arc-systems

controllers/actions.github.com/ephemeralrunnerset_controller_test.go

@@ -1141,12 +1141,13 @@ var _ = Describe("Test EphemeralRunnerSet controller with custom root CA", func(
 		err = k8sClient.Status().Patch(ctx, runner, client.MergeFrom(&runnerList.Items[0]))
 		Expect(err).NotTo(HaveOccurred(), "failed to update ephemeral runner status")
 
-		updatedRunnerSet := new(actionsv1alpha1.EphemeralRunnerSet)
+		currentRunnerSet := new(actionsv1alpha1.EphemeralRunnerSet)
-		err = k8sClient.Get(ctx, client.ObjectKey{Namespace: ephemeralRunnerSet.Namespace, Name: ephemeralRunnerSet.Name}, updatedRunnerSet)
+		err = k8sClient.Get(ctx, client.ObjectKey{Namespace: ephemeralRunnerSet.Namespace, Name: ephemeralRunnerSet.Name}, currentRunnerSet)
 		Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
 
+		updatedRunnerSet := currentRunnerSet.DeepCopy()
 		updatedRunnerSet.Spec.Replicas = 0
-		err = k8sClient.Update(ctx, updatedRunnerSet)
+		err = k8sClient.Patch(ctx, updatedRunnerSet, client.MergeFrom(currentRunnerSet))
 		Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
 
 		// wait for server to be called

test_e2e_arc/arc_jobs_test.go

@@ -92,11 +92,11 @@ func TestARCJobs(t *testing.T) {
 	)
 	t.Run("Get available pods during job run", func(t *testing.T) {
 		c := http.Client{}
-		dateTime := os.Getenv("DATE_TIME")
+		targetArcName := os.Getenv("ARC_NAME")
 		// We are triggering manually a workflow that already exists in the repo.
 		// This workflow is expected to spin up a number of runner pods matching the runners value set in podCountsByType.
-		url := "https://api.github.com/repos/actions-runner-controller/arc_e2e_test_dummy/actions/workflows/e2e-test-dispatch-workflow.yaml/dispatches"
+		url := "https://api.github.com/repos/actions-runner-controller/arc_e2e_test_dummy/actions/workflows/arc-test-workflow.yaml/dispatches"
-		jsonStr := []byte(fmt.Sprintf(`{"ref":"main", "inputs":{"date_time":"%s"}}`, dateTime))
+		jsonStr := []byte(fmt.Sprintf(`{"ref":"main", "inputs":{"arc_name":"%s"}}`, targetArcName))
 
 		req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
 		if err != nil {