diff --git a/.github/actions/e2e-arc-test/action.yaml b/.github/actions/e2e-arc-test/action.yaml deleted file mode 100644 index 65739102..00000000 --- a/.github/actions/e2e-arc-test/action.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: 'E2E ARC Test Action' -description: 'Includes common arc installation, setup and test file run' - -inputs: - github-token: - description: 'JWT generated with Github App inputs' - required: true - config-url: - description: "URL of the repo, org or enterprise where the runner scale sets will be registered" - required: true - docker-image-repo: - description: "Local docker image repo for testing" - required: true - docker-image-tag: - description: "Tag of ARC Docker image for testing" - required: true - -runs: - using: "composite" - steps: - - name: Install ARC - run: helm install arc --namespace "arc-systems" --create-namespace --set image.tag=${{ inputs.docker-image-tag }} --set image.repository=${{ inputs.docker-image-repo }} ./charts/gha-runner-scale-set-controller - shell: bash - - name: Get datetime - # We are using this value further in the runner installation to avoid runner name collision that are a risk with hard coded values. - # A datetime including the 3 nanoseconds are a good option for this and also adds to readability and runner sorting if needed. - run: echo "DATE_TIME=$(date +'%Y-%m-%d-%H-%M-%S-%3N')" >> $GITHUB_ENV - shell: bash - - name: Install runners - run: | - helm install "arc-runner-${{ env.DATE_TIME }}" \ - --namespace "arc-runners" \ - --create-namespace \ - --set githubConfigUrl="${{ inputs.config-url }}" \ - --set githubConfigSecret.github_token="${{ inputs.github-token }}" \ - ./charts/gha-runner-scale-set \ - --debug - kubectl get pods -A - shell: bash - - name: Test ARC scales pods up and down - run: | - export GITHUB_TOKEN="${{ inputs.github-token }}" - export DATE_TIME="${{ env.DATE_TIME }}" - go test ./test_e2e_arc -v - shell: bash diff --git a/.github/actions/setup-arc-e2e/action.yaml b/.github/actions/setup-arc-e2e/action.yaml new file mode 100644 index 00000000..1f533b16 --- /dev/null +++ b/.github/actions/setup-arc-e2e/action.yaml @@ -0,0 +1,65 @@ +name: 'Setup ARC E2E Test Action' +description: 'Build controller image, create kind cluster, load the image, and exchange ARC configure token.' + +inputs: + github-app-id: + description: 'GitHub App Id for exchange access token' + required: true + github-app-pk: + description: "GitHub App private key for exchange access token" + required: true + github-app-org: + description: 'The organization the GitHub App has installed on' + required: true + docker-image-name: + description: "Local docker image name for building" + required: true + docker-image-tag: + description: "Tag of ARC Docker image for building" + required: true + +outputs: + token: + description: 'Token to use for configure ARC' + value: ${{steps.config-token.outputs.token}} + +runs: + using: "composite" + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + # Pinning v0.9.1 for Buildx and BuildKit v0.10.6 + # BuildKit v0.11 which has a bug causing intermittent + # failures pushing images to GHCR + version: v0.9.1 + driver-opts: image=moby/buildkit:v0.10.6 + + - name: Build controller image + uses: docker/build-push-action@v3 + with: + file: Dockerfile + platforms: linux/amd64 + load: true + build-args: | + DOCKER_IMAGE_NAME=${{inputs.docker-image-name}} + VERSION=${{inputs.docker-image-tag}} + tags: | + ${{inputs.docker-image-name}}:${{inputs.docker-image-tag}} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Create Kind cluster and load image + shell: bash + run: | + PATH=$(go env GOPATH)/bin:$PATH + kind create cluster --name arc-e2e + kind load docker-image ${{inputs.docker-image-name}}:${{inputs.docker-image-tag}} --name arc-e2e + + - name: Get configure token + id: config-token + uses: peter-murray/workflow-application-token-action@8e1ba3bf1619726336414f1014e37f17fbadf1db + with: + application_id: ${{ inputs.github-app-id }} + application_private_key: ${{ inputs.github-app-pk }} + organization: ${{ inputs.github-app-org }} \ No newline at end of file diff --git a/.github/workflows/e2e-test-linux-vm.yaml b/.github/workflows/e2e-test-linux-vm.yaml index 08f575b7..88ba0dd1 100644 --- a/.github/workflows/e2e-test-linux-vm.yaml +++ b/.github/workflows/e2e-test-linux-vm.yaml @@ -5,47 +5,122 @@ on: branches: - master pull_request: + branches: + - master workflow_dispatch: + inputs: + target_org: + description: The org of the test repository. + required: true + default: actions-runner-controller + target_repo: + description: The repository to install the ARC. + required: true + default: arc_e2e_test_dummy env: TARGET_ORG: actions-runner-controller - CLUSTER_NAME: e2e-test - RUNNER_VERSION: 2.302.1 - IMAGE_REPO: "test/test-image" + TARGET_REPO: arc_e2e_test_dummy + IMAGE_NAME: "arc-test-image" + IMAGE_VERSION: "dev" jobs: - setup-steps: - runs-on: [ubuntu-latest] + default-setup: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Add env variables + + - name: Resolve inputs + id: resolved_inputs run: | - TAG=$(echo "0.0.$GITHUB_SHA") - echo "TAG=$TAG" >> $GITHUB_ENV - echo "IMAGE=$IMAGE_REPO:$TAG" >> $GITHUB_ENV - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + TARGET_ORG="${{env.TARGET_ORG}}" + TARGET_REPO="${{env.TARGET_REPO}}" + if [ ! -z "${{inputs.target_org}}" ]; then + TARGET_ORG="${{inputs.target_org}}" + fi + if [ ! -z "${{inputs.target_repo}}" ]; then + TARGET_REPO="${{inputs.target_repo}}" + fi + echo "TARGET_ORG=$TARGET_ORG" >> $GITHUB_OUTPUT + echo "TARGET_REPO=$TARGET_REPO" >> $GITHUB_OUTPUT + + - uses: ./.github/actions/setup-arc-e2e + id: setup with: - version: latest - - name: Docker Build Test Image + github-app-id: ${{secrets.ACTIONS_ACCESS_APP_ID}} + github-app-pk: ${{secrets.ACTIONS_ACCESS_PK}} + github-app-org: ${{steps.resolved_inputs.outputs.TARGET_ORG}} + docker-image-name: ${{env.IMAGE_NAME}} + docker-image-tag: ${{env.IMAGE_VERSION}} + + - name: Install gha-runner-scale-set-controller + id: install_arc_controller run: | - DOCKER_CLI_EXPERIMENTAL=enabled DOCKER_BUILDKIT=1 docker buildx build --build-arg RUNNER_VERSION=$RUNNER_VERSION --build-arg TAG=$TAG -t $IMAGE . --load - - name: Create Kind cluster + helm install arc \ + --namespace "arc-systems" \ + --create-namespace \ + --set image.repository=${{ env.IMAGE_NAME }} \ + --set image.tag=${{ env.IMAGE_VERSION }} \ + ./charts/gha-runner-scale-set-controller \ + --debug + count=0 + while true; do + POD_NAME=$(kubectl get pods -n arc-systems -l app.kubernetes.io/name=gha-runner-scale-set-controller -o name) + if [ -n "$POD_NAME" ]; then + echo "Pod found: $POD_NAME" + break + fi + if [ "$count" -ge 10 ]; then + echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-runner-scale-set-controller" + exit 1 + fi + sleep 1 + done + kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l app.kubernetes.io/name=gha-runner-scale-set-controller + kubectl get pod -n arc-systems + kubectl describe deployment arc-gha-runner-scale-set-controller -n arc-systems + + - name: Install gha-runner-scale-set + id: install_arc run: | - PATH=$(go env GOPATH)/bin:$PATH - kind create cluster --name $CLUSTER_NAME - - name: Load Image to Kind Cluster - run: kind load docker-image $IMAGE --name $CLUSTER_NAME - - name: Get Token - id: get_workflow_token - uses: peter-murray/workflow-application-token-action@8e1ba3bf1619726336414f1014e37f17fbadf1db - with: - application_id: ${{ secrets.ACTIONS_ACCESS_APP_ID }} - application_private_key: ${{ secrets.ACTIONS_ACCESS_PK }} - organization: ${{ env.TARGET_ORG }} - - uses: ./.github/actions/e2e-arc-test - with: - github-token: ${{ steps.get_workflow_token.outputs.token }} - config-url: "https://github.com/actions-runner-controller/arc_e2e_test_dummy" - docker-image-repo: $IMAGE_REPO - docker-image-tag: $TAG + ARC_NAME=arc-runner-${{github.job}}-$(date +'%M-%S')-$(($RANDOM % 100 + 1)) + helm install "$ARC_NAME" \ + --namespace "arc-runners" \ + --create-namespace \ + --set githubConfigUrl="https://github.com/${{ steps.resolved_inputs.outputs.TARGET_ORG }}/${{steps.resolved_inputs.outputs.TARGET_REPO}}" \ + --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \ + ./charts/gha-runner-scale-set \ + --debug + echo "ARC_NAME=$ARC_NAME" >> $GITHUB_OUTPUT + count=0 + while true; do + POD_NAME=$(kubectl get pods -n arc-systems -l auto-scaling-runner-set-name=$ARC_NAME -o name) + if [ -n "$POD_NAME" ]; then + echo "Pod found: $POD_NAME" + break + fi + if [ "$count" -ge 10 ]; then + echo "Timeout waiting for listener pod with label auto-scaling-runner-set-name=$ARC_NAME" + exit 1 + fi + sleep 1 + done + kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l auto-scaling-runner-set-name=$ARC_NAME + kubectl get pod -n arc-systems + + - name: Test ARC scales pods up and down + run: | + export GITHUB_TOKEN="${{ steps.setup.outputs.token }}" + export ARC_NAME="${{ steps.install_arc.outputs.ARC_NAME }}" + go test ./test_e2e_arc -v + + - name: Uninstall gha-runner-scale-set + if: always() && steps.install_arc.outcome == 'success' + run: | + helm uninstall ${{ steps.install_arc.outputs.ARC_NAME }} --namespace arc-runners + kubectl wait --timeout=10s --for=delete AutoScalingRunnerSet -n demo -l app.kubernetes.io/instance=${{ steps.install_arc.outputs.ARC_NAME }} + + - name: Dump gha-runner-scale-set-controller logs + if: always() && steps.install_arc_controller.outcome == 'success' + run: | + kubectl logs deployment/arc-gha-runner-scale-set-controller -n arc-systems \ No newline at end of file diff --git a/controllers/actions.github.com/ephemeralrunnerset_controller_test.go b/controllers/actions.github.com/ephemeralrunnerset_controller_test.go index 4459a3f3..fecf1b00 100644 --- a/controllers/actions.github.com/ephemeralrunnerset_controller_test.go +++ b/controllers/actions.github.com/ephemeralrunnerset_controller_test.go @@ -1141,12 +1141,13 @@ var _ = Describe("Test EphemeralRunnerSet controller with custom root CA", func( err = k8sClient.Status().Patch(ctx, runner, client.MergeFrom(&runnerList.Items[0])) Expect(err).NotTo(HaveOccurred(), "failed to update ephemeral runner status") - updatedRunnerSet := new(actionsv1alpha1.EphemeralRunnerSet) - err = k8sClient.Get(ctx, client.ObjectKey{Namespace: ephemeralRunnerSet.Namespace, Name: ephemeralRunnerSet.Name}, updatedRunnerSet) + currentRunnerSet := new(actionsv1alpha1.EphemeralRunnerSet) + err = k8sClient.Get(ctx, client.ObjectKey{Namespace: ephemeralRunnerSet.Namespace, Name: ephemeralRunnerSet.Name}, currentRunnerSet) Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") + updatedRunnerSet := currentRunnerSet.DeepCopy() updatedRunnerSet.Spec.Replicas = 0 - err = k8sClient.Update(ctx, updatedRunnerSet) + err = k8sClient.Patch(ctx, updatedRunnerSet, client.MergeFrom(currentRunnerSet)) Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") // wait for server to be called diff --git a/test_e2e_arc/arc_jobs_test.go b/test_e2e_arc/arc_jobs_test.go index 02db92f6..8b3ca9f8 100644 --- a/test_e2e_arc/arc_jobs_test.go +++ b/test_e2e_arc/arc_jobs_test.go @@ -92,11 +92,11 @@ func TestARCJobs(t *testing.T) { ) t.Run("Get available pods during job run", func(t *testing.T) { c := http.Client{} - dateTime := os.Getenv("DATE_TIME") + targetArcName := os.Getenv("ARC_NAME") // We are triggering manually a workflow that already exists in the repo. // This workflow is expected to spin up a number of runner pods matching the runners value set in podCountsByType. - url := "https://api.github.com/repos/actions-runner-controller/arc_e2e_test_dummy/actions/workflows/e2e-test-dispatch-workflow.yaml/dispatches" - jsonStr := []byte(fmt.Sprintf(`{"ref":"main", "inputs":{"date_time":"%s"}}`, dateTime)) + url := "https://api.github.com/repos/actions-runner-controller/arc_e2e_test_dummy/actions/workflows/arc-test-workflow.yaml/dispatches" + jsonStr := []byte(fmt.Sprintf(`{"ref":"main", "inputs":{"arc_name":"%s"}}`, targetArcName)) req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr)) if err != nil {