diff --git a/.github/workflows/gha-e2e-tests.yaml b/.github/workflows/gha-e2e-tests.yaml index bbf14631..f35c7a2e 100644 --- a/.github/workflows/gha-e2e-tests.yaml +++ b/.github/workflows/gha-e2e-tests.yaml @@ -26,6 +26,30 @@ concurrency: cancel-in-progress: true jobs: + default-setup-v2: + runs-on: ubuntu-latest + timeout-minutes: 20 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + env: + TARGET_ORG: ${{ env.TARGET_ORG }} + TARGET_REPO: ${{ env.TARGET_REPO }} + steps: + - uses: actions/checkout@v5 + with: + ref: ${{github.head_ref}} + + - name: Get configure token + id: config-token + uses: peter-murray/workflow-application-token-action@dc0413987a085fa17d19df9e47d4677cf81ffef3 + with: + application_id: ${{ secrets.E2E_TESTS_ACCESS_APP_ID }} + application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} + organization: ${{ env.TARGET_ORG }} + + - name: Run default setup test + run: hack/e2e-test.sh default-setup + shell: bash + default-setup: runs-on: ubuntu-latest timeout-minutes: 20 @@ -117,6 +141,31 @@ jobs: arc-namespace: "arc-runners" arc-controller-namespace: "arc-systems" + single-namespace-setup-v2: + runs-on: ubuntu-latest + timeout-minutes: 20 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + env: + TARGET_ORG: ${{ env.TARGET_ORG }} + TARGET_REPO: ${{ env.TARGET_REPO }} + + steps: + - uses: actions/checkout@v5 + with: + ref: ${{github.head_ref}} + + - name: Get configure token + id: config-token + uses: peter-murray/workflow-application-token-action@dc0413987a085fa17d19df9e47d4677cf81ffef3 + with: + application_id: ${{ secrets.E2E_TESTS_ACCESS_APP_ID }} + application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} + organization: ${{ env.TARGET_ORG }} + + - name: Run single namespace setup test + run: hack/e2e-test.sh single-namespace-setup + shell: bash + single-namespace-setup: runs-on: ubuntu-latest timeout-minutes: 20 @@ -210,6 +259,31 @@ jobs: arc-namespace: "arc-runners" arc-controller-namespace: "arc-systems" + dind-mode-setup-v2: + runs-on: ubuntu-latest + timeout-minutes: 20 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + env: + TARGET_ORG: ${{ env.TARGET_ORG }} + TARGET_REPO: ${{ env.TARGET_REPO }} + + steps: + - uses: actions/checkout@v5 + with: + ref: ${{github.head_ref}} + + - name: Get configure token + id: config-token + uses: peter-murray/workflow-application-token-action@dc0413987a085fa17d19df9e47d4677cf81ffef3 + with: + application_id: ${{ secrets.E2E_TESTS_ACCESS_APP_ID }} + application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} + organization: ${{ env.TARGET_ORG }} + + - name: Run dind mode setup test + run: hack/e2e-test.sh dind-mode-setup + shell: bash + dind-mode-setup: runs-on: ubuntu-latest timeout-minutes: 20 @@ -302,6 +376,31 @@ jobs: arc-namespace: "arc-runners" arc-controller-namespace: "arc-systems" + kubernetes-mode-setup-v2: + runs-on: ubuntu-latest + timeout-minutes: 20 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + env: + TARGET_ORG: ${{ env.TARGET_ORG }} + TARGET_REPO: ${{ env.TARGET_REPO }} + + steps: + - uses: actions/checkout@v5 + with: + ref: ${{github.head_ref}} + + - name: Get configure token + id: config-token + uses: peter-murray/workflow-application-token-action@dc0413987a085fa17d19df9e47d4677cf81ffef3 + with: + application_id: ${{ secrets.E2E_TESTS_ACCESS_APP_ID }} + application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} + organization: ${{ env.TARGET_ORG }} + + - name: Run kubernetes mode setup test + run: hack/e2e-test.sh kubernetes-mode-setup + shell: bash + kubernetes-mode-setup: runs-on: ubuntu-latest timeout-minutes: 20 @@ -403,6 +502,31 @@ jobs: arc-namespace: "arc-runners" arc-controller-namespace: "arc-systems" + auth-proxy-setup-v2: + runs-on: ubuntu-latest + timeout-minutes: 20 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + env: + TARGET_ORG: ${{ env.TARGET_ORG }} + TARGET_REPO: ${{ env.TARGET_REPO }} + + steps: + - uses: actions/checkout@v5 + with: + ref: ${{github.head_ref}} + + - name: Get configure token + id: config-token + uses: peter-murray/workflow-application-token-action@dc0413987a085fa17d19df9e47d4677cf81ffef3 + with: + application_id: ${{ secrets.E2E_TESTS_ACCESS_APP_ID }} + application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} + organization: ${{ env.TARGET_ORG }} + + - name: Run single namespace setup test + run: hack/e2e-test.sh single-namespace-setup + shell: bash + auth-proxy-setup: runs-on: ubuntu-latest timeout-minutes: 20 @@ -506,6 +630,31 @@ jobs: arc-namespace: "arc-runners" arc-controller-namespace: "arc-systems" + anonymous-proxy-setup-v2: + runs-on: ubuntu-latest + timeout-minutes: 20 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + env: + TARGET_ORG: ${{ env.TARGET_ORG }} + TARGET_REPO: ${{ env.TARGET_REPO }} + + steps: + - uses: actions/checkout@v5 + with: + ref: ${{github.head_ref}} + + - name: Get configure token + id: config-token + uses: peter-murray/workflow-application-token-action@dc0413987a085fa17d19df9e47d4677cf81ffef3 + with: + application_id: ${{ secrets.E2E_TESTS_ACCESS_APP_ID }} + application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} + organization: ${{ env.TARGET_ORG }} + + - name: Run anonymous proxy setup test + run: hack/e2e-test.sh anonymous-proxy-setup + shell: bash + anonymous-proxy-setup: runs-on: ubuntu-latest timeout-minutes: 20 @@ -603,6 +752,31 @@ jobs: arc-namespace: "arc-runners" arc-controller-namespace: "arc-systems" + self-signed-ca-setup-v2: + runs-on: ubuntu-latest + timeout-minutes: 20 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + env: + TARGET_ORG: ${{ env.TARGET_ORG }} + TARGET_REPO: ${{ env.TARGET_REPO }} + + steps: + - uses: actions/checkout@v5 + with: + ref: ${{github.head_ref}} + + - name: Get configure token + id: config-token + uses: peter-murray/workflow-application-token-action@dc0413987a085fa17d19df9e47d4677cf81ffef3 + with: + application_id: ${{ secrets.E2E_TESTS_ACCESS_APP_ID }} + application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} + organization: ${{ env.TARGET_ORG }} + + - name: Run self signed CA setup test + run: hack/e2e-test.sh self-signed-ca-setup + shell: bash + self-signed-ca-setup: runs-on: ubuntu-latest timeout-minutes: 20 @@ -725,6 +899,31 @@ jobs: arc-namespace: "arc-runners" arc-controller-namespace: "arc-systems" + update-strategy-tests-v2: + runs-on: ubuntu-latest + timeout-minutes: 20 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + env: + TARGET_ORG: ${{ env.TARGET_ORG }} + TARGET_REPO: ${{ env.TARGET_REPO }} + + steps: + - uses: actions/checkout@v5 + with: + ref: ${{github.head_ref}} + + - name: Get configure token + id: config-token + uses: peter-murray/workflow-application-token-action@dc0413987a085fa17d19df9e47d4677cf81ffef3 + with: + application_id: ${{ secrets.E2E_TESTS_ACCESS_APP_ID }} + application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} + organization: ${{ env.TARGET_ORG }} + + - name: Run update strategy test + run: hack/e2e-test.sh update-strategy + shell: bash + update-strategy-tests: runs-on: ubuntu-latest timeout-minutes: 20 @@ -897,6 +1096,31 @@ jobs: kubectl wait --timeout=10s --for=delete AutoScalingRunnerSet -n "${{ steps.install_arc.outputs.ARC_NAME }}" -l app.kubernetes.io/instance="${{ steps.install_arc.outputs.ARC_NAME }}" kubectl logs deployment/arc-gha-rs-controller -n "arc-systems" + init-with-min-runners-v2: + runs-on: ubuntu-latest + timeout-minutes: 20 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + env: + TARGET_ORG: ${{ env.TARGET_ORG }} + TARGET_REPO: ${{ env.TARGET_REPO }} + + steps: + - uses: actions/checkout@v5 + with: + ref: ${{github.head_ref}} + + - name: Get configure token + id: config-token + uses: peter-murray/workflow-application-token-action@dc0413987a085fa17d19df9e47d4677cf81ffef3 + with: + application_id: ${{ secrets.E2E_TESTS_ACCESS_APP_ID }} + application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }} + organization: ${{ env.TARGET_ORG }} + + - name: Run init with min runners test + run: hack/e2e-test.sh init-with-min-runners + shell: bash + init-with-min-runners: runs-on: ubuntu-latest timeout-minutes: 20 diff --git a/Makefile b/Makefile index ed96d07a..c7ecf843 100644 --- a/Makefile +++ b/Makefile @@ -210,8 +210,6 @@ docker-buildx: docker buildx create --platform ${PLATFORMS} --name container-builder --use;\ fi docker buildx build --platform ${PLATFORMS} \ - --build-arg RUNNER_VERSION=${RUNNER_VERSION} \ - --build-arg DOCKER_VERSION=${DOCKER_VERSION} \ --build-arg VERSION=${VERSION} \ --build-arg COMMIT_SHA=${COMMIT_SHA} \ -t "${DOCKER_IMAGE_NAME}:${VERSION}" \ @@ -297,6 +295,10 @@ acceptance/runner/startup: e2e: go test -count=1 -v -timeout 600s -run '^TestE2E$$' ./test/e2e +.PHONY: gha-e2e +gha-e2e: + bash hack/e2e-test.sh + # Upload release file to GitHub. github-release: release ghr ${VERSION} release/ diff --git a/hack/e2e-test.sh b/hack/e2e-test.sh new file mode 100755 index 00000000..d532662e --- /dev/null +++ b/hack/e2e-test.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +DIR="$(dirname "${BASH_SOURCE[0]}")" + +DIR="$(realpath "${DIR}")" + +TEST_DIR="$(realpath "${DIR}/../test/actions.github.com")" + +export PLATFORMS="linux/amd64" + +TARGETS=() + +function set_targets() { + local cases="$(find "${TEST_DIR}" -name '*.test.sh' | sed "s#^${TEST_DIR}/##g" )" + + mapfile -t TARGETS < <(echo "${cases}") + + echo $TARGETS +} + +function env_test() { + if [[ -z "${GITHUB_TOKEN}" ]]; then + echo "Error: GITHUB_TOKEN is not set" + exit 1 + fi + + if [[ -z "${TARGET_ORG}" ]]; then + echo "Error: TARGET_ORG is not set" + exit 1 + fi + + if [[ -z "${TARGET_REPO}" ]]; then + echo "Error: TARGET_REPO is not set" + exit 1 + fi +} + +function usage() { + echo "Usage: $0 [test_name]" + echo " test_name: the name of the test to run" + echo " if not specified, all tests will be run" + echo " test_name should be the name of the test file without the .test.sh suffix" + echo "" + exit 1 +} + +function main() { + local failed=() + + env_test + + if [[ -z "${1}" ]]; then + echo "Running all tests" + set_targets + elif [[ -f "${TEST_DIR}/${1}.test.sh" ]]; then + echo "Running test ${1}" + TARGETS=("${1}.test.sh") + else + usage + fi + + for target in "${TARGETS[@]}"; do + echo "============================================================" + test="${TEST_DIR}/${target}" + if [[ ! -x "${test}" ]]; then + echo "Error: test ${test} is not executable or not found" + failed+=("${test}") + continue + fi + + echo "Running test ${target}" + if ! "${test}"; then + failed+=("${target}") + echo "---------------------------------" + echo "FAILED: ${target}" + else + echo "---------------------------------" + echo "PASSED: ${target}" + fi + echo "============================================================" + done + + if [[ "${#failed[@]}" -gt 0 ]]; then + echo "Failed tests:" + for fail in "${failed[@]}"; do + echo " ${fail}" + done + exit 1 + fi +} + +main $@ diff --git a/test/actions.github.com/anonymous-proxy-setup.squid.yaml b/test/actions.github.com/anonymous-proxy-setup.squid.yaml new file mode 100644 index 00000000..b56bd181 --- /dev/null +++ b/test/actions.github.com/anonymous-proxy-setup.squid.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: squid +spec: + replicas: 1 + selector: + matchLabels: + app: squid + template: + metadata: + labels: + app: squid + spec: + containers: + - name: squid + image: ubuntu/squid:latest + ports: + - containerPort: 3128 +--- +apiVersion: v1 +kind: Service +metadata: + name: squid +spec: + selector: + app: squid + ports: + - protocol: TCP + port: 3128 + targetPort: 3128 diff --git a/test/actions.github.com/anonymous-proxy-setup.test.sh b/test/actions.github.com/anonymous-proxy-setup.test.sh new file mode 100755 index 00000000..fb79fb10 --- /dev/null +++ b/test/actions.github.com/anonymous-proxy-setup.test.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +set -euo pipefail + +DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +source "${DIR}/helper.sh" + +SCALE_SET_NAME="anonymous-proxy-$(date +'%M%S')$(((RANDOM + 100) % 100 + 1))" +SCALE_SET_NAMESPACE="arc-runners" +WORKFLOW_FILE="arc-test-workflow.yaml" +ARC_NAME="arc" +ARC_NAMESPACE="arc-systems" + +function install_arc() { + echo "Creating namespace ${ARC_NAMESPACE}" + kubectl create namespace "${SCALE_SET_NAMESPACE}" + + echo "Installing ARC" + helm install "${ARC_NAME}" \ + --namespace "${ARC_NAMESPACE}" \ + --create-namespace \ + --set image.repository="${IMAGE_NAME}" \ + --set image.tag="${IMAGE_TAG}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-controller" \ + --debug + + if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function install_squid() { + echo "Starting squid-proxy" + kubectl apply -f "${DIR}/anonymous-proxy-setup.squid.yaml" +} + +function install_scale_set() { + echo "Installing scale set ${SCALE_SET_NAMESPACE}/${SCALE_SET_NAME}" + + echo helm install "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ + --set proxy.https.url="http://squid.default.svc.cluster.local:3128" \ + --set "proxy.noProxy[0]=10.96.0.1:443" \ + "${ROOT_DIR}/charts/gha-runner-scale-set" \ + --debug + + if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function main() { + local failed=() + + build_image + create_cluster + + install_arc + install_squid + + install_scale_set || { + echo "Scale set installation failed" + NAMESPACE="${ARC_NAMESPACE}" log_arc + delete_cluster + exit 1 + } + + WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow") + + INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") + + NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") + + delete_cluster + + print_results "${failed[@]}" +} + +main diff --git a/test/actions.github.com/auth-proxy-setup.squid.yaml b/test/actions.github.com/auth-proxy-setup.squid.yaml new file mode 100644 index 00000000..7ba99495 --- /dev/null +++ b/test/actions.github.com/auth-proxy-setup.squid.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: squid +spec: + replicas: 1 + selector: + matchLabels: + app: squid + template: + metadata: + labels: + app: squid + spec: + containers: + - name: squid + image: huangtingluo/squid-proxy:latest + ports: + - containerPort: 3128 +--- +apiVersion: v1 +kind: Service +metadata: + name: squid +spec: + selector: + app: squid + ports: + - protocol: TCP + port: 3128 + targetPort: 3128 diff --git a/test/actions.github.com/auth-proxy-setup.test.sh b/test/actions.github.com/auth-proxy-setup.test.sh new file mode 100755 index 00000000..d13054fd --- /dev/null +++ b/test/actions.github.com/auth-proxy-setup.test.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +set -euo pipefail + +DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +source "${DIR}/helper.sh" + +SCALE_SET_NAME="default-$(date +'%M%S')$(((RANDOM + 100) % 100 + 1))" +SCALE_SET_NAMESPACE="arc-runners" +WORKFLOW_FILE="arc-test-workflow.yaml" +ARC_NAME="arc" +ARC_NAMESPACE="arc-systems" + +function install_arc() { + echo "Install openebs/dynamic-localpv-provisioner" + helm repo add openebs https://openebs.github.io/charts + helm repo update + helm install openebs openebs/openebs -n openebs --create-namespace + + echo "Creating namespace ${ARC_NAMESPACE}" + kubectl create namespace "${SCALE_SET_NAMESPACE}" + + echo "Installing ARC" + helm install "${ARC_NAME}" \ + --namespace "${ARC_NAMESPACE}" \ + --create-namespace \ + --set image.repository="${IMAGE_NAME}" \ + --set image.tag="${IMAGE_TAG}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-controller" \ + --debug + + if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function install_squid() { + echo "Starting squid-proxy" + kubectl apply -f "${DIR}/auth-proxy-setup.squid.yaml" + + echo "Creating scale set namespace" + kubectl create namespace "${SCALE_SET_NAMESPACE}" || true + + echo "Creating squid proxy secret" + kubectl create secret generic proxy-auth \ + --namespace=arc-runners \ + --from-literal=username=github \ + --from-literal=password='actions' +} + +function install_scale_set() { + echo "Installing scale set ${SCALE_SET_NAMESPACE}/${SCALE_SET_NAME}" + helm install "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ + --set proxy.https.url="http://squid.default.svc.cluster.local:3128" \ + --set proxy.https.credentialSecretRef="proxy-auth" \ + --set "proxy.noProxy[0]=10.96.0.1:443" \ + "${ROOT_DIR}/charts/gha-runner-scale-set" \ + --version="${VERSION}" \ + --debug + + if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function main() { + local failed=() + + build_image + create_cluster + + install_arc + install_squid + + install_scale_set || { + echo "Scale set installation failed" + NAMESPACE="${ARC_NAMESPACE}" log_arc + delete_cluster + exit 1 + } + + WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow") + + INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") + + NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") + + delete_cluster + + print_results "${failed[@]}" +} + +main diff --git a/test/actions.github.com/default-setup.test.sh b/test/actions.github.com/default-setup.test.sh new file mode 100755 index 00000000..2669320b --- /dev/null +++ b/test/actions.github.com/default-setup.test.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +set -euo pipefail + +DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +source "${DIR}/helper.sh" + +SCALE_SET_NAME="default-$(date +'%M%S')$(((RANDOM + 100) % 100 + 1))" +SCALE_SET_NAMESPACE="arc-runners" +WORKFLOW_FILE="arc-test-workflow.yaml" +ARC_NAME="arc" +ARC_NAMESPACE="arc-systems" + +function install_arc() { + echo "Creating namespace ${ARC_NAMESPACE}" + kubectl create namespace "${SCALE_SET_NAMESPACE}" + + echo "Installing ARC" + helm install "${ARC_NAME}" \ + --namespace "${ARC_NAMESPACE}" \ + --create-namespace \ + --set image.repository="${IMAGE_NAME}" \ + --set image.tag="${IMAGE_TAG}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-controller" \ + --debug + + if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function install_scale_set() { + echo "Installing scale set ${SCALE_SET_NAMESPACE}/${SCALE_SET_NAME}" + helm install "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set" \ + --version="${VERSION}" \ + --debug + + if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function main() { + local failed=() + + build_image + create_cluster + + install_arc + install_scale_set + + WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow") + + INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") + + NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") + + delete_cluster + + print_results "${failed[@]}" +} + +main diff --git a/test/actions.github.com/dind-mode-setup.test.sh b/test/actions.github.com/dind-mode-setup.test.sh new file mode 100755 index 00000000..2c70af76 --- /dev/null +++ b/test/actions.github.com/dind-mode-setup.test.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +set -euo pipefail + +DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +source "${DIR}/helper.sh" + +SCALE_SET_NAME="default-$(date +'%M%S')$(((RANDOM + 100) % 100 + 1))" +SCALE_SET_NAMESPACE="arc-runners" +WORKFLOW_FILE="arc-test-dind-workflow.yaml" +ARC_NAME="arc" +ARC_NAMESPACE="arc-systems" + +function install_arc() { + echo "Creating namespace ${ARC_NAMESPACE}" + kubectl create namespace "${SCALE_SET_NAMESPACE}" + + echo "Installing ARC" + helm install "${ARC_NAME}" \ + --namespace "${ARC_NAMESPACE}" \ + --create-namespace \ + --set image.repository="${IMAGE_NAME}" \ + --set image.tag="${IMAGE_TAG}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-controller" \ + --debug + + if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function install_scale_set() { + echo "Installing scale set ${SCALE_SET_NAMESPACE}/${SCALE_SET_NAME}" + helm install "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ + --set containerMode.type="dind" \ + "${ROOT_DIR}/charts/gha-runner-scale-set" \ + --version="${VERSION}" \ + --debug + + if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function main() { + local failed=() + + build_image + create_cluster + + install_arc + install_scale_set + + WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow") + + INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") + + NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") + + delete_cluster + + print_results "${failed[@]}" +} + +main diff --git a/test/actions.github.com/envrc.example b/test/actions.github.com/envrc.example new file mode 100644 index 00000000..5db64c18 --- /dev/null +++ b/test/actions.github.com/envrc.example @@ -0,0 +1,3 @@ +export TARGET_ORG="org" +export TARGET_REPO="repo" +export GITHUB_TOKEN="token" diff --git a/test/actions.github.com/helper.sh b/test/actions.github.com/helper.sh new file mode 100644 index 00000000..32718035 --- /dev/null +++ b/test/actions.github.com/helper.sh @@ -0,0 +1,193 @@ +#!/bin/bash + +DIR="$(dirname "${BASH_SOURCE[0]}")" + +DIR="$(realpath "${DIR}")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +export TARGET_ORG="${TARGET_ORG:-actions-runner-controller}" +export TARGET_REPO="${TARGET_REPO:-arc_e2e_test_dummy}" +export IMAGE_NAME="${IMAGE_NAME:-arc-test-image}" +export VERSION="${VERSION:-$(yq .version <"${ROOT_DIR}/charts/gha-runner-scale-set-controller/Chart.yaml")}" +export IMAGE_TAG="${VERSION}" +export IMAGE="${IMAGE_NAME}:${IMAGE_TAG}" + +export PLATFORMS="linux/amd64" +export COMMIT_SHA="$(git rev-parse HEAD)" + +function build_image() { + echo "Building ARC image ${IMAGE}" + + cd "${ROOT_DIR}" || exit 1 + + docker buildx build --platform "${PLATFORMS}" \ + --build-arg VERSION="${VERSION}" \ + --build-arg COMMIT_SHA="${COMMIT_SHA}" \ + -t "${IMAGE}" \ + -f Dockerfile \ + . --load + + echo "Created image ${IMAGE}" + cd - || exit 1 +} + +function create_cluster() { + echo "Deleting minikube cluster if exists" + minikube delete || true + + echo "Creating minikube cluster" + minikube start --driver=docker --container-runtime=docker --wait=all + + echo "Verifying ns works" + if ! minikube ssh "nslookup github.com >/dev/null 2>&1"; then + echo "Nameserver configuration failed" + exit 1 + fi + + echo "Loading image into minikube cluster" + minikube image load "${IMAGE}" + + echo "Loading runner image into minikube cluster" + minikube image load "ghcr.io/actions/actions-runner:latest" +} + +function delete_cluster() { + echo "Deleting minikube cluster" + minikube delete +} + +function log_arc() { + echo "ARC logs" + kubectl logs -n "${NAMESPACE}" -l app.kubernetes.io/name=gha-rs-controller +} + +function wait_for_arc() { + echo "Waiting for ARC to be ready" + local count=0 + while true; do + POD_NAME=$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=gha-rs-controller -o name) + if [ -n "$POD_NAME" ]; then + echo "Pod found: $POD_NAME" + break + fi + if [ "$count" -ge 60 ]; then + echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-rs-controller" + return 1 + fi + sleep 1 + count=$((count + 1)) + done + + kubectl wait --timeout=30s --for=condition=ready pod -n "${NAMESPACE}" -l app.kubernetes.io/name=gha-rs-controller + kubectl get pod -n "${NAMESPACE}" + kubectl describe deployment "${NAME}" -n "${NAMESPACE}" +} + +function wait_for_scale_set() { + local count=0 + while true; do + POD_NAME=$(kubectl get pods -n "${NAMESPACE}" -l "actions.github.com/scale-set-name=${NAME}" -o name) + if [ -n "$POD_NAME" ]; then + echo "Pod found: ${POD_NAME}" + break + fi + + if [ "$count" -ge 60 ]; then + echo "Timeout waiting for listener pod with label actions.github.com/scale-set-name=${NAME}" + return 1 + fi + + sleep 1 + count=$((count + 1)) + done + kubectl wait --timeout=30s --for=condition=ready pod -n "${NAMESPACE}" -l "actions.github.com/scale-set-name=${NAME}" + kubectl get pod -n "${NAMESPACE}" -l "actions.github.com/scale-set-name=${NAME}" +} + +function cleanup_scale_set() { + helm uninstall "${INSTALLATION_NAME}" --namespace "${NAMESPACE}" --debug + + kubectl wait --timeout=40s --for=delete autoscalingrunnersets -n "${NAMESPACE}" -l app.kubernetes.io/instance="${INSTALLATION_NAME}" +} + +function install_openebs() { + echo "Install openebs/dynamic-localpv-provisioner" + helm repo add openebs https://openebs.github.io/charts + helm repo update + helm install openebs openebs/openebs --namespace openebs --create-namespace +} + +function print_results() { + local failed=("$@") + + if [[ "${#failed[@]}" -ne 0 ]]; then + echo "----------------------------------" + echo "The following tests failed:" + for test in "${failed[@]}"; do + echo " - ${test}" + done + return 1 + else + echo "----------------------------------" + echo "All tests passed!" + fi +} + +function run_workflow() { + echo "Checking if the workflow file exists" + gh workflow view -R "${TARGET_ORG}/${TARGET_REPO}" "${WORKFLOW_FILE}" || return 1 + + local queue_time="$(date -u +%FT%TZ)" + + echo "Running workflow ${WORKFLOW_FILE}" + gh workflow run -R "${TARGET_ORG}/${TARGET_REPO}" "${WORKFLOW_FILE}" --ref main -f arc_name="${SCALE_SET_NAME}" || return 1 + + echo "Waiting for run to start" + local count=0 + local run_id= + while true; do + if [[ "${count}" -ge 12 ]]; then + echo "Timeout waiting for run to start" + return 1 + fi + run_id=$(gh run list -R "${TARGET_ORG}/${TARGET_REPO}" --workflow "${WORKFLOW_FILE}" --created ">${queue_time}" --json "name,databaseId" --jq ".[] | select(.name | contains(\"${SCALE_SET_NAME}\")) | .databaseId") + echo "Run ID: ${run_id}" + if [ -n "$run_id" ]; then + echo "Run found!" + break + fi + + echo "Run not found yet, waiting 5 seconds" + sleep 5 + count=$((count + 1)) + done + + echo "Waiting for run to complete" + local code=$(gh run watch "${run_id}" -R "${TARGET_ORG}/${TARGET_REPO}" --exit-status &>/dev/null) + if [[ "${code}" -ne 0 ]]; then + echo "Run failed with exit code ${code}" + return 1 + fi + + echo "Run completed successfully" +} + +function retry() { + local retries=$1 + shift + local delay=$1 + shift + local n=1 + + until "$@"; do + if [[ $n -ge $retries ]]; then + echo "Attempt $n failed! No more retries left." + return 1 + else + echo "Attempt $n failed! Retrying in $delay seconds..." + sleep "$delay" + n=$((n + 1)) + fi + done +} diff --git a/test/actions.github.com/init-with-min-runners.test.sh b/test/actions.github.com/init-with-min-runners.test.sh new file mode 100755 index 00000000..8ca08ab9 --- /dev/null +++ b/test/actions.github.com/init-with-min-runners.test.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +set -euo pipefail + +DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +source "${DIR}/helper.sh" || { + echo "Failed to source helper.sh" + exit 1 +} + +SCALE_SET_NAME="init-min-runners-$(date +'%M%S')$(((RANDOM + 100) % 100 + 1))" +SCALE_SET_NAMESPACE="arc-runners" +WORKFLOW_FILE="arc-test-workflow.yaml" +ARC_NAME="arc" +ARC_NAMESPACE="arc-systems" + +function install_arc() { + echo "Installing ARC" + helm install arc \ + --namespace "arc-systems" \ + --create-namespace \ + --set image.repository="${IMAGE_NAME}" \ + --set image.tag="${IMAGE_TAG}" \ + --set flags.updateStrategy="eventual" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-controller" \ + --debug + + if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function install_scale_set() { + echo "Installing scale set ${SCALE_SET_NAMESPACE}/${SCALE_SET_NAME}" + helm install "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ + --set minRunners=5 \ + "${ROOT_DIR}/charts/gha-runner-scale-set" \ + --debug + + if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function assert_5_runners() { + echo "[*] Asserting 5 runners are created" + local count=0 + while true; do + pod_count=$(kubectl get pods -n arc-runners --no-headers | wc -l) + + if [[ "${pod_count}" = 5 ]]; then + echo "[*] Found 5 runners as expected" + break + fi + + if [[ "$count" -ge 30 ]]; then + echo "Timeout waiting for 5 pods to be created" + exit 1 + fi + sleep 1 + count=$((count + 1)) + done +} + +function main() { + local failed=() + + build_image + create_cluster + + install_arc + install_scale_set + + assert_5_runners || failed+=("assert_5_runners") + + INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") + + NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") + + delete_cluster + + print_results "${failed[@]}" +} + +main diff --git a/test/actions.github.com/kubernetes-mode-setup.test.sh b/test/actions.github.com/kubernetes-mode-setup.test.sh new file mode 100755 index 00000000..8a2dd1fe --- /dev/null +++ b/test/actions.github.com/kubernetes-mode-setup.test.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -euo pipefail + +DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +source "${DIR}/helper.sh" + +SCALE_SET_NAME="kubernetes-mode-$(date +'%M%S')$(((RANDOM + 100) % 100 + 1))" +SCALE_SET_NAMESPACE="arc-runners" +WORKFLOW_FILE="arc-test-kubernetes-workflow.yaml" +ARC_NAME="arc" +ARC_NAMESPACE="arc-systems" + +function install_arc() { + echo "Install openebs/dynamic-localpv-provisioner" + helm repo add openebs https://openebs.github.io/charts + helm repo update + helm install openebs openebs/openebs -n openebs --create-namespace + + echo "Creating namespace ${ARC_NAMESPACE}" + kubectl create namespace "${SCALE_SET_NAMESPACE}" + + echo "Installing ARC" + helm install "${ARC_NAME}" \ + --namespace "${ARC_NAMESPACE}" \ + --create-namespace \ + --set image.repository="${IMAGE_NAME}" \ + --set image.tag="${IMAGE_TAG}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-controller" \ + --debug + + if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function install_scale_set() { + echo "Installing scale set ${SCALE_SET_NAMESPACE}/${SCALE_SET_NAME}" + helm install "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ + --set containerMode.type="kubernetes" \ + --set containerMode.kubernetesModeWorkVolumeClaim.accessModes={"ReadWriteOnce"} \ + --set containerMode.kubernetesModeWorkVolumeClaim.storageClassName="openebs-hostpath" \ + --set containerMode.kubernetesModeWorkVolumeClaim.resources.requests.storage="1Gi" \ + "${ROOT_DIR}/charts/gha-runner-scale-set" \ + --debug + + if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function main() { + local failed=() + + build_image + create_cluster + + install_arc + install_scale_set + + WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow") + + INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") + + NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") + + delete_cluster + + print_results "${failed[@]}" +} + +main diff --git a/test/actions.github.com/self-signed-ca-setup.mitm.yaml b/test/actions.github.com/self-signed-ca-setup.mitm.yaml new file mode 100644 index 00000000..00382d6d --- /dev/null +++ b/test/actions.github.com/self-signed-ca-setup.mitm.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +metadata: + name: mitmproxy + namespace: mitmproxy + labels: + app: mitmproxy +spec: + containers: + - name: mitmproxy + image: mitmproxy/mitmproxy:latest + command: ["mitmdump"] + ports: + - containerPort: 8080 + name: proxy +--- +apiVersion: v1 +kind: Service +metadata: + name: mitmproxy + namespace: mitmproxy +spec: + selector: + app: mitmproxy + ports: + - port: 8080 + targetPort: 8080 + name: proxy + diff --git a/test/actions.github.com/self-signed-ca-setup.test.sh b/test/actions.github.com/self-signed-ca-setup.test.sh new file mode 100755 index 00000000..1264433d --- /dev/null +++ b/test/actions.github.com/self-signed-ca-setup.test.sh @@ -0,0 +1,148 @@ +#!/bin/bash + +set -euo pipefail + +DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +source "${DIR}/helper.sh" || { + echo "Failed to source helper.sh" + exit 1 +} + +TEMP_DIR=$(mktemp -d) +LOCAL_CERT_PATH="${TEMP_DIR}/mitmproxy-ca-cert.crt" +MITM_CERT_PATH="/root/.mitmproxy/mitmproxy-ca-cert.pem" + +trap 'rm -rf "$TEMP_DIR"' EXIT + +SCALE_SET_NAME="self-signed-crt-$(date '+%M%S')$(((RANDOM + 100) % 100 + 1))" +SCALE_SET_NAMESPACE="arc-runners" +WORKFLOW_FILE="arc-test-workflow.yaml" +ARC_NAME="arc" +ARC_NAMESPACE="arc-systems" + +MITMPROXY_NAMESPACE="mitmproxy" +MITMPROXY_POD_NAME="mitmproxy" + +function install_arc() { + echo "Installing ARC" + helm install "${ARC_NAME}" \ + --namespace "${ARC_NAMESPACE}" \ + --create-namespace \ + --set image.repository="${IMAGE_NAME}" \ + --set image.tag="${IMAGE_TAG}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-controller" \ + --debug + + if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function install_scale_set() { + echo "Creating namespace ${SCALE_SET_NAMESPACE}" + kubectl create namespace "${SCALE_SET_NAMESPACE}" + + echo "Installing ca-cert config map" + kubectl -n "${SCALE_SET_NAMESPACE}" create configmap ca-cert \ + --from-file=mitmproxy-ca-cert.crt="${LOCAL_CERT_PATH}" + + echo "Installing scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}" + helm install "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ + --set proxy.https.url="http://mitmproxy.mitmproxy.svc.cluster.local:8080" \ + --set "proxy.noProxy[0]=10.96.0.1:443" \ + --set "githubServerTLS.certificateFrom.configMapKeyRef.name=ca-cert" \ + --set "githubServerTLS.certificateFrom.configMapKeyRef.key=mitmproxy-ca-cert.crt" \ + --set "githubServerTLS.runnerMountPath=/usr/local/share/ca-certificates/" \ + "${ROOT_DIR}/charts/gha-runner-scale-set" \ + --debug + + if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function wait_for_mitmproxy_ready() { + echo "Waiting for mitmproxy pod to be ready" + + # Wait for pod to be running + if ! kubectl wait --for=condition=ready pod -n "${MITMPROXY_NAMESPACE}" "${MITMPROXY_POD_NAME}" --timeout=60s; then + echo "Timeout waiting for mitmproxy pod" + kubectl get pods -n "${MITMPROXY_NAMESPACE}" || true + kubectl describe pod -n "${MITMPROXY_NAMESPACE}" "${MITMPROXY_POD_NAME}" || true + kubectl logs -n "${MITMPROXY_NAMESPACE}" "${MITMPROXY_POD_NAME}" || true + return 1 + fi + + echo "Mitmproxy pod is ready, trying to copy the certitficate..." + + # Verify certificate exists + retry 15 1 kubectl exec -n "${MITMPROXY_NAMESPACE}" "${MITMPROXY_POD_NAME}" -- test -f "${MITM_CERT_PATH}" + + echo "Getting mitmproxy CA certificate from pod" + if ! kubectl exec -n "${MITMPROXY_NAMESPACE}" "${MITMPROXY_POD_NAME}" -- cat "${MITM_CERT_PATH}" >"${LOCAL_CERT_PATH}"; then + echo "Failed to get mitmproxy CA certificate from pod" + return 1 + fi + echo "Mitmproxy certificate generated successfully and stored to ${LOCAL_CERT_PATH}" + return 0 +} + +function run_mitmproxy() { + echo "Deploying mitmproxy to Kubernetes" + + # Create namespace + kubectl create namespace "${MITMPROXY_NAMESPACE}" || true + + # Create mitmproxy pod and service + kubectl apply -f "${DIR}/self-signed-ca-setup.mitm.yaml" + + if ! wait_for_mitmproxy_ready; then + return 1 + fi + + echo "Mitmproxy is ready" +} + +function main() { + local failed=() + + build_image + create_cluster + install_arc + run_mitmproxy || { + echo "Failed to run mitmproxy" + echo "ARC logs:" + NAMESPACE="${ARC_NAMESPACE}" log_arc + echo "Deleting cluster..." + delete_cluster + exit 1 + } + install_scale_set || { + echo "Failed to run mitmproxy" + echo "ARC logs:" + NAMESPACE="${ARC_NAMESPACE}" log_arc + echo "Deleting cluster..." + delete_cluster + exit 1 + } + + WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow") + INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") + + NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") + + delete_cluster + + print_results "${failed[@]}" +} + +main diff --git a/test/actions.github.com/single-namespace-setup.test.sh b/test/actions.github.com/single-namespace-setup.test.sh new file mode 100755 index 00000000..f6462944 --- /dev/null +++ b/test/actions.github.com/single-namespace-setup.test.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +set -euo pipefail + +DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +source "${DIR}/helper.sh" + +SCALE_SET_NAME="default-$(date +'%M%S')$(((RANDOM + 100) % 100 + 1))" +SCALE_SET_NAMESPACE="arc-runners" +WORKFLOW_FILE="arc-test-workflow.yaml" +ARC_NAME="arc" +ARC_NAMESPACE="${SCALE_SET_NAMESPACE}" + +function install_arc() { + echo "Creating namespace ${ARC_NAMESPACE}" + kubectl create namespace "${SCALE_SET_NAMESPACE}" + + echo "Installing ARC" + helm install "${ARC_NAME}" \ + --namespace "${ARC_NAMESPACE}" \ + --create-namespace \ + --set image.repository="${IMAGE_NAME}" \ + --set image.tag="${IMAGE_TAG}" \ + --set flags.watchSingleNamespace="${ARC_NAMESPACE}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-controller" \ + --debug + + if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function install_scale_set() { + echo "Installing scale set ${SCALE_SET_NAMESPACE}/${SCALE_SET_NAME}" + helm install "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set" \ + --version="${VERSION}" \ + --debug + + if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function main() { + local failed=() + + build_image + create_cluster + + install_arc + install_scale_set + + WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow") + + INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") + + NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") + + delete_cluster + + print_results "${failed[@]}" +} + +main diff --git a/test/actions.github.com/update-strategy.test.sh b/test/actions.github.com/update-strategy.test.sh new file mode 100755 index 00000000..f7fce53b --- /dev/null +++ b/test/actions.github.com/update-strategy.test.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +set -euo pipefail + +DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +source "${DIR}/helper.sh" || { + echo "Failed to source helper.sh" + exit 1 +} + +SCALE_SET_NAME="update-strategy-$(date '+%M%S')$(((RANDOM + 100) % 100 + 1))" +SCALE_SET_NAMESPACE="arc-runners" +WORKFLOW_FILE="arc-test-sleepy-matrix.yaml" +ARC_NAME="arc" +ARC_NAMESPACE="arc-systems" + +function install_arc() { + echo "Installing ARC" + + helm install "${ARC_NAME}" \ + --namespace "${ARC_NAMESPACE}" \ + --create-namespace \ + --set image.repository="${IMAGE_NAME}" \ + --set image.tag="${IMAGE_TAG}" \ + --set flags.updateStrategy="eventual" \ + "${ROOT_DIR}/charts/gha-runner-scale-set-controller" \ + --debug + + if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function install_scale_set() { + echo "Installing scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}" + helm install "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ + "${ROOT_DIR}/charts/gha-runner-scale-set" \ + --debug + + if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then + NAMESPACE="${ARC_NAMESPACE}" log_arc + return 1 + fi +} + +function upgrade_scale_set() { + echo "Upgrading scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}" + helm upgrade "${SCALE_SET_NAME}" \ + --namespace "${SCALE_SET_NAMESPACE}" \ + --set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \ + --set githubConfigSecret.github_token="${GITHUB_TOKEN}" \ + --set template.spec.containers[0].name="runner" \ + --set template.spec.containers[0].image="ghcr.io/actions/actions-runner:latest" \ + --set template.spec.containers[0].command={"/home/runner/run.sh"} \ + --set template.spec.containers[0].env[0].name="TEST" \ + --set template.spec.containers[0].env[0].value="E2E TESTS" \ + "${ROOT_DIR}/charts/gha-runner-scale-set" \ + --version="${VERSION}" \ + --debug + +} + +function assert_listener_deleted() { + local count=0 + while true; do + LISTENER_COUNT="$(kubectl get pods -l actions.github.com/scale-set-name="${SCALE_SET_NAME}" -n "${ARC_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')" + RUNNERS_COUNT="$(kubectl get pods -l app.kubernetes.io/component=runner -n "${SCALE_SET_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')" + RESOURCES="$(kubectl get pods -A)" + + if [ "${LISTENER_COUNT}" -eq 0 ]; then + echo "Listener has been deleted" + echo "${RESOURCES}" + return 0 + fi + if [ "${count}" -ge 60 ]; then + echo "Timeout waiting for listener to be deleted" + echo "${RESOURCES}" + return 1 + fi + + echo "Waiting for listener to be deleted" + echo "Listener count: ${LISTENER_COUNT} target: 0 | Runners count: ${RUNNERS_COUNT} target: 3" + + sleep 1 + count=$((count + 1)) + done +} + +function assert_listener_recreated() { + count=0 + while true; do + LISTENER_COUNT="$(kubectl get pods -l actions.github.com/scale-set-name="${SCALE_SET_NAME}" -n "${ARC_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')" + RUNNERS_COUNT="$(kubectl get pods -l app.kubernetes.io/component=runner -n "${SCALE_SET_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')" + RESOURCES="$(kubectl get pods -A)" + + if [ "${LISTENER_COUNT}" -eq 1 ]; then + echo "Listener is up!" + echo "${RESOURCES}" + return 0 + fi + if [ "${count}" -ge 120 ]; then + echo "Timeout waiting for listener to be recreated" + echo "${RESOURCES}" + return 1 + fi + + echo "Waiting for listener to be recreated" + echo "Listener count: ${LISTENER_COUNT} target: 1 | Runners count: ${RUNNERS_COUNT} target: 0" + + sleep 1 + count=$((count + 1)) + done +} + +function main() { + local failed=() + + build_image + create_cluster + install_arc + install_scale_set + + WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow") + + upgrade_scale_set || failed+=("upgrade_scale_set") + assert_listener_deleted || failed+=("assert_listener_deleted") + assert_listener_recreated || failed+=("assert_listener_recreated") + + INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set") + + NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc") + + delete_cluster + + print_results "${failed[@]}" +} + +main