mirror of
https://github.com/actions-runner-controller/actions-runner-controller.git
synced 2026-09-30 16:46:44 +02:00
Upgrade resources in-place, causing 1-1 mapping between autoscaling runner set and ephemeral runner set (#4516)
This commit is contained in:
Nikola Jokic
@@ -10,6 +10,10 @@ export TARGET_ORG="${TARGET_ORG:-actions-runner-controller}"
|
||||
export TARGET_REPO="${TARGET_REPO:-arc_e2e_test_dummy}"
|
||||
export IMAGE_NAME="${IMAGE_NAME:-arc-test-image}"
|
||||
|
||||
function log() {
|
||||
echo "[$(date -u +%FT%T%Z)] $*" >&2
|
||||
}
|
||||
|
||||
# Trims a single pair of matching surrounding quotes from the provided string.
|
||||
# Examples:
|
||||
# trim_quotes '"1.2.3"' -> 1.2.3
|
||||
@@ -56,7 +60,7 @@ function chart_version() {
|
||||
|
||||
version="$(trim_quotes "${version}" | tr -d "[:space:]")"
|
||||
if [[ -z "${version}" ]]; then
|
||||
echo "Failed to extract version from ${chart_yaml} via yq" >&2
|
||||
log "Failed to extract version from ${chart_yaml} via yq"
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "${version}"
|
||||
@@ -65,14 +69,14 @@ function chart_version() {
|
||||
|
||||
function ensure_version_set() {
|
||||
if [[ -z "${VERSION:-}" ]]; then
|
||||
echo 'VERSION is not set. Set it in the test, e.g. export VERSION="$(chart_version path/to/Chart.yaml)".' >&2
|
||||
log 'VERSION is not set. Set it in the test, e.g. export VERSION="$(chart_version path/to/Chart.yaml)".'
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Defensive: if a tool produced quoted output, normalize it before using in tags/args.
|
||||
export VERSION="$(printf '%s' "${VERSION}" | tr -d "\"'[:space:]")"
|
||||
if [[ -z "${VERSION}" ]]; then
|
||||
echo "VERSION resolved to an empty value" >&2
|
||||
log "VERSION resolved to an empty value"
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -86,7 +90,7 @@ export COMMIT_SHA
|
||||
|
||||
function build_image() {
|
||||
ensure_version_set || return 1
|
||||
echo "Building ARC image ${IMAGE}"
|
||||
log "Building ARC image ${IMAGE}"
|
||||
|
||||
cd "${ROOT_DIR}" || exit 1
|
||||
|
||||
@@ -103,46 +107,46 @@ function build_image() {
|
||||
|
||||
function create_cluster() {
|
||||
ensure_version_set || return 1
|
||||
echo "Deleting minikube cluster if exists"
|
||||
log "Deleting minikube cluster if exists"
|
||||
minikube delete || true
|
||||
|
||||
echo "Creating minikube cluster"
|
||||
log "Creating minikube cluster"
|
||||
minikube start --driver=docker --container-runtime=docker --wait=all
|
||||
|
||||
echo "Verifying ns works"
|
||||
log "Verifying ns works"
|
||||
if ! minikube ssh "nslookup github.com >/dev/null 2>&1"; then
|
||||
echo "Nameserver configuration failed"
|
||||
log "Nameserver configuration failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Loading image into minikube cluster"
|
||||
log "Loading image into minikube cluster"
|
||||
minikube image load "${IMAGE}"
|
||||
|
||||
echo "Loading runner image into minikube cluster"
|
||||
log "Loading runner image into minikube cluster"
|
||||
minikube image load "ghcr.io/actions/actions-runner:latest"
|
||||
}
|
||||
|
||||
function delete_cluster() {
|
||||
echo "Deleting minikube cluster"
|
||||
log "Deleting minikube cluster"
|
||||
minikube delete
|
||||
}
|
||||
|
||||
function log_arc() {
|
||||
echo "ARC logs"
|
||||
log "ARC logs"
|
||||
kubectl logs -n "${NAMESPACE}" -l "app.kubernetes.io/part-of=gha-rs-controller,app.kubernetes.io/component=controller-manager"
|
||||
}
|
||||
|
||||
function wait_for_arc() {
|
||||
echo "Waiting for ARC to be ready"
|
||||
log "Waiting for ARC to be ready"
|
||||
local count=0
|
||||
while true; do
|
||||
POD_NAME=$(kubectl get pods -n "${NAMESPACE}" -l "app.kubernetes.io/part-of=gha-rs-controller,app.kubernetes.io/component=controller-manager" -o name 2>/dev/null || true)
|
||||
if [ -n "$POD_NAME" ]; then
|
||||
echo "Pod found: $POD_NAME"
|
||||
log "Pod found: $POD_NAME"
|
||||
break
|
||||
fi
|
||||
if [ "$count" -ge 60 ]; then
|
||||
echo "Timeout waiting for controller pod with labels app.kubernetes.io/part-of=gha-rs-controller,app.kubernetes.io/component=controller-manager"
|
||||
log "Timeout waiting for controller pod with labels app.kubernetes.io/part-of=gha-rs-controller,app.kubernetes.io/component=controller-manager"
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
@@ -159,12 +163,12 @@ function wait_for_scale_set() {
|
||||
while true; do
|
||||
POD_NAME=$(kubectl get pods -n "${NAMESPACE}" -l "actions.github.com/scale-set-name=${NAME}" -o name)
|
||||
if [ -n "$POD_NAME" ]; then
|
||||
echo "Pod found: ${POD_NAME}"
|
||||
log "Pod found: ${POD_NAME}"
|
||||
break
|
||||
fi
|
||||
|
||||
if [ "$count" -ge 60 ]; then
|
||||
echo "Timeout waiting for listener pod with label actions.github.com/scale-set-name=${NAME}"
|
||||
log "Timeout waiting for listener pod with label actions.github.com/scale-set-name=${NAME}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -176,8 +180,10 @@ function wait_for_scale_set() {
|
||||
}
|
||||
|
||||
function cleanup_scale_set() {
|
||||
log "Uninstalling Helm release ${INSTALLATION_NAME}"
|
||||
helm uninstall "${INSTALLATION_NAME}" --namespace "${NAMESPACE}" --debug
|
||||
|
||||
log "Waiting for autoscaling runner sets to be deleted"
|
||||
kubectl wait --timeout=40s --for=delete autoscalingrunnersets -n "${NAMESPACE}" -l app.kubernetes.io/instance="${INSTALLATION_NAME}"
|
||||
}
|
||||
|
||||
@@ -197,7 +203,24 @@ function print_results() {
|
||||
fi
|
||||
}
|
||||
|
||||
function run_workflow() {
|
||||
function extract_run_id() {
|
||||
local workflow_output="$1"
|
||||
local run_id
|
||||
|
||||
run_id="$(printf '%s\n' "${workflow_output}" | awk '/^[[:space:]]*[0-9]+[[:space:]]*$/ { gsub(/[[:space:]]/, ""); print; exit }')"
|
||||
if [[ -z "${run_id}" ]]; then
|
||||
run_id="$(printf '%s\n' "${workflow_output}" | awk 'match($0, /actions\/runs\/[0-9]+/) { run=substr($0, RSTART, RLENGTH); sub(/^actions\/runs\//, "", run); print run; exit }')"
|
||||
fi
|
||||
|
||||
if [[ -z "${run_id}" ]]; then
|
||||
log "Failed to extract run id from output: ${workflow_output}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf '%s\n' "${run_id}"
|
||||
}
|
||||
|
||||
function start_workflow() {
|
||||
local repo
|
||||
repo="${TARGET_ORG}/${TARGET_REPO}"
|
||||
|
||||
@@ -240,50 +263,74 @@ function run_workflow() {
|
||||
fi
|
||||
|
||||
if [[ -z "${workflow_id}" ]]; then
|
||||
echo "Workflow not found in ${repo}: ${WORKFLOW_FILE}" >&2
|
||||
echo "Available workflows in ${repo}:" >&2
|
||||
log "Workflow not found in ${repo}: ${WORKFLOW_FILE}" >&2
|
||||
log "Available workflows in ${repo}:" >&2
|
||||
gh workflow list -R "${repo}" --limit 50 || true
|
||||
echo "Hint: set TARGET_ORG/TARGET_REPO to a repo that contains the workflow on its default branch, or set WORKFLOW_FILE to a valid workflow name/id/filename." >&2
|
||||
log "Hint: set TARGET_ORG/TARGET_REPO to a repo that contains the workflow on its default branch, or set WORKFLOW_FILE to a valid workflow name/id/filename." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Resolved workflow id: ${workflow_id} (ref: ${WORKFLOW_REF})"
|
||||
log "Resolved workflow id: ${workflow_id} (ref: ${WORKFLOW_REF})"
|
||||
|
||||
local queue_time
|
||||
queue_time="$(date -u +%FT%TZ)"
|
||||
|
||||
echo "Running workflow ${WORKFLOW_FILE}"
|
||||
gh workflow run -R "${repo}" "${workflow_id}" --ref "${WORKFLOW_REF}" -f arc_name="${SCALE_SET_NAME}" || return 1
|
||||
log "Running workflow ${WORKFLOW_FILE}"
|
||||
local workflow_output
|
||||
workflow_output="$(gh workflow run -R "${repo}" "${workflow_id}" --ref "${WORKFLOW_REF}" -f arc_name="${SCALE_SET_NAME}")" || return 1
|
||||
if [[ -n "${workflow_output}" ]]; then
|
||||
log "${workflow_output}"
|
||||
fi
|
||||
|
||||
echo "Waiting for run to start"
|
||||
log "Waiting for run to start"
|
||||
local count=0
|
||||
local run_id=
|
||||
local run_id_output=
|
||||
while true; do
|
||||
if [[ "${count}" -ge 12 ]]; then
|
||||
echo "Timeout waiting for run to start"
|
||||
log "Timeout waiting for run to start"
|
||||
return 1
|
||||
fi
|
||||
run_id=$(gh run list -R "${repo}" --workflow "${workflow_id}" --created ">${queue_time}" --json "name,databaseId" --jq ".[] | select(.name | contains(\"${SCALE_SET_NAME}\")) | .databaseId")
|
||||
echo "Run ID: ${run_id}"
|
||||
run_id_output=$(gh run list -R "${repo}" --workflow "${workflow_id}" --created ">${queue_time}" --json "name,databaseId" --jq ".[] | select(.name | contains(\"${SCALE_SET_NAME}\")) | .databaseId")
|
||||
if [[ -n "${run_id_output}" ]]; then
|
||||
run_id=$(extract_run_id "${run_id_output}" || true)
|
||||
fi
|
||||
log "Run ID: ${run_id}"
|
||||
if [ -n "$run_id" ]; then
|
||||
echo "Run found!"
|
||||
log "Run found!"
|
||||
break
|
||||
fi
|
||||
|
||||
echo "Run not found yet, waiting 5 seconds"
|
||||
log "Run not found yet, waiting 5 seconds"
|
||||
sleep 5
|
||||
count=$((count + 1))
|
||||
done
|
||||
|
||||
echo "Waiting for run to complete"
|
||||
echo "${run_id}"
|
||||
}
|
||||
|
||||
function wait_for_run_completion() {
|
||||
local run_id="$1"
|
||||
local repo="${TARGET_ORG}/${TARGET_REPO}"
|
||||
|
||||
log "Waiting for run ${run_id} to complete"
|
||||
gh run watch "${run_id}" -R "${repo}" --exit-status &>/dev/null
|
||||
local status=$?
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
echo "Run failed with exit code ${status}"
|
||||
log "Run failed with exit code ${status}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Run completed successfully"
|
||||
log "Run completed successfully"
|
||||
}
|
||||
|
||||
function run_workflow() {
|
||||
local run_id
|
||||
if ! run_id=$(start_workflow); then
|
||||
log "Failed to start workflow"
|
||||
return 1
|
||||
fi
|
||||
wait_for_run_completion "${run_id}"
|
||||
}
|
||||
|
||||
function retry() {
|
||||
@@ -295,10 +342,10 @@ function retry() {
|
||||
|
||||
until "$@"; do
|
||||
if [[ $n -ge $retries ]]; then
|
||||
echo "Attempt $n failed! No more retries left."
|
||||
log "Attempt $n failed! No more retries left."
|
||||
return 1
|
||||
else
|
||||
echo "Attempt $n failed! Retrying in $delay seconds..."
|
||||
log "Attempt $n failed! Retrying in $delay seconds..."
|
||||
sleep "$delay"
|
||||
n=$((n + 1))
|
||||
fi
|
||||
@@ -306,7 +353,7 @@ function retry() {
|
||||
}
|
||||
|
||||
function install_openebs() {
|
||||
echo "Install openebs/dynamic-localpv-provisioner"
|
||||
log "Install openebs/dynamic-localpv-provisioner"
|
||||
helm repo add openebs https://openebs.github.io/openebs
|
||||
helm repo update
|
||||
helm install openebs openebs/openebs -n openebs --create-namespace
|
||||
|
||||
+37
-46
@@ -19,13 +19,14 @@ WORKFLOW_FILE="arc-test-sleepy-matrix.yaml"
|
||||
ARC_NAME="arc"
|
||||
ARC_NAMESPACE="arc-systems"
|
||||
|
||||
UPGRADE_MARKER="e2e-upgrade-${SCALE_SET_NAME}-$(date +%s)"
|
||||
|
||||
function install_arc() {
|
||||
echo "Installing ARC"
|
||||
helm install "${ARC_NAME}" \
|
||||
--namespace "${ARC_NAMESPACE}" \
|
||||
--create-namespace \
|
||||
--set controller.manager.container.image="${IMAGE_NAME}:${IMAGE_TAG}" \
|
||||
--set controller.manager.config.updateStrategy="eventual" \
|
||||
"${ROOT_DIR}/charts/gha-runner-scale-set-controller-experimental" \
|
||||
--debug
|
||||
|
||||
@@ -44,6 +45,8 @@ function install_scale_set() {
|
||||
--set controllerServiceAccount.namespace="${ARC_NAMESPACE}" \
|
||||
--set auth.url="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \
|
||||
--set auth.githubToken="${GITHUB_TOKEN}" \
|
||||
--set scaleset.name="${SCALE_SET_NAME}" \
|
||||
--set scaleset.minRunners=5 \
|
||||
"${ROOT_DIR}/charts/gha-runner-scale-set-experimental" \
|
||||
--version="${VERSION}" \
|
||||
--debug
|
||||
@@ -56,87 +59,75 @@ function install_scale_set() {
|
||||
|
||||
function upgrade_scale_set() {
|
||||
echo "Upgrading scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}"
|
||||
|
||||
echo "Generated upgrade marker: ${UPGRADE_MARKER}"
|
||||
|
||||
helm upgrade "${SCALE_SET_NAME}" \
|
||||
--namespace "${SCALE_SET_NAMESPACE}" \
|
||||
--set controllerServiceAccount.name="${ARC_NAME}-gha-rs-controller" \
|
||||
--set controllerServiceAccount.namespace="${ARC_NAMESPACE}" \
|
||||
--set auth.url="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \
|
||||
--set auth.githubToken="${GITHUB_TOKEN}" \
|
||||
--set scaleset.name="${SCALE_SET_NAME}" \
|
||||
--set runner.container.image="ghcr.io/actions/actions-runner:latest" \
|
||||
--set runner.container.command={"/home/runner/run.sh"} \
|
||||
--set runner.env[0].name="TEST" \
|
||||
--set runner.env[0].value="E2E TESTS" \
|
||||
--set runner.container.env[0].name="TEST" \
|
||||
--set runner.container.env[0].value="E2E TESTS" \
|
||||
--set "runner.pod.metadata.labels.e2e\.arc/upgrade-marker=${UPGRADE_MARKER}" \
|
||||
"${ROOT_DIR}/charts/gha-runner-scale-set-experimental" \
|
||||
--version="${VERSION}" \
|
||||
--debug
|
||||
|
||||
}
|
||||
|
||||
function assert_listener_deleted() {
|
||||
function assert_idle_pod_recreated() {
|
||||
echo "Waiting for idle pod recreation"
|
||||
local count=0
|
||||
while true; do
|
||||
LISTENER_COUNT="$(kubectl get pods -l actions.github.com/scale-set-name="${SCALE_SET_NAME}" -n "${ARC_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')"
|
||||
RUNNERS_COUNT="$(kubectl get pods -l app.kubernetes.io/component=runner -n "${SCALE_SET_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')"
|
||||
RESOURCES="$(kubectl get pods -A)"
|
||||
|
||||
if [ "${LISTENER_COUNT}" -eq 0 ]; then
|
||||
echo "Listener has been deleted"
|
||||
echo "${RESOURCES}"
|
||||
return 0
|
||||
fi
|
||||
if [ "${count}" -ge 60 ]; then
|
||||
echo "Timeout waiting for listener to be deleted"
|
||||
echo "${RESOURCES}"
|
||||
while true; do
|
||||
local pods
|
||||
if ! pods=$(kubectl get pods -n "${SCALE_SET_NAMESPACE}" -l "actions.github.com/scale-set-name=${SCALE_SET_NAME},e2e.arc/upgrade-marker=${UPGRADE_MARKER}" -o jsonpath='{.items[*].metadata.name}'); then
|
||||
echo "Failed to get pods: $pods"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Waiting for listener to be deleted"
|
||||
echo "Listener count: ${LISTENER_COUNT} target: 0 | Runners count: ${RUNNERS_COUNT} target: 3"
|
||||
|
||||
sleep 1
|
||||
count=$((count + 1))
|
||||
done
|
||||
}
|
||||
|
||||
function assert_listener_recreated() {
|
||||
count=0
|
||||
while true; do
|
||||
LISTENER_COUNT="$(kubectl get pods -l actions.github.com/scale-set-name="${SCALE_SET_NAME}" -n "${ARC_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')"
|
||||
RUNNERS_COUNT="$(kubectl get pods -l app.kubernetes.io/component=runner -n "${SCALE_SET_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')"
|
||||
RESOURCES="$(kubectl get pods -A)"
|
||||
|
||||
if [ "${LISTENER_COUNT}" -eq 1 ]; then
|
||||
echo "Listener is up!"
|
||||
echo "${RESOURCES}"
|
||||
if [[ -n "$pods" ]]; then
|
||||
echo "Found idle pod with upgrade marker: $pods"
|
||||
return 0
|
||||
fi
|
||||
if [ "${count}" -ge 120 ]; then
|
||||
echo "Timeout waiting for listener to be recreated"
|
||||
echo "${RESOURCES}"
|
||||
|
||||
if ((count >= 30)); then
|
||||
echo "Timeout waiting for idle pod recreation after upgrade"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Waiting for listener to be recreated"
|
||||
echo "Listener count: ${LISTENER_COUNT} target: 1 | Runners count: ${RUNNERS_COUNT} target: 0"
|
||||
|
||||
sleep 1
|
||||
count=$((count + 1))
|
||||
echo "No idle pod with upgrade marker found yet, retrying... ($((count + 1))/30)"
|
||||
sleep 10
|
||||
((count++))
|
||||
done
|
||||
|
||||
}
|
||||
|
||||
function main() {
|
||||
local failed=()
|
||||
local run_id=""
|
||||
|
||||
build_image
|
||||
create_cluster
|
||||
install_arc
|
||||
install_scale_set
|
||||
|
||||
WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow")
|
||||
|
||||
upgrade_scale_set || failed+=("upgrade_scale_set")
|
||||
assert_listener_deleted || failed+=("assert_listener_deleted")
|
||||
assert_listener_recreated || failed+=("assert_listener_recreated")
|
||||
|
||||
if ! run_id=$(WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" start_workflow); then
|
||||
failed+=("run_workflow")
|
||||
fi
|
||||
|
||||
assert_idle_pod_recreated || failed+=("assert_idle_pod_recreated")
|
||||
|
||||
if [[ -n "${run_id}" ]]; then
|
||||
wait_for_run_completion "${run_id}" || failed+=("wait_for_run_completion")
|
||||
fi
|
||||
|
||||
INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set")
|
||||
|
||||
+37
-44
@@ -19,6 +19,8 @@ WORKFLOW_FILE="arc-test-sleepy-matrix.yaml"
|
||||
ARC_NAME="arc"
|
||||
ARC_NAMESPACE="arc-systems"
|
||||
|
||||
UPGRADE_MARKER="e2e-upgrade-${SCALE_SET_NAME}-$(date +%s)"
|
||||
|
||||
function install_arc() {
|
||||
echo "Installing ARC"
|
||||
helm install "${ARC_NAME}" \
|
||||
@@ -26,7 +28,6 @@ function install_arc() {
|
||||
--create-namespace \
|
||||
--set image.repository="${IMAGE_NAME}" \
|
||||
--set image.tag="${IMAGE_TAG}" \
|
||||
--set flags.updateStrategy="eventual" \
|
||||
"${ROOT_DIR}/charts/gha-runner-scale-set-controller" \
|
||||
--debug
|
||||
|
||||
@@ -43,6 +44,7 @@ function install_scale_set() {
|
||||
--create-namespace \
|
||||
--set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \
|
||||
--set githubConfigSecret.github_token="${GITHUB_TOKEN}" \
|
||||
--set minRunners=1 \
|
||||
"${ROOT_DIR}/charts/gha-runner-scale-set" \
|
||||
--debug
|
||||
|
||||
@@ -54,86 +56,77 @@ function install_scale_set() {
|
||||
|
||||
function upgrade_scale_set() {
|
||||
echo "Upgrading scale set ${SCALE_SET_NAME}/${SCALE_SET_NAMESPACE}"
|
||||
|
||||
echo "Generated upgrade marker: ${UPGRADE_MARKER}"
|
||||
|
||||
PATCH_APPLIED_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
echo "Captured PATCH_APPLIED_TIME: ${PATCH_APPLIED_TIME}"
|
||||
|
||||
helm upgrade "${SCALE_SET_NAME}" \
|
||||
--namespace "${SCALE_SET_NAMESPACE}" \
|
||||
--set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \
|
||||
--set githubConfigSecret.github_token="${GITHUB_TOKEN}" \
|
||||
--set minRunners=1 \
|
||||
--set template.spec.containers[0].name="runner" \
|
||||
--set template.spec.containers[0].image="ghcr.io/actions/actions-runner:latest" \
|
||||
--set template.spec.containers[0].command={"/home/runner/run.sh"} \
|
||||
--set template.spec.containers[0].env[0].name="TEST" \
|
||||
--set template.spec.containers[0].env[0].value="E2E TESTS" \
|
||||
--set "template.metadata.labels.e2e\.arc/upgrade-marker=${UPGRADE_MARKER}" \
|
||||
"${ROOT_DIR}/charts/gha-runner-scale-set" \
|
||||
--version="${VERSION}" \
|
||||
--debug
|
||||
|
||||
}
|
||||
|
||||
function assert_listener_deleted() {
|
||||
function assert_idle_pod_recreated() {
|
||||
echo "Waiting for idle pod recreation"
|
||||
local count=0
|
||||
while true; do
|
||||
LISTENER_COUNT="$(kubectl get pods -l actions.github.com/scale-set-name="${SCALE_SET_NAME}" -n "${ARC_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')"
|
||||
RUNNERS_COUNT="$(kubectl get pods -l app.kubernetes.io/component=runner -n "${SCALE_SET_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')"
|
||||
RESOURCES="$(kubectl get pods -A)"
|
||||
|
||||
if [ "${LISTENER_COUNT}" -eq 0 ]; then
|
||||
echo "Listener has been deleted"
|
||||
echo "${RESOURCES}"
|
||||
return 0
|
||||
fi
|
||||
if [ "${count}" -ge 60 ]; then
|
||||
echo "Timeout waiting for listener to be deleted"
|
||||
echo "${RESOURCES}"
|
||||
while true; do
|
||||
local pods
|
||||
if ! pods=$(kubectl get pods -n "${SCALE_SET_NAMESPACE}" -l "actions.github.com/scale-set-name=${SCALE_SET_NAME},e2e.arc/upgrade-marker=${UPGRADE_MARKER}" -o jsonpath='{.items[*].metadata.name}'); then
|
||||
echo "Failed to get pods: $pods"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Waiting for listener to be deleted"
|
||||
echo "Listener count: ${LISTENER_COUNT} target: 0 | Runners count: ${RUNNERS_COUNT} target: 3"
|
||||
|
||||
sleep 1
|
||||
count=$((count + 1))
|
||||
done
|
||||
}
|
||||
|
||||
function assert_listener_recreated() {
|
||||
count=0
|
||||
while true; do
|
||||
LISTENER_COUNT="$(kubectl get pods -l actions.github.com/scale-set-name="${SCALE_SET_NAME}" -n "${ARC_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')"
|
||||
RUNNERS_COUNT="$(kubectl get pods -l app.kubernetes.io/component=runner -n "${SCALE_SET_NAMESPACE}" --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')"
|
||||
RESOURCES="$(kubectl get pods -A)"
|
||||
|
||||
if [ "${LISTENER_COUNT}" -eq 1 ]; then
|
||||
echo "Listener is up!"
|
||||
echo "${RESOURCES}"
|
||||
if [[ -n "$pods" ]]; then
|
||||
echo "Found idle pod with upgrade marker: $pods"
|
||||
return 0
|
||||
fi
|
||||
if [ "${count}" -ge 120 ]; then
|
||||
echo "Timeout waiting for listener to be recreated"
|
||||
echo "${RESOURCES}"
|
||||
|
||||
if ((count >= 30)); then
|
||||
echo "Timeout waiting for idle pod recreation after upgrade"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Waiting for listener to be recreated"
|
||||
echo "Listener count: ${LISTENER_COUNT} target: 1 | Runners count: ${RUNNERS_COUNT} target: 0"
|
||||
|
||||
sleep 1
|
||||
count=$((count + 1))
|
||||
echo "No idle pod with upgrade marker found yet, retrying... ($((count + 1))/30)"
|
||||
sleep 10
|
||||
((count++))
|
||||
done
|
||||
|
||||
}
|
||||
|
||||
function main() {
|
||||
local failed=()
|
||||
local run_id=""
|
||||
|
||||
build_image
|
||||
create_cluster
|
||||
install_arc
|
||||
install_scale_set
|
||||
|
||||
WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" run_workflow || failed+=("run_workflow")
|
||||
|
||||
upgrade_scale_set || failed+=("upgrade_scale_set")
|
||||
assert_listener_deleted || failed+=("assert_listener_deleted")
|
||||
assert_listener_recreated || failed+=("assert_listener_recreated")
|
||||
|
||||
if ! run_id=$(WORKFLOW_FILE="${WORKFLOW_FILE}" SCALE_SET_NAME="${SCALE_SET_NAME}" start_workflow); then
|
||||
failed+=("run_workflow")
|
||||
fi
|
||||
|
||||
assert_idle_pod_recreated || failed+=("assert_idle_pod_recreated")
|
||||
|
||||
if [[ -n "${run_id}" ]]; then
|
||||
wait_for_run_completion "${run_id}" || failed+=("wait_for_run_completion")
|
||||
fi
|
||||
|
||||
INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set")
|
||||
|
||||
Reference in New Issue
Block a user