public_git
/
actions-runner-controller
mirror of https://github.com/actions-runner-controller/actions-runner-controller.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: remove callbacks resulting in scales due to incomplete response (#2671) 

Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>
This commit is contained in:
Lars Lange 2023-07-25 02:04:54 +02:00 committed by GitHub
parent 5271f316e6
commit 297442975e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 71 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
controllers/actions.summerwind.net/autoscaling.go
View File

@ -118,10 +118,10 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
} }
var total, inProgress, queued, completed, unknown int var total, inProgress, queued, completed, unknown int
type callback func() listWorkflowJobs := func(user string, repoName string, runID int64) {
listWorkflowJobs := func(user string, repoName string, runID int64, fallback_cb callback) {
if runID == 0 { if runID == 0 {
fallback_cb() // should not happen in reality
r.Log.Info("Detected run with no runID of 0, ignoring the case and not scaling.", "repo_name", repoName, "run_id", runID)
return return
} }
opt := github.ListWorkflowJobsOptions{ListOptions: github.ListOptions{PerPage: 50}} opt := github.ListWorkflowJobsOptions{ListOptions: github.ListOptions{PerPage: 50}}
@ -139,7 +139,8 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
opt.Page = resp.NextPage opt.Page = resp.NextPage
} }
if len(allJobs) == 0 { if len(allJobs) == 0 {
fallback_cb() // GitHub API can return run with empty job array - should be ignored
r.Log.Info("Detected run with no jobs, ignoring the case and not scaling.", "repo_name", repoName, "run_id", runID)
} else { } else {
JOB: JOB:
for _, job := range allJobs { for _, job := range allJobs {
@ -201,9 +202,9 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
case "completed": case "completed":
completed++ completed++
case "in_progress": case "in_progress":
listWorkflowJobs(user, repoName, run.GetID(), func() { inProgress++ }) listWorkflowJobs(user, repoName, run.GetID())
case "queued": case "queued":
listWorkflowJobs(user, repoName, run.GetID(), func() { queued++ }) listWorkflowJobs(user, repoName, run.GetID())
default: default:
unknown++ unknown++
} }

77
controllers/actions.summerwind.net/autoscaling_test.go
View File

@ -61,8 +61,9 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
want int want int
err string err string
}{ }{
// case_0
// Legacy functionality // Legacy functionality
// 3 demanded, max at 3 // 0 demanded due to zero runID, min at 2
{ {
repo: "test/valid", repo: "test/valid",
min: intPtr(2), min: intPtr(2),
@ -70,9 +71,10 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`, workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`, workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
want: 3, want: 2,
}, },
// Explicitly speified the default `self-hosted` label which is ignored by the simulator, // case_1
// Explicitly specified the default `self-hosted` label which is ignored by the simulator,
// as we assume that GitHub Actions automatically associates the `self-hosted` label to every self-hosted runner. // as we assume that GitHub Actions automatically associates the `self-hosted` label to every self-hosted runner.
// 3 demanded, max at 3 // 3 demanded, max at 3
{ {
@ -80,11 +82,17 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
labels: []string{"self-hosted"}, labels: []string{"self-hosted"},
min: intPtr(2), min: intPtr(2),
max: intPtr(3), max: intPtr(3),
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`, workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`, workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
want: 3, workflowJobs: map[int]string{
1: `{"jobs": [{"status": "queued", "labels":["self-hosted"]}]}`,
2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}]}`,
3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}]}`,
},
want: 3,
}, },
// case_2
// 2 demanded, max at 3, currently 3, delay scaling down due to grace period // 2 demanded, max at 3, currently 3, delay scaling down due to grace period
{ {
repo: "test/valid", repo: "test/valid",
@ -97,6 +105,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 3, want: 3,
}, },
// case_3
// 3 demanded, max at 2 // 3 demanded, max at 2
{ {
repo: "test/valid", repo: "test/valid",
@ -107,6 +116,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
want: 2, want: 2,
}, },
// case_4
// 2 demanded, min at 2 // 2 demanded, min at 2
{ {
repo: "test/valid", repo: "test/valid",
@ -117,6 +127,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 2, want: 2,
}, },
// case_5
// 1 demanded, min at 2 // 1 demanded, min at 2
{ {
repo: "test/valid", repo: "test/valid",
@ -127,6 +138,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`, workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
want: 2, want: 2,
}, },
// case_6
// 1 demanded, min at 2 // 1 demanded, min at 2
{ {
repo: "test/valid", repo: "test/valid",
@ -137,6 +149,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 2, want: 2,
}, },
// case_7
// 1 demanded, min at 1 // 1 demanded, min at 1
{ {
repo: "test/valid", repo: "test/valid",
@ -147,6 +160,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`, workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
want: 1, want: 1,
}, },
// case_8
// 1 demanded, min at 1 // 1 demanded, min at 1
{ {
repo: "test/valid", repo: "test/valid",
@ -157,6 +171,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 1, want: 1,
}, },
// case_9
// fixed at 3 // fixed at 3
{ {
repo: "test/valid", repo: "test/valid",
@ -166,9 +181,36 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`, workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`, workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`,
workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}]}"`,
want: 3, want: 1,
},
// Case for empty GitHub Actions reponse - should not trigger scale up
{
description: "GitHub Actions Jobs Array is empty - no scale up",
repo: "test/valid",
min: intPtr(0),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
workflowJobs: map[int]string{
1: `{"jobs": []}`,
},
want: 0,
},
// Case for hosted GitHub Actions run
{
description: "Hosted GitHub Actions run - no scale up",
repo: "test/valid",
min: intPtr(0),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"id": 1, "status":"queued"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
workflowJobs: map[int]string{
1: `{"jobs": [{"status":"queued"}]}`,
},
want: 0,
}, },
{ {
description: "Job-level autoscaling with no explicit runner label (runners have implicit self-hosted, requested self-hosted, 5 jobs from 3 workflows)", description: "Job-level autoscaling with no explicit runner label (runners have implicit self-hosted, requested self-hosted, 5 jobs from 3 workflows)",
repo: "test/valid", repo: "test/valid",
@ -422,7 +464,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
want int want int
err string err string
}{ }{
// 3 demanded, max at 3 // case_0
// 0 demanded due to zero runID, min at 2
{ {
org: "test", org: "test",
repos: []string{"valid"}, repos: []string{"valid"},
@ -431,8 +474,9 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`, workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`, workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
want: 3, want: 2,
}, },
// case_1
// 2 demanded, max at 3, currently 3, delay scaling down due to grace period // 2 demanded, max at 3, currently 3, delay scaling down due to grace period
{ {
org: "test", org: "test",
@ -446,6 +490,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 3, want: 3,
}, },
// case_2
// 3 demanded, max at 2 // 3 demanded, max at 2
{ {
org: "test", org: "test",
@ -457,6 +502,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
want: 2, want: 2,
}, },
// case_3
// 2 demanded, min at 2 // 2 demanded, min at 2
{ {
org: "test", org: "test",
@ -468,6 +514,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 2, want: 2,
}, },
// case_4
// 1 demanded, min at 2 // 1 demanded, min at 2
{ {
org: "test", org: "test",
@ -479,6 +526,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`, workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
want: 2, want: 2,
}, },
// case_5
// 1 demanded, min at 2 // 1 demanded, min at 2
{ {
org: "test", org: "test",
@ -512,6 +560,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 1, want: 1,
}, },
// case_6
// fixed at 3 // fixed at 3
{ {
org: "test", org: "test",
@ -522,8 +571,9 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`, workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`, workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`,
workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"},{"status":"in_progress"},{"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"},{"status":"in_progress"},{"status":"in_progress"}]}"`,
want: 3, want: 1,
}, },
// case_7
// org runner, fixed at 3 // org runner, fixed at 3
{ {
org: "test", org: "test",
@ -534,8 +584,9 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`, workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`, workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`,
workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"},{"status":"in_progress"},{"status":"in_progress"}]}"`, workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"},{"status":"in_progress"},{"status":"in_progress"}]}"`,
want: 3, want: 1,
}, },
// case_8
// org runner, 1 demanded, min at 1, no repos // org runner, 1 demanded, min at 1, no repos
{ {
org: "test", org: "test",