public_git
/
actions-runner-controller
mirror of https://github.com/actions-runner-controller/actions-runner-controller.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix TotalNumberOfQueuedAndInProgressWorkflowRuns to work with a lot of remaining `completed` jobs (#316) 

I have heard from some user that they have hundred thousands of `status=completed` workflow runs in their repository which effectively blocked TotalNumberOfQueuedAndInProgressWorkflowRuns from working because of GitHub API rate limit due to excessive paginated requests.

This fixes that by separating list-workflow-runs calls to two - one for `queued` and one for `in_progress`, which can make the minimum API call from 1 to 2, but allows it to work regardless of number of remaining `completed` workflow runs.
This commit is contained in:
Yusuke Kuoka 2021-02-16 18:55:55 +09:00 committed by GitHub
parent 7bf712d0d4
commit eb2eaf8130
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 246 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
controllers/autoscaling_test.go
View File

@ -47,7 +47,11 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
min *int
sReplicas *int
sTime *metav1.Time
workflowRuns string
workflowRuns_queued string
workflowRuns_in_progress string
workflowJobs map[int]string
want int
err string
@ -59,6 +63,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
min: intPtr(2),
max: intPtr(3),
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
want: 3,
},
// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
@ -68,7 +74,9 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
max: intPtr(3),
sReplicas: intPtr(3),
sTime: &metav1Now,
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns: `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 3,
},
// 3 demanded, max at 2
@ -77,6 +85,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
min: intPtr(2),
max: intPtr(2),
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
want: 2,
},
// 2 demanded, min at 2
@ -85,6 +95,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
min: intPtr(2),
max: intPtr(3),
workflowRuns: `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 2,
},
// 1 demanded, min at 2
@ -93,6 +105,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
min: intPtr(2),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
want: 2,
},
// 1 demanded, min at 2
@ -101,6 +115,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
min: intPtr(2),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`,
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 2,
},
// 1 demanded, min at 1
@ -109,6 +125,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
min: intPtr(1),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
want: 1,
},
// 1 demanded, min at 1
@ -117,6 +135,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
min: intPtr(1),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`,
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 1,
},
// fixed at 3
@ -126,6 +146,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
max: intPtr(3),
fixed: intPtr(3),
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`,
workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}]}"`,
want: 3,
},
@ -136,6 +158,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
min: intPtr(2),
max: intPtr(10),
workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
workflowJobs: map[int]string{
1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
@ -158,7 +182,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
server := fake.NewServer(
fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns),
fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns, tc.workflowRuns_queued, tc.workflowRuns_in_progress),
fake.WithListWorkflowJobsResponse(200, tc.workflowJobs),
fake.WithListRunnersResponse(200, fake.RunnersListBody),
)
@ -235,7 +259,11 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
min *int
sReplicas *int
sTime *metav1.Time
workflowRuns string
workflowRuns_queued string
workflowRuns_in_progress string
workflowJobs map[int]string
want int
err string
@ -247,6 +275,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
min: intPtr(2),
max: intPtr(3),
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
want: 3,
},
// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
@ -258,6 +288,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
sReplicas: intPtr(3),
sTime: &metav1Now,
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 3,
},
// 3 demanded, max at 2
@ -267,6 +299,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
min: intPtr(2),
max: intPtr(2),
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
want: 2,
},
// 2 demanded, min at 2
@ -276,6 +310,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
min: intPtr(2),
max: intPtr(3),
workflowRuns: `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 2,
},
// 1 demanded, min at 2
@ -285,6 +321,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
min: intPtr(2),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
want: 2,
},
// 1 demanded, min at 2
@ -294,6 +332,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
min: intPtr(2),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`,
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 2,
},
// 1 demanded, min at 1
@ -303,6 +343,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
min: intPtr(1),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
want: 1,
},
// 1 demanded, min at 1
@ -312,6 +354,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
min: intPtr(1),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`,
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
want: 1,
},
// fixed at 3
@ -321,7 +365,9 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
fixed: intPtr(1),
min: intPtr(1),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`,
workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"},{"status":"in_progress"},{"status":"in_progress"}]}"`,
want: 3,
},
// org runner, fixed at 3
@ -331,7 +377,9 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
fixed: intPtr(1),
min: intPtr(1),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`,
workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"},{"status":"in_progress"},{"status":"in_progress"}]}"`,
want: 3,
},
// org runner, 1 demanded, min at 1, no repos
@ -340,6 +388,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
min: intPtr(1),
max: intPtr(3),
workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 0, "workflow_runs":[]}"`,
workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
err: "validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment",
},
@ -351,6 +401,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
min: intPtr(2),
max: intPtr(10),
workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
workflowRuns_queued: `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
workflowJobs: map[int]string{
1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
@ -373,7 +425,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
server := fake.NewServer(
fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns),
fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns, tc.workflowRuns_queued, tc.workflowRuns_in_progress),
fake.WithListWorkflowJobsResponse(200, tc.workflowJobs),
fake.WithListRunnersResponse(200, fake.RunnersListBody),
)

10
controllers/integration_test.go
View File

@ -32,7 +32,11 @@ type testEnvironment struct {
var (
workflowRunsFor3Replicas = `{"total_count": 5, "workflow_runs":[{"status":"queued"}, {"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`
workflowRunsFor3Replicas_queued = `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"queued"}]}"`
workflowRunsFor3Replicas_in_progress = `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`
workflowRunsFor1Replicas = `{"total_count": 6, "workflow_runs":[{"status":"queued"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}]}"`
workflowRunsFor1Replicas_queued = `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`
workflowRunsFor1Replicas_in_progress = `{"total_count": 0, "workflow_runs":[]}"`
)
var webhookServer *httptest.Server
@ -56,6 +60,10 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
responses.ListRepositoryWorkflowRuns = &fake.Handler{
Status: 200,
Body: workflowRunsFor3Replicas,
Statuses: map[string]string{
"queued": workflowRunsFor3Replicas_queued,
"in_progress": workflowRunsFor3Replicas_in_progress,
},
}
fakeRunnerList = fake.NewRunnersList()
responses.ListRunners = fakeRunnerList.HandleList()
@ -342,6 +350,8 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
time.Sleep(time.Second)
responses.ListRepositoryWorkflowRuns.Body = workflowRunsFor1Replicas
responses.ListRepositoryWorkflowRuns.Statuses["queued"] = workflowRunsFor1Replicas_queued
responses.ListRepositoryWorkflowRuns.Statuses["in_progress"] = workflowRunsFor1Replicas_in_progress
var hra actionsv1alpha1.HorizontalRunnerAutoscaler

11
github/fake/fake.go
View File

@ -37,10 +37,21 @@ func (h *ListRunnersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
type Handler struct {
Status int
Body string
Statuses map[string]string
}
func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(h.Status)
status := req.URL.Query().Get("status")
if h.Statuses != nil {
if body, ok := h.Statuses[status]; ok {
fmt.Fprintf(w, body)
return
}
}
fmt.Fprintf(w, h.Body)
}

6
github/fake/options.go
View File

@ -10,11 +10,15 @@ type FixedResponses struct {
type Option func(*ServerConfig)
func WithListRepositoryWorkflowRunsResponse(status int, body string) Option {
func WithListRepositoryWorkflowRunsResponse(status int, body, queued, in_progress string) Option {
return func(c *ServerConfig) {
c.FixedResponses.ListRepositoryWorkflowRuns = &Handler{
Status: status,
Body: body,
Statuses: map[string]string{
"queued": queued,
"in_progress": in_progress,
},
}
}
}

22
github/github.go
View File

@ -210,12 +210,34 @@ func (c *Client) listRunners(ctx context.Context, enterprise, org, repo string,
func (c *Client) ListRepositoryWorkflowRuns(ctx context.Context, user string, repoName string) ([]*github.WorkflowRun, error) {
c.Client.Actions.ListRepositoryWorkflowRuns(ctx, user, repoName, nil)
queued, err := c.listRepositoryWorkflowRuns(ctx, user, repoName, "queued")
if err != nil {
return nil, fmt.Errorf("listing queued workflow runs: %w", err)
}
inProgress, err := c.listRepositoryWorkflowRuns(ctx, user, repoName, "in_progress")
if err != nil {
return nil, fmt.Errorf("listing in_progress workflow runs: %w", err)
}
var workflowRuns []*github.WorkflowRun
workflowRuns = append(workflowRuns, queued...)
workflowRuns = append(workflowRuns, inProgress...)
return workflowRuns, nil
}
func (c *Client) listRepositoryWorkflowRuns(ctx context.Context, user string, repoName, status string) ([]*github.WorkflowRun, error) {
c.Client.Actions.ListRepositoryWorkflowRuns(ctx, user, repoName, nil)
var workflowRuns []*github.WorkflowRun
opts := github.ListWorkflowRunsOptions{
ListOptions: github.ListOptions{
PerPage: 100,
},
Status: status,
}
for {