From d0d316252ed1c5f715deacc0b5770deb497521d2 Mon Sep 17 00:00:00 2001 From: Felipe Galindo Sanchez Date: Wed, 16 Feb 2022 02:08:56 -0800 Subject: [PATCH] Option to consider runner group visibility on scale based on webhook (#1062) This will work on GHES but GitHub Enterprise Cloud due to excessive GitHub API calls required. More work is needed, like adding a cache layer to the GitHub client, to make it usable on GitHub Enterprise Cloud. Fixes additional cases from https://github.com/actions-runner-controller/actions-runner-controller/pull/1012 If GitHub auth is provided in the webhooks controller then runner groups with custom visibility are supported. Otherwise, all runner groups will be assumed to be visible to all repositories `getScaleUpTargetWithFunction()` will check if there is an HRA available with the following flow: 1. Search for **repository** HRAs - if so it ends here 2. Get available HRAs in k8s 3. Compute visible runner groups a. If GitHub auth is provided - get all the runner groups that are visible to the repository of the incoming webhook using GitHub API calls. b. If GitHub auth is not provided - assume all runner groups are visible to all repositories 4. Search for **default organization** runners (a.k.a runners from organization's visible default runner group) with matching labels 5. Search for **default enterprise** runners (a.k.a runners from enterprise's visible default runner group) with matching labels 6. Search for **custom organization runner groups** with matching labels 7. Search for **custom enterprise runner groups** with matching labels Co-authored-by: Yusuke Kuoka --- README.md | 14 ++ charts/actions-runner-controller/README.md | 1 + .../templates/githubwebhook.deployment.yaml | 2 +- charts/actions-runner-controller/values.yaml | 1 + cmd/githubwebhookserver/main.go | 7 + .../horizontal_runner_autoscaler_webhook.go | 184 +++++++++-------- controllers/integration_test.go | 185 ++++++++++++++++++ github/github.go | 101 +++------- go.mod | 2 + go.sum | 1 - simulator/runnergroup_visibility.go | 63 ++++++ simulator/runnergroups.go | 180 +++++++++++++++++ simulator/runnergroups_test.go | 94 +++++++++ 13 files changed, 679 insertions(+), 156 deletions(-) create mode 100644 simulator/runnergroup_visibility.go create mode 100644 simulator/runnergroups.go create mode 100644 simulator/runnergroups_test.go diff --git a/README.md b/README.md index 7e014559..69606a70 100644 --- a/README.md +++ b/README.md @@ -1071,6 +1071,20 @@ spec: group: NewGroup ``` +GitHub supports custom visilibity in a Runner Group to make it available to a specific set of repositories only. By default if no GitHub +authentication is included in the GitHub webhook server it will be assumed that all runner groups to be usable in all repositories. +Supporting custom visibility requires to do a few GitHub API calls to find out what are the potential runner groups that are visible to +the webhook's repository, this may incur in increased API rate limiting when using github.com + +This option will be enabled when proper GitHub authentication options (token, app or basic auth is provided) in the GitHub webhook server and `useRunnerGroupsVisibility` is set to true, e.g. + +```yaml +githubWebhookServer: + enabled: false + replicaCount: 1 + useRunnerGroupsVisibility: true +``` + ### Runner Entrypoint Features > Environment variable values must all be strings diff --git a/charts/actions-runner-controller/README.md b/charts/actions-runner-controller/README.md index 89bfdac4..40b76bb9 100644 --- a/charts/actions-runner-controller/README.md +++ b/charts/actions-runner-controller/README.md @@ -75,6 +75,7 @@ All additional docs are kept in the `docs/` folder, this README is solely for do | `admissionWebHooks.caBundle` | Base64-encoded PEM bundle containing the CA that signed the webhook's serving certificate | | | `githubWebhookServer.logLevel` | Set the log level of the githubWebhookServer container | | | `githubWebhookServer.replicaCount` | Set the number of webhook server pods | 1 | +| `githubWebhookServer.useRunnerGroupsVisibility` | Enable supporting runner groups with custom visibility. This will incur in extra API calls and may blow up your budget | false | | `githubWebhookServer.syncPeriod` | Set the period in which the controller reconciles the resources | 10m | | `githubWebhookServer.enabled` | Deploy the webhook server pod | false | | `githubWebhookServer.secret.create` | Deploy the webhook hook secret | false | diff --git a/charts/actions-runner-controller/templates/githubwebhook.deployment.yaml b/charts/actions-runner-controller/templates/githubwebhook.deployment.yaml index 624a98c2..176ee346 100644 --- a/charts/actions-runner-controller/templates/githubwebhook.deployment.yaml +++ b/charts/actions-runner-controller/templates/githubwebhook.deployment.yaml @@ -69,7 +69,7 @@ spec: - name: GITHUB_UPLOAD_URL value: {{ .Values.githubUploadURL }} {{- end }} - {{- if .Values.authSecret.enabled }} + {{- if and .Values.githubWebhookServer.useRunnerGroupsVisibility .Values.authSecret.enabled }} - name: GITHUB_TOKEN valueFrom: secretKeyRef: diff --git a/charts/actions-runner-controller/values.yaml b/charts/actions-runner-controller/values.yaml index e724622c..e39e896a 100644 --- a/charts/actions-runner-controller/values.yaml +++ b/charts/actions-runner-controller/values.yaml @@ -169,6 +169,7 @@ githubWebhookServer: enabled: false replicaCount: 1 syncPeriod: 10m + useRunnerGroupsVisibility: false secret: create: false name: "github-webhook-server" diff --git a/cmd/githubwebhookserver/main.go b/cmd/githubwebhookserver/main.go index 2c410a52..8b4982ed 100644 --- a/cmd/githubwebhookserver/main.go +++ b/cmd/githubwebhookserver/main.go @@ -142,6 +142,11 @@ func main() { } }) + // In order to support runner groups with custom visibility (selected repositories), we need to perform some GitHub API calls. + // Let the user define if they want to opt-in supporting this option by providing the proper GitHub authentication parameters + // Without an opt-in, runner groups with custom visibility won't be supported to save API calls + // That is, all runner groups managed by ARC are assumed to be visible to any repositories, + // which is wrong when you have one or more non-default runner groups in your organization or enterprise. if len(c.Token) > 0 || (c.AppID > 0 && c.AppInstallationID > 0 && c.AppPrivateKey != "") || (len(c.BasicauthUsername) > 0 && len(c.BasicauthPassword) > 0) { ghClient, err = c.NewClient() if err != nil { @@ -149,6 +154,8 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Runner") os.Exit(1) } + } else { + setupLog.Info("GitHub client is not initialized. Runner groups with custom visibility are not supported. If needed, please provide GitHub authentication. This will incur in extra GitHub API calls") } ctrl.SetLogger(logger) diff --git a/controllers/horizontal_runner_autoscaler_webhook.go b/controllers/horizontal_runner_autoscaler_webhook.go index 3db381a7..794a9590 100644 --- a/controllers/horizontal_runner_autoscaler_webhook.go +++ b/controllers/horizontal_runner_autoscaler_webhook.go @@ -38,6 +38,7 @@ import ( "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1" "github.com/actions-runner-controller/actions-runner-controller/github" + "github.com/actions-runner-controller/actions-runner-controller/simulator" ) const ( @@ -476,37 +477,14 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleUpTargetWithF return nil, nil } - // Search for organization runner HRAs in default runner group - if target, err := scaleTarget(owner); err != nil { - log.Error(err, "finding organizational runner", "organization", owner) - return nil, err - } else if target != nil { - log.Info("job scale up target is organizational runners", "organization", owner) - return target, nil - } - - if enterprise != "" { - // Search for enterprise runner HRAs in default runner group - if target, err := scaleTarget(enterpriseKey(enterprise)); err != nil { - log.Error(err, "finding enterprise runner", "enterprise", enterprise) - return nil, err - } else if target != nil { - log.Info("scale up target is default enterprise runners", "enterprise", enterprise) - return target, nil - } - } - - // At this point there were no default organization/enterprise runners available to use, try now - // searching in runner groups - - // We need to get the potential runner groups first to avoid spending API queries needless. Once/if GitHub improves an + // Find the potential runner groups first to avoid spending API queries needless. Once/if GitHub improves an // API to find related/linked runner groups from a specific repository this logic could be removed - availableEnterpriseGroups, availableOrganizationGroups, err := autoscaler.getPotentialGroupsFromHRAs(ctx, enterprise, owner) + managedRunnerGroups, err := autoscaler.getManagedRunnerGroupsFromHRAs(ctx, enterprise, owner) if err != nil { - log.Error(err, "finding potential organization runner groups from HRAs", "organization", owner) + log.Error(err, "finding potential organization/enterprise runner groups from HRAs", "organization", owner) return nil, err } - if len(availableEnterpriseGroups) == 0 && len(availableOrganizationGroups) == 0 { + if managedRunnerGroups.IsEmpty() { log.V(1).Info("no repository/organizational/enterprise runner found", "repository", repositoryRunnerKey, "organization", owner, @@ -514,57 +492,88 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleUpTargetWithF ) } - var enterpriseGroups []string - var organizationGroups []string + var visibleGroups *simulator.VisibleRunnerGroups if autoscaler.GitHubClient != nil { + simu := &simulator.Simulator{ + Client: autoscaler.GitHubClient, + } // Get available organization runner groups and enterprise runner groups for a repository - // These are the sum of runner groups with repository access = All repositories plus - // runner groups where owner/repo has access to - enterpriseGroups, organizationGroups, err = autoscaler.GitHubClient.GetRunnerGroupsFromRepository(ctx, owner, repositoryRunnerKey, availableEnterpriseGroups, availableOrganizationGroups) - log.V(1).Info("Searching in runner groups", "enterprise.groups", enterpriseGroups, "organization.groups", organizationGroups) + // These are the sum of runner groups with repository access = All repositories and runner groups + // where owner/repo has access to as well. The list will include default runner group also if it has access to + visibleGroups, err = simu.GetRunnerGroupsVisibleToRepository(ctx, owner, repositoryRunnerKey, managedRunnerGroups) + log.V(1).Info("Searching in runner groups", "groups", visibleGroups) if err != nil { log.Error(err, "Unable to find runner groups from repository", "organization", owner, "repository", repo) - return nil, nil + return nil, fmt.Errorf("error while finding visible runner groups: %v", err) } } else { // For backwards compatibility if GitHub authentication is not configured, we assume all runner groups have // visibility=all to honor the previous implementation, therefore any available enterprise/organization runner - // is a potential target for scaling - enterpriseGroups = availableEnterpriseGroups - organizationGroups = availableOrganizationGroups + // is a potential target for scaling. This will also avoid doing extra API calls caused by + // GitHubClient.GetRunnerGroupsVisibleToRepository in case users are not using custom visibility on their runner + // groups or they are using only default runner groups + visibleGroups = managedRunnerGroups } - for _, group := range organizationGroups { - if target, err := scaleTarget(organizationalRunnerGroupKey(owner, group)); err != nil { - log.Error(err, "finding organizational runner group", "organization", owner) - return nil, err - } else if target != nil { - log.Info(fmt.Sprintf("job scale up target is organizational runner group %s", target.Name), "organization", owner) - return target, nil + scaleTargetKey := func(rg simulator.RunnerGroup) string { + switch rg.Kind { + case simulator.Default: + switch rg.Scope { + case simulator.Organization: + return owner + case simulator.Enterprise: + return enterpriseKey(enterprise) + } + case simulator.Custom: + switch rg.Scope { + case simulator.Organization: + return organizationalRunnerGroupKey(owner, rg.Name) + case simulator.Enterprise: + return enterpriseRunnerGroupKey(enterprise, rg.Name) + } } + return "" } - for _, group := range enterpriseGroups { - if target, err := scaleTarget(enterpriseRunnerGroupKey(enterprise, group)); err != nil { - log.Error(err, "finding enterprise runner group", "enterprise", owner) - return nil, err - } else if target != nil { - log.Info(fmt.Sprintf("job scale up target is enterprise runner group %s", target.Name), "enterprise", owner) - return target, nil + log.Info("groups", "groups", visibleGroups) + + var t *ScaleTarget + + traverseErr := visibleGroups.Traverse(func(rg simulator.RunnerGroup) (bool, error) { + key := scaleTargetKey(rg) + + target, err := scaleTarget(key) + + if err != nil { + log.Error(err, "finding runner group", "enterprise", enterprise, "organization", owner, "repository", repo, "key", key) + return false, err + } else if target == nil { + return false, nil } + + t = target + log.Info("job scale up target found", "enterprise", enterprise, "organization", owner, "repository", repo, "key", key) + + return true, nil + }) + + if traverseErr != nil { + return nil, err } - log.V(1).Info("no repository/organizational/enterprise runner found", - "repository", repositoryRunnerKey, - "organization", owner, - "enterprises", enterprise, - ) - return nil, nil + if t == nil { + log.V(1).Info("no repository/organizational/enterprise runner found", + "repository", repositoryRunnerKey, + "organization", owner, + "enterprise", enterprise, + ) + } + + return t, nil } -func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getPotentialGroupsFromHRAs(ctx context.Context, enterprise, org string) ([]string, []string, error) { - var enterpriseRunnerGroups []string - var orgRunnerGroups []string +func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getManagedRunnerGroupsFromHRAs(ctx context.Context, enterprise, org string) (*simulator.VisibleRunnerGroups, error) { + groups := simulator.NewVisibleRunnerGroups() ns := autoscaler.Namespace var defaultListOpts []client.ListOption @@ -579,36 +588,57 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getPotentialGroupsFro var hraList v1alpha1.HorizontalRunnerAutoscalerList if err := autoscaler.List(ctx, &hraList, opts...); err != nil { - return orgRunnerGroups, enterpriseRunnerGroups, err + return groups, err } for _, hra := range hraList.Items { - switch hra.Spec.ScaleTargetRef.Kind { + var o, e, g string + + kind := hra.Spec.ScaleTargetRef.Kind + switch kind { case "RunnerSet": var rs v1alpha1.RunnerSet if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rs); err != nil { - return orgRunnerGroups, enterpriseRunnerGroups, err - } - if rs.Spec.Organization == org && rs.Spec.Group != "" { - orgRunnerGroups = append(orgRunnerGroups, rs.Spec.Group) - } - if rs.Spec.Enterprise == enterprise && rs.Spec.Group != "" { - enterpriseRunnerGroups = append(enterpriseRunnerGroups, rs.Spec.Group) + return groups, err } + o, e, g = rs.Spec.Organization, rs.Spec.Enterprise, rs.Spec.Group case "RunnerDeployment", "": var rd v1alpha1.RunnerDeployment if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rd); err != nil { - return orgRunnerGroups, enterpriseRunnerGroups, err - } - if rd.Spec.Template.Spec.Organization == org && rd.Spec.Template.Spec.Group != "" { - orgRunnerGroups = append(orgRunnerGroups, rd.Spec.Template.Spec.Group) - } - if rd.Spec.Template.Spec.Enterprise == enterprise && rd.Spec.Template.Spec.Group != "" { - enterpriseRunnerGroups = append(enterpriseRunnerGroups, rd.Spec.Template.Spec.Group) + return groups, err } + o, e, g = rd.Spec.Template.Spec.Organization, rd.Spec.Template.Spec.Enterprise, rd.Spec.Template.Spec.Group + default: + return nil, fmt.Errorf("unsupported scale target kind: %v", kind) + } + + if g == "" { + continue + } + + if e == "" && o == "" { + autoscaler.Log.V(1).Info( + "invalid runner group config in scale target: spec.group must be set along with either spec.enterprise or spec.organization", + "scaleTargetKind", kind, + "group", g, + "enterprise", e, + "organization", o, + ) + + continue + } + + if e != enterprise && o != org { + continue + } + + rg := simulator.NewRunnerGroupFromProperties(e, o, g) + + if err := groups.Add(rg); err != nil { + return groups, fmt.Errorf("failed adding visible group from HRA %s/%s: %w", hra.Namespace, hra.Name, err) } } - return enterpriseRunnerGroups, orgRunnerGroups, nil + return groups, nil } func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getJobScaleTarget(ctx context.Context, name string, labels []string) (*ScaleTarget, error) { diff --git a/controllers/integration_test.go b/controllers/integration_test.go index 328294f8..d87bdb18 100644 --- a/controllers/integration_test.go +++ b/controllers/integration_test.go @@ -1077,6 +1077,169 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() { } }) + It("should be able to scale visible organization runner group with default labels", func() { + name := "example-runnerdeploy" + + { + rd := &actionsv1alpha1.RunnerDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns.Name, + }, + Spec: actionsv1alpha1.RunnerDeploymentSpec{ + Replicas: intPtr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + }, + }, + Template: actionsv1alpha1.RunnerTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "foo": "bar", + }, + }, + Spec: actionsv1alpha1.RunnerSpec{ + RunnerConfig: actionsv1alpha1.RunnerConfig{ + Repository: "test/valid", + Image: "bar", + Group: "baz", + }, + RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{ + Env: []corev1.EnvVar{ + {Name: "FOO", Value: "FOOVALUE"}, + }, + }, + }, + }, + }, + } + + ExpectCreate(ctx, rd, "test RunnerDeployment") + + hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns.Name, + }, + Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{ + ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{ + Name: name, + }, + MinReplicas: intPtr(1), + MaxReplicas: intPtr(5), + ScaleDownDelaySecondsAfterScaleUp: intPtr(1), + ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{ + { + GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{}, + Amount: 1, + Duration: metav1.Duration{Duration: time.Minute}, + }, + }, + }, + } + + ExpectCreate(ctx, hra, "test HorizontalRunnerAutoscaler") + + ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1) + ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1) + } + + { + env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners") + } + + // Scale-up to 2 replicas on first workflow_job webhook event + { + env.SendWorkflowJobEvent("test", "valid", "pending", "created", []string{"self-hosted"}) + ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook") + ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event") + env.ExpectRegisteredNumberCountEventuallyEquals(2, "count of fake list runners") + } + }) + + It("should be able to scale visible organization runner group with custom labels", func() { + name := "example-runnerdeploy" + + { + rd := &actionsv1alpha1.RunnerDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns.Name, + }, + Spec: actionsv1alpha1.RunnerDeploymentSpec{ + Replicas: intPtr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + }, + }, + Template: actionsv1alpha1.RunnerTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "foo": "bar", + }, + }, + Spec: actionsv1alpha1.RunnerSpec{ + RunnerConfig: actionsv1alpha1.RunnerConfig{ + Repository: "test/valid", + Image: "bar", + Group: "baz", + Labels: []string{"custom-label"}, + }, + RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{ + Env: []corev1.EnvVar{ + {Name: "FOO", Value: "FOOVALUE"}, + }, + }, + }, + }, + }, + } + + ExpectCreate(ctx, rd, "test RunnerDeployment") + + hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns.Name, + }, + Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{ + ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{ + Name: name, + }, + MinReplicas: intPtr(1), + MaxReplicas: intPtr(5), + ScaleDownDelaySecondsAfterScaleUp: intPtr(1), + ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{ + { + GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{}, + Amount: 1, + Duration: metav1.Duration{Duration: time.Minute}, + }, + }, + }, + } + + ExpectCreate(ctx, hra, "test HorizontalRunnerAutoscaler") + + ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1) + ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1) + } + + { + env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners") + } + + // Scale-up to 2 replicas on first workflow_job webhook event + { + env.SendWorkflowJobEvent("test", "valid", "pending", "created", []string{"custom-label"}) + ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook") + ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event") + env.ExpectRegisteredNumberCountEventuallyEquals(2, "count of fake list runners") + } + }) + }) }) @@ -1208,6 +1371,28 @@ func (env *testEnvironment) SendUserCheckRunEvent(owner, repo, status, action st ExpectWithOffset(1, resp.StatusCode).To(Equal(200)) } +func (env *testEnvironment) SendWorkflowJobEvent(owner, repo, status, action string, labels []string) { + resp, err := sendWebhook(env.webhookServer, "workflow_job", &github.WorkflowJobEvent{ + Org: &github.Organization{ + Name: github.String(owner), + }, + WorkflowJob: &github.WorkflowJob{ + Labels: labels, + }, + Action: github.String("queued"), + Repo: &github.Repository{ + Name: github.String(repo), + Owner: &github.User{ + Login: github.String(owner), + Type: github.String("Organization"), + }, + }, + }) + + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "failed to send check_run event") + + ExpectWithOffset(1, resp.StatusCode).To(Equal(200)) +} func (env *testEnvironment) SyncRunnerRegistrations() { var runnerList actionsv1alpha1.RunnerList diff --git a/github/github.go b/github/github.go index 76ed7e08..89a49913 100644 --- a/github/github.go +++ b/github/github.go @@ -224,74 +224,9 @@ func (c *Client) ListRunners(ctx context.Context, enterprise, org, repo string) return runners, nil } -func (c *Client) GetRunnerGroupsFromRepository(ctx context.Context, org, repo string, potentialEnterpriseGroups []string, potentialOrgGroups []string) ([]string, []string, error) { - - var enterpriseRunnerGroups []string - var orgRunnerGroups []string - - if org != "" { - runnerGroups, err := c.getOrganizationRunnerGroups(ctx, org, repo) - if err != nil { - return enterpriseRunnerGroups, orgRunnerGroups, err - } - for _, runnerGroup := range runnerGroups { - if runnerGroup.GetInherited() { // enterprise runner groups - if !containsString(potentialEnterpriseGroups, runnerGroup.GetName()) { - continue - } - if runnerGroup.GetVisibility() == "all" { - enterpriseRunnerGroups = append(enterpriseRunnerGroups, runnerGroup.GetName()) - } else { - hasAccess, err := c.hasRepoAccessToOrganizationRunnerGroup(ctx, org, runnerGroup.GetID(), repo) - if err != nil { - return enterpriseRunnerGroups, orgRunnerGroups, err - } - if hasAccess { - enterpriseRunnerGroups = append(enterpriseRunnerGroups, runnerGroup.GetName()) - } - } - } else { // organization runner groups - if !containsString(potentialOrgGroups, runnerGroup.GetName()) { - continue - } - if runnerGroup.GetVisibility() == "all" { - orgRunnerGroups = append(orgRunnerGroups, runnerGroup.GetName()) - } else { - hasAccess, err := c.hasRepoAccessToOrganizationRunnerGroup(ctx, org, runnerGroup.GetID(), repo) - if err != nil { - return enterpriseRunnerGroups, orgRunnerGroups, err - } - if hasAccess { - orgRunnerGroups = append(orgRunnerGroups, runnerGroup.GetName()) - } - } - } - } - } - return enterpriseRunnerGroups, orgRunnerGroups, nil -} - -func (c *Client) hasRepoAccessToOrganizationRunnerGroup(ctx context.Context, org string, runnerGroupId int64, repo string) (bool, error) { - opts := github.ListOptions{PerPage: 100} - for { - list, res, err := c.Client.Actions.ListRepositoryAccessRunnerGroup(ctx, org, runnerGroupId, &opts) - if err != nil { - return false, fmt.Errorf("failed to list repository access for runner group: %w", err) - } - for _, githubRepo := range list.Repositories { - if githubRepo.GetFullName() == repo { - return true, nil - } - } - if res.NextPage == 0 { - break - } - opts.Page = res.NextPage - } - return false, nil -} - -func (c *Client) getOrganizationRunnerGroups(ctx context.Context, org, repo string) ([]*github.RunnerGroup, error) { +// ListOrganizationRunnerGroups returns all the runner groups defined in the organization and +// inherited to the organization from an enterprise. +func (c *Client) ListOrganizationRunnerGroups(ctx context.Context, org string) ([]*github.RunnerGroup, error) { var runnerGroups []*github.RunnerGroup opts := github.ListOptions{PerPage: 100} @@ -311,6 +246,27 @@ func (c *Client) getOrganizationRunnerGroups(ctx context.Context, org, repo stri return runnerGroups, nil } +func (c *Client) ListRunnerGroupRepositoryAccesses(ctx context.Context, org string, runnerGroupId int64) ([]*github.Repository, error) { + var repos []*github.Repository + + opts := github.ListOptions{PerPage: 100} + for { + list, res, err := c.Client.Actions.ListRepositoryAccessRunnerGroup(ctx, org, runnerGroupId, &opts) + if err != nil { + return nil, fmt.Errorf("failed to list repository access for runner group: %w", err) + } + + repos = append(repos, list.Repositories...) + if res.NextPage == 0 { + break + } + + opts.Page = res.NextPage + } + + return repos, nil +} + // cleanup removes expired registration tokens. func (c *Client) cleanup() { c.mu.Lock() @@ -480,12 +436,3 @@ func (r *Client) IsRunnerBusy(ctx context.Context, enterprise, org, repo, name s return false, &RunnerNotFound{runnerName: name} } - -func containsString(list []string, value string) bool { - for _, item := range list { - if item == value { - return true - } - } - return false -} diff --git a/go.mod b/go.mod index f14fc984..f9627220 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.17.0 github.com/prometheus/client_golang v1.11.0 + github.com/stretchr/testify v1.7.0 github.com/teambition/rrule-go v1.7.2 go.uber.org/zap v1.20.0 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 @@ -47,6 +48,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.28.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect diff --git a/go.sum b/go.sum index 01075c2c..d9d4388c 100644 --- a/go.sum +++ b/go.sum @@ -508,7 +508,6 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= diff --git a/simulator/runnergroup_visibility.go b/simulator/runnergroup_visibility.go new file mode 100644 index 00000000..43d33cff --- /dev/null +++ b/simulator/runnergroup_visibility.go @@ -0,0 +1,63 @@ +package simulator + +import ( + "context" + "fmt" + + "github.com/actions-runner-controller/actions-runner-controller/github" +) + +type Simulator struct { + Client *github.Client +} + +func (c *Simulator) GetRunnerGroupsVisibleToRepository(ctx context.Context, org, repo string, managed *VisibleRunnerGroups) (*VisibleRunnerGroups, error) { + visible := NewVisibleRunnerGroups() + + if org == "" { + panic(fmt.Sprintf("BUG: owner should not be empty in this context. repo=%v", repo)) + } + + runnerGroups, err := c.Client.ListOrganizationRunnerGroups(ctx, org) + if err != nil { + return visible, err + } + + for _, runnerGroup := range runnerGroups { + ref := NewRunnerGroupFromGitHub(runnerGroup) + + if !managed.Includes(ref) { + continue + } + + if runnerGroup.GetVisibility() != "all" { + hasAccess, err := c.hasRepoAccessToOrganizationRunnerGroup(ctx, org, runnerGroup.GetID(), repo) + if err != nil { + return visible, err + } + + if !hasAccess { + continue + } + } + + visible.Add(ref) + } + + return visible, nil +} + +func (c *Simulator) hasRepoAccessToOrganizationRunnerGroup(ctx context.Context, org string, runnerGroupId int64, repo string) (bool, error) { + repos, err := c.Client.ListRunnerGroupRepositoryAccesses(ctx, org, runnerGroupId) + if err != nil { + return false, err + } + + for _, githubRepo := range repos { + if githubRepo.GetFullName() == repo { + return true, nil + } + } + + return false, nil +} diff --git a/simulator/runnergroups.go b/simulator/runnergroups.go new file mode 100644 index 00000000..ccc961c3 --- /dev/null +++ b/simulator/runnergroups.go @@ -0,0 +1,180 @@ +package simulator + +import ( + "fmt" + "sort" + + "github.com/google/go-github/v39/github" +) + +type RunnerGroupScope int + +const ( + Organization RunnerGroupScope = iota + Enterprise +) + +func (s RunnerGroupScope) String() string { + switch s { + case Organization: + return "Organization" + case Enterprise: + return "Enterprise" + default: + panic(fmt.Sprintf("unimplemented RunnerGroupScope: %v", int(s))) + } +} + +type RunnerGroupKind int + +const ( + Default RunnerGroupKind = iota + Custom +) + +func (s RunnerGroupKind) String() string { + switch s { + case Default: + return "Default" + case Custom: + return "Custom" + default: + panic(fmt.Sprintf("unimplemented RunnerGroupKind: %v", int(s))) + } +} + +func NewRunnerGroupFromGitHub(g *github.RunnerGroup) RunnerGroup { + var name string + if !g.GetDefault() { + name = g.GetName() + } + + var scope RunnerGroupScope + + if g.GetInherited() { + scope = Enterprise + } else { + scope = Organization + } + + return newRunnerGroup(scope, name) +} + +func NewRunnerGroupFromProperties(enterprise, organization, group string) RunnerGroup { + var scope RunnerGroupScope + + if enterprise != "" { + scope = Enterprise + } else { + scope = Organization + } + + return newRunnerGroup(scope, group) +} + +// newRunnerGroup creates a new RunnerGroup instance from the provided arguments. +// There's a convention that an empty name implies a default runner group. +func newRunnerGroup(scope RunnerGroupScope, name string) RunnerGroup { + if name == "" { + return RunnerGroup{ + Scope: scope, + Kind: Default, + Name: "", + } + } + + return RunnerGroup{ + Scope: scope, + Kind: Custom, + Name: name, + } +} + +type RunnerGroup struct { + Scope RunnerGroupScope + Kind RunnerGroupKind + Name string +} + +// VisibleRunnerGroups is a set of enterprise and organization runner groups +// that are visible to a GitHub repository. +// GitHub Actions chooses one of such visible group on which the workflow job is scheduled. +// ARC chooses the same group as Actions as the scale target. +type VisibleRunnerGroups struct { + // sortedGroups is a pointer to a mutable list of RunnerGroups that contains all the runner sortedGroups + // that are visible to the repository, including organization sortedGroups defined at the organization level, + // and enterprise sortedGroups that are inherited down to the organization. + sortedGroups []RunnerGroup +} + +func NewVisibleRunnerGroups() *VisibleRunnerGroups { + return &VisibleRunnerGroups{} +} + +func (g *VisibleRunnerGroups) IsEmpty() bool { + return len(g.sortedGroups) == 0 +} + +func (r *VisibleRunnerGroups) Includes(ref RunnerGroup) bool { + for _, r := range r.sortedGroups { + if r.Scope == ref.Scope && r.Kind == ref.Kind && r.Name == ref.Name { + return true + } + } + return false +} + +// Add adds a runner group into VisibleRunnerGroups +// at a certain position in the list so that +// Traverse can return runner groups in order of higher precedence to lower precedence. +func (g *VisibleRunnerGroups) Add(rg RunnerGroup) error { + n := len(g.sortedGroups) + i := sort.Search(n, func(i int) bool { + data := g.sortedGroups[i] + + if rg.Kind > data.Kind { + return false + } else if rg.Kind < data.Kind { + return true + } + + if rg.Scope > data.Scope { + return false + } else if rg.Scope < data.Scope { + return true + } + + return false + }) + + g.insert(rg, i) + + return nil +} + +func (g *VisibleRunnerGroups) insert(rg RunnerGroup, i int) { + var result []RunnerGroup + + result = append(result, g.sortedGroups[:i]...) + result = append(result, rg) + result = append(result, g.sortedGroups[i:]...) + + g.sortedGroups = result +} + +// Traverse traverses all the runner groups visible to a repository +// in order of higher precedence to lower precedence. +func (g *VisibleRunnerGroups) Traverse(f func(RunnerGroup) (bool, error)) error { + for _, rg := range g.sortedGroups { + ok, err := f(rg) + if err != nil { + return err + } + + if ok { + return nil + } + } + + return nil +} diff --git a/simulator/runnergroups_test.go b/simulator/runnergroups_test.go new file mode 100644 index 00000000..f95ff406 --- /dev/null +++ b/simulator/runnergroups_test.go @@ -0,0 +1,94 @@ +package simulator + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestVisibleRunnerGroupsInsert(t *testing.T) { + g := NewVisibleRunnerGroups() + + orgDefault := NewRunnerGroupFromProperties("", "myorg1", "") + orgCustom := NewRunnerGroupFromProperties("", "myorg1", "myorg1group1") + enterpriseDefault := NewRunnerGroupFromProperties("myenterprise1", "", "") + + g.insert(orgCustom, 0) + g.insert(orgDefault, 0) + g.insert(enterpriseDefault, 1) + + var got []RunnerGroup + + err := g.Traverse(func(rg RunnerGroup) (bool, error) { + got = append(got, rg) + return false, nil + }) + + require.NoError(t, err) + require.Equal(t, []RunnerGroup{orgDefault, enterpriseDefault, orgCustom}, got, "Unexpected result") +} + +func TestVisibleRunnerGroups(t *testing.T) { + v := NewVisibleRunnerGroups() + + requireGroups := func(t *testing.T, included, notIncluded []RunnerGroup) { + t.Helper() + + for _, rg := range included { + if !v.Includes(rg) { + t.Errorf("%v must be included", rg) + } + } + + for _, rg := range notIncluded { + if v.Includes(rg) { + t.Errorf("%v must not be included", rg) + } + } + + var got []RunnerGroup + + err := v.Traverse(func(rg RunnerGroup) (bool, error) { + got = append(got, rg) + + return false, nil + }) + + require.NoError(t, err) + require.Equal(t, included, got) + } + + orgDefault := NewRunnerGroupFromProperties("", "myorg1", "") + orgCustom := NewRunnerGroupFromProperties("", "myorg1", "myorg1group1") + enterpriseDefault := NewRunnerGroupFromProperties("myenterprise1", "", "") + enterpriseCustom := NewRunnerGroupFromProperties("myenterprise1", "", "myenterprise1group1") + + requireGroups(t, nil, []RunnerGroup{orgDefault, enterpriseDefault, orgCustom, enterpriseCustom}) + + v.Add(orgCustom) + + requireGroups(t, []RunnerGroup{orgCustom}, []RunnerGroup{orgDefault, enterpriseDefault, enterpriseCustom}) + + v.Add(orgDefault) + + requireGroups(t, []RunnerGroup{orgDefault, orgCustom}, []RunnerGroup{enterpriseDefault, enterpriseCustom}) + + v.Add(enterpriseCustom) + + requireGroups(t, []RunnerGroup{orgDefault, orgCustom, enterpriseCustom}, []RunnerGroup{enterpriseDefault}) + + v.Add(enterpriseDefault) + + requireGroups(t, []RunnerGroup{orgDefault, enterpriseDefault, orgCustom, enterpriseCustom}, nil) + + var first []RunnerGroup + + err := v.Traverse(func(rg RunnerGroup) (bool, error) { + first = append(first, rg) + + return true, nil + }) + + require.NoError(t, err) + require.Equal(t, []RunnerGroup{orgDefault}, first) +}