diff --git a/Dockerfile b/Dockerfile index 88481606..96054a56 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,8 @@ COPY . . RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \ export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) && \ GOARM=$(echo ${TARGETPLATFORM} | cut -d / -f3 | cut -c2-) && \ - go build -a -o manager main.go + go build -a -o manager main.go && \ + go build -a -o github-webhook-server ./cmd/githubwebhookserver # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details @@ -31,6 +32,7 @@ FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/manager . +COPY --from=builder /workspace/github-webhook-server . USER nonroot:nonroot diff --git a/README.md b/README.md index d96b70de..e43b35ed 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,30 @@ This controller operates self-hosted runners for GitHub Actions on your Kubernetes cluster. +ToC: + +- [Motivation](#motivation) +- [Installation](#installation) + - [GitHub Enterprise support](#github-enterprise-support) +- [Setting up authentication with GitHub API](#setting-up-authentication-with-github-api) + - [Using GitHub App](#using-github-app) + - [Using Personal AccessToken ](#using-personal-access-token) +- [Usage](#usage) + - [Repository Runners](#repository-runners) + - [Organization Runners](#organization-runners) + - [Runner Deployments](#runnerdeployments) + - [Autoscaling](#autoscaling) + - [Faster Autoscaling with GitHub Webhook](#faster-autoscaling-with-github-webhook) + - [Runner with DinD](#runner-with-dind) + - [Additional tweaks](#additional-tweaks) + - [Runner labels](#runner-labels) + - [Runer groups](#runner-groups) + - [Using EKS IAM role for service accounts](#using-eks-iam-role-for-service-accounts) + - [Software installed in the runner image](#software-installed-in-the-runner-image) + - [Common errors](#common-errors) +- [Developing](#developing) +- [Alternatives](#alternatives) + ## Motivation [GitHub Actions](https://github.com/features/actions) is a very useful tool for automating development. GitHub Actions jobs are run in the cloud by default, but you may want to run your jobs in your environment. [Self-hosted runner](https://github.com/actions/runner) can be used for such use cases, but requires the provisioning and configuration of a virtual machine instance. Instead if you already have a Kubernetes cluster, it makes more sense to run the self-hosted runner on top of it. @@ -339,7 +363,119 @@ spec: scaleDownFactor: '0.7' ``` -## Runner with DinD +#### Faster Autoscaling with GitHub Webhook + +> This feature is an ADVANCED feature which may require more work to set up. +> Please get prepared to put some time and effort to learn and leverage this feature! + +`actions-runner-controller` has an optional Webhook server that receives GitHub Webhook events and scale +[`RunnerDeployment`s](#runnerdeployments) by updating corresponding [`HorizontalRunnerAutoscaler`s](#autoscaling). + +Today, the Webhook server can be configured to respond GitHub `check_run`, `pull_request`, and `push` events +by scaling up the matching `HorizontalRunnerAutoscaler` by N replica(s), where `N` is configurable within +`HorizontalRunerAutoscaler`'s `Spec`. + +More concretely, you can configure the targeted GitHub event types and the `N` in +`scaleUpTriggers`: + +```yaml +kind: HorizontalRunnerAutoscaler +spec: + scaleTargetRef: + name: myrunners + scaleUpTrigggers: + - githubEvent: + checkRun: + types: ["created"] + status: "queued" + amount: 1 + duration: "5m" +``` + +With the above example, the webhook server scales `myrunners` by `1` replica for 5 minutes on each `check_run` event +with the type of `created` and the status of `queued` received. + +The primary benefit of autoscaling on Webhook compared to the standard autoscaling is that this one allows you to +immediately add "resource slack" for future GitHub Actions job runs. + +In contrast, the standard autoscaling requires you to wait next sync period to add +insufficient runners. You can definitely shorten the sync period to make the standard autoscaling more responsive. +But doing so eventually result in the controller not functional due to GitHub API rate limit. + +> You can learn the implementation details in #282 + +To enable this feature, you firstly need to install the webhook server. + +Currently, only our Helm chart has the ability install it. + +```console +$ helm --upgrade install actions-runner-controller/actions-runner-controller \ + githubWebhookServer.enabled=true \ + githubWebhookServer.ports[0].nodePort=33080 +``` + +The above command will result in exposing the node port 33080 for Webhook events. Usually, you need to create an +external loadbalancer targeted to the node port, and register the hostname or the IP address of the external loadbalancer +to the GitHub Webhook. + +Once you were able to confirm that the Webhook server is ready and running from GitHub - this is usually verified by the +GitHub sending PING events to the Webhook server - create or update your `HorizontalRunnerAutoscaler` resources +by learning the following configuration examples. + +- [Example 1: Scale up on each `check_run` event](#example-1-scale-up-on-each-check_run-event) +- [Example 2: Scale on each `pull_request` event against `develop` or `main` branches](#example-2-scale-on-each-pull_request-event-against-develop-or-main-branches) + +##### Example 1: Scale up on each `check_run` event + +> Note: This should work almost like https://github.com/philips-labs/terraform-aws-github-runner + +To scale up replicas of the runners for `example/myrepo` by 1 for 5 minutes on each `check_run`, you write manifests like the below: + +```yaml +kind: RunnerDeployment +metadata: + name: myrunners +spec: + repository: example/myrepo +--- +kind: HorizontalRunnerAutoscaler +spec: + scaleTargetRef: + name: myrunners + scaleUpTrigggers: + - githubEvent: + checkRun: + types: ["created"] + status: "queued" + amount: 1 + duration: "5m" +``` + +###### Example 2: Scale on each `pull_request` event against `develop` or `main` branches + +```yaml +kind: RunnerDeployment: +metadata: + name: myrunners +spec: + repository: example/myrepo +--- +kind: HorizontalRunnerAutoscaler +spec: + scaleTargetRef: + name: myrunners + scaleUpTrigggers: + - githubEvent: + pullRequest: + types: ["synchronize"] + branches: ["main", "develop"] + amount: 1 + duration: "5m" +``` + +See ["activity types"](https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request) for the list of valid values for `scaleUpTriggers[].githubEvent.pullRequest.types`. + +### Runner with DinD When using default runner, runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). This might create issues if there's `LimitRange` set to namespace. @@ -361,7 +497,7 @@ spec: This also helps with resources, as you don't need to give resources separately to docker and runner. -## Additional tweaks +### Additional tweaks You can pass details through the spec selector. Here's an eg. of what you may like to do: @@ -420,7 +556,7 @@ spec: workDir: /home/runner/work ``` -## Runner labels +### Runner labels To run a workflow job on a self-hosted runner, you can use the following syntax in your workflow: @@ -457,7 +593,7 @@ jobs: Note that if you specify `self-hosted` in your workflow, then this will run your job on _any_ self-hosted runner, regardless of the labels that they have. -## Runner Groups +### Runner Groups Runner groups can be used to limit which repositories are able to use the GitHub Runner at an Organisation level. Runner groups have to be [created in GitHub first](https://docs.github.com/en/actions/hosting-your-own-runners/managing-access-to-self-hosted-runners-using-groups) before they can be referenced. @@ -476,7 +612,7 @@ spec: group: NewGroup ``` -## Using EKS IAM role for service accounts +### Using EKS IAM role for service accounts `actions-runner-controller` v0.15.0 or later has support for EKS IAM role for service accounts. @@ -502,7 +638,7 @@ spec: fsGroup: 1447 ``` -## Software installed in the runner image +### Software installed in the runner image The GitHub hosted runners include a large amount of pre-installed software packages. For Ubuntu 18.04, this list can be found at @@ -537,9 +673,9 @@ spec: image: YOUR_CUSTOM_DOCKER_IMAGE ``` -## Common Errors +### Common Errors -### invalid header field value +#### invalid header field value ```json 2020-11-12T22:17:30.693Z ERROR controller-runtime.controller Reconciler error {"controller": "runner", "request": "actions-runner-system/runner-deployment-dk7q8-dk5c9", "error": "failed to create registration token: Post \"https://api.github.com/orgs/$YOUR_ORG_HERE/actions/runners/registration-token\": net/http: invalid header field value \"Bearer $YOUR_TOKEN_HERE\\n\" for key Authorization"} diff --git a/api/v1alpha1/horizontalrunnerautoscaler_types.go b/api/v1alpha1/horizontalrunnerautoscaler_types.go index 0f920f4e..a19eacd7 100644 --- a/api/v1alpha1/horizontalrunnerautoscaler_types.go +++ b/api/v1alpha1/horizontalrunnerautoscaler_types.go @@ -41,6 +41,56 @@ type HorizontalRunnerAutoscalerSpec struct { // Metrics is the collection of various metric targets to calculate desired number of runners // +optional Metrics []MetricSpec `json:"metrics,omitempty"` + + // ScaleUpTriggers is an experimental feature to increase the desired replicas by 1 + // on each webhook requested received by the webhookBasedAutoscaler. + // + // This feature requires you to also enable and deploy the webhookBasedAutoscaler onto your cluster. + // + // Note that the added runners remain until the next sync period at least, + // and they may or may not be used by GitHub Actions depending on the timing. + // They are intended to be used to gain "resource slack" immediately after you + // receive a webhook from GitHub, so that you can loosely expect MinReplicas runners to be always available. + ScaleUpTriggers []ScaleUpTrigger `json:"scaleUpTriggers,omitempty"` + + CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty" patchStrategy:"merge" patchMergeKey:"name"` +} + +type ScaleUpTrigger struct { + GitHubEvent *GitHubEventScaleUpTriggerSpec `json:"githubEvent,omitempty"` + Amount int `json:"amount,omitempty"` + Duration metav1.Duration `json:"duration,omitempty"` +} + +type GitHubEventScaleUpTriggerSpec struct { + CheckRun *CheckRunSpec `json:"checkRun,omitempty"` + PullRequest *PullRequestSpec `json:"pullRequest,omitempty"` + Push *PushSpec `json:"push,omitempty"` +} + +// https://docs.github.com/en/actions/reference/events-that-trigger-workflows#check_run +type CheckRunSpec struct { + Types []string `json:"types,omitempty"` + Status string `json:"status,omitempty"` +} + +// https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request +type PullRequestSpec struct { + Types []string `json:"types,omitempty"` + Branches []string `json:"branches,omitempty"` +} + +// PushSpec is the condition for triggering scale-up on push event +// Also see https://docs.github.com/en/actions/reference/events-that-trigger-workflows#push +type PushSpec struct { +} + +// CapacityReservation specifies the number of replicas temporarily added +// to the scale target until ExpirationTime. +type CapacityReservation struct { + Name string `json:"name,omitempty"` + ExpirationTime metav1.Time `json:"expirationTime,omitempty"` + Replicas int `json:"replicas,omitempty"` } type ScaleTargetRef struct { @@ -91,6 +141,17 @@ type HorizontalRunnerAutoscalerStatus struct { // +optional LastSuccessfulScaleOutTime *metav1.Time `json:"lastSuccessfulScaleOutTime,omitempty"` + + // +optional + CacheEntries []CacheEntry `json:"cacheEntries,omitempty"` +} + +const CacheEntryKeyDesiredReplicas = "desiredReplicas" + +type CacheEntry struct { + Key string `json:"key,omitempty"` + Value int `json:"value,omitempty"` + ExpirationTime metav1.Time `json:"expirationTime,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 937b43b6..70ce7606 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,88 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CacheEntry) DeepCopyInto(out *CacheEntry) { + *out = *in + in.ExpirationTime.DeepCopyInto(&out.ExpirationTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheEntry. +func (in *CacheEntry) DeepCopy() *CacheEntry { + if in == nil { + return nil + } + out := new(CacheEntry) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CapacityReservation) DeepCopyInto(out *CapacityReservation) { + *out = *in + in.ExpirationTime.DeepCopyInto(&out.ExpirationTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapacityReservation. +func (in *CapacityReservation) DeepCopy() *CapacityReservation { + if in == nil { + return nil + } + out := new(CapacityReservation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CheckRunSpec) DeepCopyInto(out *CheckRunSpec) { + *out = *in + if in.Types != nil { + in, out := &in.Types, &out.Types + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CheckRunSpec. +func (in *CheckRunSpec) DeepCopy() *CheckRunSpec { + if in == nil { + return nil + } + out := new(CheckRunSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitHubEventScaleUpTriggerSpec) DeepCopyInto(out *GitHubEventScaleUpTriggerSpec) { + *out = *in + if in.CheckRun != nil { + in, out := &in.CheckRun, &out.CheckRun + *out = new(CheckRunSpec) + (*in).DeepCopyInto(*out) + } + if in.PullRequest != nil { + in, out := &in.PullRequest, &out.PullRequest + *out = new(PullRequestSpec) + (*in).DeepCopyInto(*out) + } + if in.Push != nil { + in, out := &in.Push, &out.Push + *out = new(PushSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitHubEventScaleUpTriggerSpec. +func (in *GitHubEventScaleUpTriggerSpec) DeepCopy() *GitHubEventScaleUpTriggerSpec { + if in == nil { + return nil + } + out := new(GitHubEventScaleUpTriggerSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HorizontalRunnerAutoscaler) DeepCopyInto(out *HorizontalRunnerAutoscaler) { *out = *in @@ -110,6 +192,20 @@ func (in *HorizontalRunnerAutoscalerSpec) DeepCopyInto(out *HorizontalRunnerAuto (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ScaleUpTriggers != nil { + in, out := &in.ScaleUpTriggers, &out.ScaleUpTriggers + *out = make([]ScaleUpTrigger, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.CapacityReservations != nil { + in, out := &in.CapacityReservations, &out.CapacityReservations + *out = make([]CapacityReservation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerSpec. @@ -134,6 +230,13 @@ func (in *HorizontalRunnerAutoscalerStatus) DeepCopyInto(out *HorizontalRunnerAu in, out := &in.LastSuccessfulScaleOutTime, &out.LastSuccessfulScaleOutTime *out = (*in).DeepCopy() } + if in.CacheEntries != nil { + in, out := &in.CacheEntries, &out.CacheEntries + *out = make([]CacheEntry, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerStatus. @@ -166,6 +269,46 @@ func (in *MetricSpec) DeepCopy() *MetricSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PullRequestSpec) DeepCopyInto(out *PullRequestSpec) { + *out = *in + if in.Types != nil { + in, out := &in.Types, &out.Types + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Branches != nil { + in, out := &in.Branches, &out.Branches + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PullRequestSpec. +func (in *PullRequestSpec) DeepCopy() *PullRequestSpec { + if in == nil { + return nil + } + out := new(PullRequestSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PushSpec) DeepCopyInto(out *PushSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushSpec. +func (in *PushSpec) DeepCopy() *PushSpec { + if in == nil { + return nil + } + out := new(PushSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Runner) DeepCopyInto(out *Runner) { *out = *in @@ -615,3 +758,24 @@ func (in *ScaleTargetRef) DeepCopy() *ScaleTargetRef { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScaleUpTrigger) DeepCopyInto(out *ScaleUpTrigger) { + *out = *in + if in.GitHubEvent != nil { + in, out := &in.GitHubEvent, &out.GitHubEvent + *out = new(GitHubEventScaleUpTriggerSpec) + (*in).DeepCopyInto(*out) + } + out.Duration = in.Duration +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleUpTrigger. +func (in *ScaleUpTrigger) DeepCopy() *ScaleUpTrigger { + if in == nil { + return nil + } + out := new(ScaleUpTrigger) + in.DeepCopyInto(out) + return out +} diff --git a/charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml b/charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml index b7a9f13c..d3669c98 100644 --- a/charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml +++ b/charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml @@ -48,6 +48,20 @@ spec: description: HorizontalRunnerAutoscalerSpec defines the desired state of HorizontalRunnerAutoscaler properties: + capacityReservations: + items: + description: CapacityReservation specifies the number of replicas + temporarily added to the scale target until ExpirationTime. + properties: + expirationTime: + format: date-time + type: string + name: + type: string + replicas: + type: integer + type: object + type: array maxReplicas: description: MinReplicas is the maximum number of replicas the deployment is allowed to scale @@ -104,9 +118,68 @@ spec: name: type: string type: object + scaleUpTriggers: + description: "ScaleUpTriggers is an experimental feature to increase + the desired replicas by 1 on each webhook requested received by the + webhookBasedAutoscaler. \n This feature requires you to also enable + and deploy the webhookBasedAutoscaler onto your cluster. \n Note that + the added runners remain until the next sync period at least, and + they may or may not be used by GitHub Actions depending on the timing. + They are intended to be used to gain \"resource slack\" immediately + after you receive a webhook from GitHub, so that you can loosely expect + MinReplicas runners to be always available." + items: + properties: + amount: + type: integer + duration: + type: string + githubEvent: + properties: + checkRun: + description: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#check_run + properties: + status: + type: string + types: + items: + type: string + type: array + type: object + pullRequest: + description: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request + properties: + branches: + items: + type: string + type: array + types: + items: + type: string + type: array + type: object + push: + description: PushSpec is the condition for triggering scale-up + on push event Also see https://docs.github.com/en/actions/reference/events-that-trigger-workflows#push + type: object + type: object + type: object + type: array type: object status: properties: + cacheEntries: + items: + properties: + expirationTime: + format: date-time + type: string + key: + type: string + value: + type: integer + type: object + type: array desiredReplicas: description: DesiredReplicas is the total number of desired, non-terminated and latest pods to be set for the primary RunnerSet This doesn't include diff --git a/charts/actions-runner-controller/templates/_github_webhook_server_helpers.tpl b/charts/actions-runner-controller/templates/_github_webhook_server_helpers.tpl new file mode 100644 index 00000000..a022bf3e --- /dev/null +++ b/charts/actions-runner-controller/templates/_github_webhook_server_helpers.tpl @@ -0,0 +1,52 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "actions-runner-controller-github-webhook-server.name" -}} +{{- default .Chart.Name .Values.githubWebhookServer.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "actions-runner-controller-github-webhook-server.instance" -}} +{{- printf "%s-%s" .Release.Name "github-webhook-server" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "actions-runner-controller-github-webhook-server.fullname" -}} +{{- if .Values.githubWebhookServer.fullnameOverride }} +{{- .Values.githubWebhookServer.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.githubWebhookServer.nameOverride }} +{{- $instance := include "actions-runner-controller-github-webhook-server.instance" . }} +{{- if contains $name $instance }} +{{- $instance | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s-%s" .Release.Name $name "github-webhook-server" | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "actions-runner-controller-github-webhook-server.selectorLabels" -}} +app.kubernetes.io/name: {{ include "actions-runner-controller-github-webhook-server.name" . }} +app.kubernetes.io/instance: {{ include "actions-runner-controller-github-webhook-server.instance" . }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "actions-runner-controller-github-webhook-server.serviceAccountName" -}} +{{- if .Values.githubWebhookServer.serviceAccount.create }} +{{- default (include "actions-runner-controller-github-webhook-server.fullname" .) .Values.githubWebhookServer.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.githubWebhookServer.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "actions-runner-controller-github-webhook-server.roleName" -}} +{{- include "actions-runner-controller-github-webhook-server.fullname" . }} +{{- end }} diff --git a/charts/actions-runner-controller/templates/githubwebhook.deployment.yaml b/charts/actions-runner-controller/templates/githubwebhook.deployment.yaml new file mode 100644 index 00000000..886a07b6 --- /dev/null +++ b/charts/actions-runner-controller/templates/githubwebhook.deployment.yaml @@ -0,0 +1,93 @@ +{{- if .Values.githubWebhookServer.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "actions-runner-controller-github-webhook-server.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "actions-runner-controller.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "actions-runner-controller-github-webhook-server.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.githubWebhookServer.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "actions-runner-controller-github-webhook-server.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.githubWebhookServer.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "actions-runner-controller-github-webhook-server.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.githubWebhookServer.podSecurityContext | nindent 8 }} + {{- with .Values.githubWebhookServer.priorityClassName }} + priorityClassName: "{{ . }}" + {{- end }} + containers: + - args: + - "--metrics-addr=127.0.0.1:8080" + - "--enable-leader-election" + - "--sync-period={{ .Values.githubWebhookServer.syncPeriod }}" + command: + - "/github-webhook-server" + env: + - name: GITHUB_WEBHOOK_SECRET_TOKEN + valueFrom: + secretKeyRef: + key: github_webhook_secret_token + name: github-webhook-server + optional: true + {{- range $key, $val := .Values.githubWebhookServer.env }} + - name: {{ $key }} + value: {{ $val | quote }} + {{- end }} + image: "{{ .Values.githubWebhookServer.image.repository }}:{{ .Values.githubWebhookServer.image.tag | default (cat "v" .Chart.AppVersion | replace " " "") }}" + name: github-webhook-server + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 8000 + name: github-webhook-server + protocol: TCP + resources: + {{- toYaml .Values.githubWebhookServer.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.githubWebhookServer.securityContext | nindent 12 }} + - args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=10" + image: "{{ .Values.kube_rbac_proxy.image.repository }}:{{ .Values.kube_rbac_proxy.image.tag }}" + name: kube-rbac-proxy + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 8443 + name: https + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + terminationGracePeriodSeconds: 10 + volumes: + - name: github-webhook-server + secret: + secretName: github-webhook-server + {{- with .Values.githubWebhookServer.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.githubWebhookServer.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.githubWebhookServer.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/actions-runner-controller/templates/githubwebhook.role.yaml b/charts/actions-runner-controller/templates/githubwebhook.role.yaml new file mode 100644 index 00000000..1c0d1523 --- /dev/null +++ b/charts/actions-runner-controller/templates/githubwebhook.role.yaml @@ -0,0 +1,70 @@ +{{- if .Values.githubWebhookServer.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: {{ include "actions-runner-controller-github-webhook-server.roleName" . }} +rules: +- apiGroups: + - actions.summerwind.dev + resources: + - horizontalrunnerautoscalers + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - actions.summerwind.dev + resources: + - horizontalrunnerautoscalers/finalizers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - actions.summerwind.dev + resources: + - horizontalrunnerautoscalers/status + verbs: + - get + - patch + - update +- apiGroups: + - actions.summerwind.dev + resources: + - runnerdeployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - actions.summerwind.dev + resources: + - runnerdeployments/finalizers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - actions.summerwind.dev + resources: + - runnerdeployments/status + verbs: + - get + - patch + - update +{{- end }} diff --git a/charts/actions-runner-controller/templates/githubwebhook.secrets.yaml b/charts/actions-runner-controller/templates/githubwebhook.secrets.yaml new file mode 100644 index 00000000..8a415abd --- /dev/null +++ b/charts/actions-runner-controller/templates/githubwebhook.secrets.yaml @@ -0,0 +1,16 @@ +{{- if .Values.githubWebhookServer.enabled }} +{{- if .Values.githubWebhookServer.secret.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: github-webhook-server + namespace: {{ .Release.Namespace }} + labels: + {{- include "actions-runner-controller.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $k, $v := .Values.githubWebhookServer.secret }} + {{ $k }}: {{ $v | toString | b64enc }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/actions-runner-controller/templates/githubwebhook.service.yaml b/charts/actions-runner-controller/templates/githubwebhook.service.yaml new file mode 100644 index 00000000..63bb2af1 --- /dev/null +++ b/charts/actions-runner-controller/templates/githubwebhook.service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.githubWebhookServer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "actions-runner-controller-github-webhook-server.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "actions-runner-controller.labels" . | nindent 4 }} +spec: + type: {{ .Values.githubWebhookServer.service.type }} + ports: + {{ range $_, $port := .Values.githubWebhookServer.service.ports -}} + - {{ $port | toYaml | nindent 6 }} + {{- end }} + selector: + {{- include "actions-runner-controller-github-webhook-server.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/actions-runner-controller/templates/githubwebhook.serviceaccount.yaml b/charts/actions-runner-controller/templates/githubwebhook.serviceaccount.yaml new file mode 100644 index 00000000..e7db91a2 --- /dev/null +++ b/charts/actions-runner-controller/templates/githubwebhook.serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if .Values.githubWebhookServer.enabled -}} +{{- if .Values.githubWebhookServer.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "actions-runner-controller-github-webhook-server.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "actions-runner-controller.labels" . | nindent 4 }} + {{- with .Values.githubWebhookServer.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/actions-runner-controller/values.yaml b/charts/actions-runner-controller/values.yaml index 8699851c..394027a4 100644 --- a/charts/actions-runner-controller/values.yaml +++ b/charts/actions-runner-controller/values.yaml @@ -107,4 +107,47 @@ priorityClassName: "" env: {} # http_proxy: "proxy.com:8080" # https_proxy: "proxy.com:8080" - # no_proxy: "" \ No newline at end of file + # no_proxy: "" + +githubWebhookServer: + enabled: false + labels: {} + replicaCount: 1 + syncPeriod: 10m + secret: + enabled: false + ### GitHub Webhook Configuration + #github_webhook_secret_token: "" + image: + repository: summerwind/actions-runner-controller + # Overrides the manager image tag whose default is the chart appVersion if the tag key is commented out + tag: "latest" + pullPolicy: IfNotPresent + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + podAnnotations: {} + podSecurityContext: {} + # fsGroup: 2000 + securityContext: {} + resources: {} + nodeSelector: {} + tolerations: [] + affinity: {} + priorityClassName: "" + service: + type: NodePort + ports: + - port: 80 + targetPort: 8000 + protocol: TCP + name: http + #nodePort: someFixedPortForUseWithTerraformCdkCfnEtc diff --git a/cmd/githubwebhookserver/main.go b/cmd/githubwebhookserver/main.go new file mode 100644 index 00000000..9a026c31 --- /dev/null +++ b/cmd/githubwebhookserver/main.go @@ -0,0 +1,169 @@ +/* +Copyright 2021 The actions-runner-controller authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "errors" + "flag" + "net/http" + "os" + "sync" + "time" + + actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1" + "github.com/summerwind/actions-runner-controller/controllers" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth/exec" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + // +kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + _ = clientgoscheme.AddToScheme(scheme) + + _ = actionsv1alpha1.AddToScheme(scheme) + // +kubebuilder:scaffold:scheme +} + +func main() { + var ( + err error + + webhookAddr string + metricsAddr string + + // The secret token of the GitHub Webhook. See https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks + webhookSecretToken string + + watchNamespace string + + enableLeaderElection bool + syncPeriod time.Duration + ) + + webhookSecretToken = os.Getenv("GITHUB_WEBHOOK_SECRET_TOKEN") + + flag.StringVar(&webhookAddr, "webhook-addr", ":8000", "The address the metric endpoint binds to.") + flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&watchNamespace, "watch-namespace", "", "The namespace to watch for HorizontalRunnerAutoscaler's to scale on Webhook. Set to empty for letting it watch for all namespaces.") + flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, + "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change") + flag.Parse() + + if webhookSecretToken == "" { + setupLog.Info("-webhook-secret-token is missing or empty. Create one following https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks") + } + + if watchNamespace == "" { + setupLog.Info("-watch-namespace is empty. HorizontalRunnerAutoscalers in all the namespaces are watched, cached, and considered as scale targets.") + } else { + setupLog.Info("-watch-namespace is %q. Only HorizontalRunnerAutoscalers in %q are watched, cached, and considered as scale targets.") + } + + logger := zap.New(func(o *zap.Options) { + o.Development = true + }) + + ctrl.SetLogger(logger) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + SyncPeriod: &syncPeriod, + LeaderElection: enableLeaderElection, + Namespace: watchNamespace, + MetricsBindAddress: metricsAddr, + Port: 9443, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + hraGitHubWebhook := &controllers.HorizontalRunnerAutoscalerGitHubWebhook{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Runner"), + Recorder: nil, + Scheme: mgr.GetScheme(), + SecretKeyBytes: []byte(webhookSecretToken), + WatchNamespace: watchNamespace, + } + + if err = hraGitHubWebhook.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Runner") + os.Exit(1) + } + + var wg sync.WaitGroup + + ctx, cancel := context.WithCancel(context.Background()) + + wg.Add(1) + go func() { + defer cancel() + defer wg.Done() + + setupLog.Info("starting webhook server") + if err := mgr.Start(ctx.Done()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } + }() + + mux := http.NewServeMux() + mux.HandleFunc("/", hraGitHubWebhook.Handle) + + srv := http.Server{ + Addr: webhookAddr, + Handler: mux, + } + + wg.Add(1) + go func() { + defer cancel() + defer wg.Done() + + go func() { + <-ctx.Done() + + srv.Shutdown(context.Background()) + }() + + if err := srv.ListenAndServe(); err != nil { + if !errors.Is(err, http.ErrServerClosed) { + setupLog.Error(err, "problem running http server") + } + } + }() + + go func() { + <-ctrl.SetupSignalHandler() + cancel() + }() + + wg.Wait() +} diff --git a/config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml b/config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml index b7a9f13c..d3669c98 100644 --- a/config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml +++ b/config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml @@ -48,6 +48,20 @@ spec: description: HorizontalRunnerAutoscalerSpec defines the desired state of HorizontalRunnerAutoscaler properties: + capacityReservations: + items: + description: CapacityReservation specifies the number of replicas + temporarily added to the scale target until ExpirationTime. + properties: + expirationTime: + format: date-time + type: string + name: + type: string + replicas: + type: integer + type: object + type: array maxReplicas: description: MinReplicas is the maximum number of replicas the deployment is allowed to scale @@ -104,9 +118,68 @@ spec: name: type: string type: object + scaleUpTriggers: + description: "ScaleUpTriggers is an experimental feature to increase + the desired replicas by 1 on each webhook requested received by the + webhookBasedAutoscaler. \n This feature requires you to also enable + and deploy the webhookBasedAutoscaler onto your cluster. \n Note that + the added runners remain until the next sync period at least, and + they may or may not be used by GitHub Actions depending on the timing. + They are intended to be used to gain \"resource slack\" immediately + after you receive a webhook from GitHub, so that you can loosely expect + MinReplicas runners to be always available." + items: + properties: + amount: + type: integer + duration: + type: string + githubEvent: + properties: + checkRun: + description: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#check_run + properties: + status: + type: string + types: + items: + type: string + type: array + type: object + pullRequest: + description: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request + properties: + branches: + items: + type: string + type: array + types: + items: + type: string + type: array + type: object + push: + description: PushSpec is the condition for triggering scale-up + on push event Also see https://docs.github.com/en/actions/reference/events-that-trigger-workflows#push + type: object + type: object + type: object + type: array type: object status: properties: + cacheEntries: + items: + properties: + expirationTime: + format: date-time + type: string + key: + type: string + value: + type: integer + type: object + type: array desiredReplicas: description: DesiredReplicas is the total number of desired, non-terminated and latest pods to be set for the primary RunnerSet This doesn't include diff --git a/controllers/autoscaling.go b/controllers/autoscaling.go index 958b5107..526ada57 100644 --- a/controllers/autoscaling.go +++ b/controllers/autoscaling.go @@ -7,6 +7,7 @@ import ( "math" "strconv" "strings" + "time" "github.com/summerwind/actions-runner-controller/api/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -19,6 +20,47 @@ const ( defaultScaleDownFactor = 0.7 ) +func getValueAvailableAt(now time.Time, from, to *time.Time, reservedValue int) *int { + if to != nil && now.After(*to) { + return nil + } + + if from != nil && now.Before(*from) { + return nil + } + + return &reservedValue +} + +func (r *HorizontalRunnerAutoscalerReconciler) getDesiredReplicasFromCache(hra v1alpha1.HorizontalRunnerAutoscaler) *int { + var entry *v1alpha1.CacheEntry + + for i := range hra.Status.CacheEntries { + ent := hra.Status.CacheEntries[i] + + if ent.Key != v1alpha1.CacheEntryKeyDesiredReplicas { + continue + } + + if !time.Now().Before(ent.ExpirationTime.Time) { + continue + } + + entry = &ent + + break + } + + if entry != nil { + v := getValueAvailableAt(time.Now(), nil, &entry.ExpirationTime.Time, entry.Value) + if v != nil { + return v + } + } + + return nil +} + func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) { if hra.Spec.MinReplicas == nil { return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing minReplicas", hra.Namespace, hra.Name) diff --git a/controllers/autoscaling_test.go b/controllers/autoscaling_test.go index edaa1a7c..d571feda 100644 --- a/controllers/autoscaling_test.go +++ b/controllers/autoscaling_test.go @@ -157,7 +157,11 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) { _ = v1alpha1.AddToScheme(scheme) t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) { - server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns), fake.WithListWorkflowJobsResponse(200, tc.workflowJobs)) + server := fake.NewServer( + fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns), + fake.WithListWorkflowJobsResponse(200, tc.workflowJobs), + fake.WithListRunnersResponse(200, fake.RunnersListBody), + ) defer server.Close() client := newGithubClient(server) @@ -368,7 +372,11 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) { _ = v1alpha1.AddToScheme(scheme) t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) { - server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns), fake.WithListWorkflowJobsResponse(200, tc.workflowJobs)) + server := fake.NewServer( + fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns), + fake.WithListWorkflowJobsResponse(200, tc.workflowJobs), + fake.WithListRunnersResponse(200, fake.RunnersListBody), + ) defer server.Close() client := newGithubClient(server) diff --git a/controllers/horizontal_runner_autoscaler_webhook.go b/controllers/horizontal_runner_autoscaler_webhook.go new file mode 100644 index 00000000..bbcfc8e6 --- /dev/null +++ b/controllers/horizontal_runner_autoscaler_webhook.go @@ -0,0 +1,375 @@ +/* +Copyright 2020 The actions-runner-controller authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "io/ioutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "net/http" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "time" + + "github.com/go-logr/logr" + gogithub "github.com/google/go-github/v33/github" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/summerwind/actions-runner-controller/api/v1alpha1" +) + +const ( + scaleTargetKey = "scaleTarget" +) + +// HorizontalRunnerAutoscalerGitHubWebhook autoscales a HorizontalRunnerAutoscaler and the RunnerDeployment on each +// GitHub Webhook received +type HorizontalRunnerAutoscalerGitHubWebhook struct { + client.Client + Log logr.Logger + Recorder record.EventRecorder + Scheme *runtime.Scheme + + // SecretKeyBytes is the byte representation of the Webhook secret token + // the administrator is generated and specified in GitHub Web UI. + SecretKeyBytes []byte + + // WatchNamespace is the namespace to watch for HorizontalRunnerAutoscaler's to be + // scaled on Webhook. + // Set to empty for letting it watch for all namespaces. + WatchNamespace string +} + +func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Reconcile(request reconcile.Request) (reconcile.Result, error) { + return ctrl.Result{}, nil +} + +// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/finalizers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch + +func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Handle(w http.ResponseWriter, r *http.Request) { + var ( + ok bool + + err error + ) + + defer func() { + if !ok { + w.WriteHeader(http.StatusInternalServerError) + + if err != nil { + msg := err.Error() + if written, err := w.Write([]byte(msg)); err != nil { + autoscaler.Log.Error(err, "failed writing http error response", "msg", msg, "written", written) + } + } + } + }() + + defer func() { + if r.Body != nil { + r.Body.Close() + } + }() + + var payload []byte + + if len(autoscaler.SecretKeyBytes) > 0 { + payload, err = gogithub.ValidatePayload(r, autoscaler.SecretKeyBytes) + if err != nil { + autoscaler.Log.Error(err, "error validating request body") + + return + } + } else { + payload, err = ioutil.ReadAll(r.Body) + if err != nil { + autoscaler.Log.Error(err, "error reading request body") + + return + } + } + + webhookType := gogithub.WebHookType(r) + event, err := gogithub.ParseWebHook(webhookType, payload) + if err != nil { + var s string + if payload != nil { + s = string(payload) + } + + autoscaler.Log.Error(err, "could not parse webhook", "webhookType", webhookType, "payload", s) + + return + } + + var target *ScaleTarget + + autoscaler.Log.Info("processing webhook event", "eventType", webhookType) + + switch e := event.(type) { + case *gogithub.PushEvent: + target, err = autoscaler.getScaleUpTarget( + context.TODO(), + *e.Repo.Name, + *e.Repo.Organization, + autoscaler.MatchPushEvent(e), + ) + case *gogithub.PullRequestEvent: + target, err = autoscaler.getScaleUpTarget( + context.TODO(), + *e.Repo.Name, + *e.Repo.Organization.Name, + autoscaler.MatchPullRequestEvent(e), + ) + case *gogithub.CheckRunEvent: + target, err = autoscaler.getScaleUpTarget( + context.TODO(), + *e.Repo.Name, + *e.Org.Name, + autoscaler.MatchCheckRunEvent(e), + ) + case *gogithub.PingEvent: + ok = true + + w.WriteHeader(http.StatusOK) + + msg := "pong" + + if written, err := w.Write([]byte(msg)); err != nil { + autoscaler.Log.Error(err, "failed writing http response", "msg", msg, "written", written) + } + + autoscaler.Log.Info("received ping event") + + return + default: + autoscaler.Log.Info("unknown event type", "eventType", webhookType) + + return + } + + if err != nil { + autoscaler.Log.Error(err, "handling check_run event") + + return + } + + if target == nil { + msg := "no horizontalrunnerautoscaler to scale for this github event" + + autoscaler.Log.Info(msg, "eventType", webhookType) + + ok = true + + w.WriteHeader(http.StatusOK) + + if written, err := w.Write([]byte(msg)); err != nil { + autoscaler.Log.Error(err, "failed writing http response", "msg", msg, "written", written) + } + + return + } + + if err := autoscaler.tryScaleUp(context.TODO(), target); err != nil { + autoscaler.Log.Error(err, "could not scale up") + + return + } + + ok = true + + w.WriteHeader(http.StatusOK) + + msg := fmt.Sprintf("scaled %s by 1", target.Name) + + autoscaler.Log.Info(msg) + + if written, err := w.Write([]byte(msg)); err != nil { + autoscaler.Log.Error(err, "failed writing http response", "msg", msg, "written", written) + } +} + +func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) findHRAsByKey(ctx context.Context, value string) ([]v1alpha1.HorizontalRunnerAutoscaler, error) { + ns := autoscaler.WatchNamespace + + var defaultListOpts []client.ListOption + + if ns != "" { + defaultListOpts = append(defaultListOpts, client.InNamespace(ns)) + } + + var hras []v1alpha1.HorizontalRunnerAutoscaler + + if value != "" { + opts := append([]client.ListOption{}, defaultListOpts...) + opts = append(opts, client.MatchingFields{scaleTargetKey: value}) + + var hraList v1alpha1.HorizontalRunnerAutoscalerList + + if err := autoscaler.List(ctx, &hraList, opts...); err != nil { + return nil, err + } + + for _, d := range hraList.Items { + hras = append(hras, d) + } + } + + return hras, nil +} + +func matchTriggerConditionAgainstEvent(types []string, eventAction *string) bool { + if len(types) == 0 { + return true + } + + if eventAction == nil { + return false + } + + for _, tpe := range types { + if tpe == *eventAction { + return true + } + } + + return false +} + +type ScaleTarget struct { + v1alpha1.HorizontalRunnerAutoscaler + v1alpha1.ScaleUpTrigger +} + +func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) searchScaleTargets(hras []v1alpha1.HorizontalRunnerAutoscaler, f func(v1alpha1.ScaleUpTrigger) bool) []ScaleTarget { + var matched []ScaleTarget + + for _, hra := range hras { + if !hra.ObjectMeta.DeletionTimestamp.IsZero() { + continue + } + + for _, scaleUpTrigger := range hra.Spec.ScaleUpTriggers { + if !f(scaleUpTrigger) { + continue + } + + matched = append(matched, ScaleTarget{ + HorizontalRunnerAutoscaler: hra, + ScaleUpTrigger: scaleUpTrigger, + }) + } + } + + return matched +} + +func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleTarget(ctx context.Context, name string, f func(v1alpha1.ScaleUpTrigger) bool) (*ScaleTarget, error) { + hras, err := autoscaler.findHRAsByKey(ctx, name) + if err != nil { + return nil, err + } + + targets := autoscaler.searchScaleTargets(hras, f) + + if len(targets) != 1 { + return nil, nil + } + + return &targets[0], nil +} + +func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleUpTarget(ctx context.Context, repoNameFromWebhook, orgNameFromWebhook string, f func(v1alpha1.ScaleUpTrigger) bool) (*ScaleTarget, error) { + if target, err := autoscaler.getScaleTarget(ctx, repoNameFromWebhook, f); err != nil { + return nil, err + } else if target != nil { + autoscaler.Log.Info("scale up target is repository-wide runners", "repository", repoNameFromWebhook) + return target, nil + } + + if target, err := autoscaler.getScaleTarget(ctx, orgNameFromWebhook, f); err != nil { + return nil, err + } else if target != nil { + autoscaler.Log.Info("scale up target is organizational runners", "repository", orgNameFromWebhook) + return target, nil + } + + return nil, nil +} + +func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) tryScaleUp(ctx context.Context, target *ScaleTarget) error { + if target == nil { + return nil + } + + log := autoscaler.Log.WithValues("horizontalrunnerautoscaler", target.HorizontalRunnerAutoscaler.Name) + + copy := target.HorizontalRunnerAutoscaler.DeepCopy() + + amount := 1 + + if target.ScaleUpTrigger.Amount > 0 { + amount = target.ScaleUpTrigger.Amount + } + + copy.Spec.CapacityReservations = append(copy.Spec.CapacityReservations, v1alpha1.CapacityReservation{ + ExpirationTime: metav1.Time{Time: time.Now().Add(target.ScaleUpTrigger.Duration.Duration)}, + Replicas: amount, + }) + + if err := autoscaler.Client.Update(ctx, copy); err != nil { + log.Error(err, "Failed to update horizontalrunnerautoscaler resource") + + return err + } + + return nil +} + +func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) SetupWithManager(mgr ctrl.Manager) error { + autoscaler.Recorder = mgr.GetEventRecorderFor("webhookbasedautoscaler") + + if err := mgr.GetFieldIndexer().IndexField(&v1alpha1.HorizontalRunnerAutoscaler{}, scaleTargetKey, func(rawObj runtime.Object) []string { + hra := rawObj.(*v1alpha1.HorizontalRunnerAutoscaler) + + if hra.Spec.ScaleTargetRef.Name == "" { + return nil + } + + var rd v1alpha1.RunnerDeployment + + if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rd); err != nil { + return nil + } + + return []string{rd.Spec.Template.Spec.Repository, rd.Spec.Template.Spec.Organization} + }); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.HorizontalRunnerAutoscaler{}). + Complete(autoscaler) +} diff --git a/controllers/horizontal_runner_autoscaler_webhook_on_check_run.go b/controllers/horizontal_runner_autoscaler_webhook_on_check_run.go new file mode 100644 index 00000000..819ad3e1 --- /dev/null +++ b/controllers/horizontal_runner_autoscaler_webhook_on_check_run.go @@ -0,0 +1,32 @@ +package controllers + +import ( + "github.com/google/go-github/v33/github" + "github.com/summerwind/actions-runner-controller/api/v1alpha1" +) + +func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchCheckRunEvent(event *github.CheckRunEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool { + return func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool { + g := scaleUpTrigger.GitHubEvent + + if g == nil { + return false + } + + cr := g.CheckRun + + if cr == nil { + return false + } + + if !matchTriggerConditionAgainstEvent(cr.Types, event.Action) { + return false + } + + if cr.Status != "" && (event.CheckRun == nil || event.CheckRun.Status == nil || *event.CheckRun.Status != cr.Status) { + return false + } + + return true + } +} diff --git a/controllers/horizontal_runner_autoscaler_webhook_on_pull_request.go b/controllers/horizontal_runner_autoscaler_webhook_on_pull_request.go new file mode 100644 index 00000000..9d383fcf --- /dev/null +++ b/controllers/horizontal_runner_autoscaler_webhook_on_pull_request.go @@ -0,0 +1,32 @@ +package controllers + +import ( + "github.com/google/go-github/v33/github" + "github.com/summerwind/actions-runner-controller/api/v1alpha1" +) + +func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPullRequestEvent(event *github.PullRequestEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool { + return func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool { + g := scaleUpTrigger.GitHubEvent + + if g == nil { + return false + } + + pr := g.PullRequest + + if pr == nil { + return false + } + + if !matchTriggerConditionAgainstEvent(pr.Types, event.Action) { + return false + } + + if !matchTriggerConditionAgainstEvent(pr.Branches, event.PullRequest.Base.Ref) { + return false + } + + return true + } +} diff --git a/controllers/horizontal_runner_autoscaler_webhook_on_push.go b/controllers/horizontal_runner_autoscaler_webhook_on_push.go new file mode 100644 index 00000000..ecf5f718 --- /dev/null +++ b/controllers/horizontal_runner_autoscaler_webhook_on_push.go @@ -0,0 +1,24 @@ +package controllers + +import ( + "github.com/google/go-github/v33/github" + "github.com/summerwind/actions-runner-controller/api/v1alpha1" +) + +func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPushEvent(event *github.PushEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool { + return func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool { + g := scaleUpTrigger.GitHubEvent + + if g == nil { + return false + } + + push := g.Push + + if push == nil { + return false + } + + return true + } +} diff --git a/controllers/horizontal_runner_autoscaler_webhook_test.go b/controllers/horizontal_runner_autoscaler_webhook_test.go new file mode 100644 index 00000000..b6c76ac5 --- /dev/null +++ b/controllers/horizontal_runner_autoscaler_webhook_test.go @@ -0,0 +1,245 @@ +package controllers + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/go-logr/logr" + "github.com/google/go-github/v33/github" + actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1" + "io" + "io/ioutil" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "net/http" + "net/http/httptest" + "net/url" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" +) + +var ( + sc = runtime.NewScheme() +) + +func init() { + _ = clientgoscheme.AddToScheme(sc) + _ = actionsv1alpha1.AddToScheme(sc) +} + +func TestWebhookCheckRun(t *testing.T) { + testServer(t, + "check_run", + &github.CheckRunEvent{ + CheckRun: &github.CheckRun{ + Status: github.String("queued"), + }, + Repo: &github.Repository{ + Name: github.String("myorg/myrepo"), + }, + Org: &github.Organization{ + Name: github.String("myorg"), + }, + Action: github.String("created"), + }, + 200, + "no horizontalrunnerautoscaler to scale for this github event", + ) +} + +func TestWebhookPullRequest(t *testing.T) { + testServer(t, + "pull_request", + &github.PullRequestEvent{ + PullRequest: &github.PullRequest{ + Base: &github.PullRequestBranch{ + Ref: github.String("main"), + }, + }, + Repo: &github.Repository{ + Name: github.String("myorg/myrepo"), + Organization: &github.Organization{ + Name: github.String("myorg"), + }, + }, + Action: github.String("created"), + }, + 200, + "no horizontalrunnerautoscaler to scale for this github event", + ) +} + +func TestWebhookPush(t *testing.T) { + testServer(t, + "push", + &github.PushEvent{ + Repo: &github.PushEventRepository{ + Name: github.String("myrepo"), + Organization: github.String("myorg"), + }, + }, + 200, + "no horizontalrunnerautoscaler to scale for this github event", + ) +} + +func TestWebhookPing(t *testing.T) { + testServer(t, + "ping", + &github.PingEvent{ + Zen: github.String("zen"), + }, + 200, + "pong", + ) +} + +func installTestLogger(webhook *HorizontalRunnerAutoscalerGitHubWebhook) *bytes.Buffer { + logs := &bytes.Buffer{} + + log := testLogger{ + name: "testlog", + writer: logs, + } + + webhook.Log = &log + + return logs +} + +func testServer(t *testing.T, eventType string, event interface{}, wantCode int, wantBody string) { + t.Helper() + + hraWebhook := &HorizontalRunnerAutoscalerGitHubWebhook{} + + var initObjs []runtime.Object + + client := fake.NewFakeClientWithScheme(sc, initObjs...) + + logs := installTestLogger(hraWebhook) + + defer func() { + if t.Failed() { + t.Logf("diagnostics: %s", logs.String()) + } + }() + + hraWebhook.Client = client + + mux := http.NewServeMux() + mux.HandleFunc("/", hraWebhook.Handle) + + server := httptest.NewServer(mux) + defer server.Close() + + resp, err := sendWebhook(server, eventType, event) + if err != nil { + t.Fatal(err) + } + + defer func() { + if resp != nil { + resp.Body.Close() + } + }() + + if resp.StatusCode != wantCode { + t.Error("status:", resp.StatusCode) + } + + respBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + if string(respBody) != wantBody { + t.Fatal("body:", string(respBody)) + } +} + +func sendWebhook(server *httptest.Server, eventType string, event interface{}) (*http.Response, error) { + jsonBuf := &bytes.Buffer{} + enc := json.NewEncoder(jsonBuf) + enc.SetIndent(" ", "") + err := enc.Encode(event) + if err != nil { + return nil, fmt.Errorf("[bug in test] encoding event to json: %+v", err) + } + + reqBody := jsonBuf.Bytes() + + u, err := url.Parse(server.URL) + if err != nil { + return nil, fmt.Errorf("parsing server url: %v", err) + } + + req := &http.Request{ + Method: http.MethodPost, + URL: u, + Header: map[string][]string{ + "X-GitHub-Event": {eventType}, + "Content-Type": {"application/json"}, + }, + Body: ioutil.NopCloser(bytes.NewBuffer(reqBody)), + } + + return http.DefaultClient.Do(req) +} + +// testLogger is a sample logr.Logger that logs in-memory. +// It's only for testing log outputs. +type testLogger struct { + name string + keyValues map[string]interface{} + + writer io.Writer +} + +var _ logr.Logger = &testLogger{} + +func (l *testLogger) Info(msg string, kvs ...interface{}) { + fmt.Fprintf(l.writer, "%s] %s\t", l.name, msg) + for k, v := range l.keyValues { + fmt.Fprintf(l.writer, "%s=%+v ", k, v) + } + for i := 0; i < len(kvs); i += 2 { + fmt.Fprintf(l.writer, "%s=%+v ", kvs[i], kvs[i+1]) + } + fmt.Fprintf(l.writer, "\n") +} + +func (_ *testLogger) Enabled() bool { + return true +} + +func (l *testLogger) Error(err error, msg string, kvs ...interface{}) { + kvs = append(kvs, "error", err) + l.Info(msg, kvs...) +} + +func (l *testLogger) V(_ int) logr.InfoLogger { + return l +} + +func (l *testLogger) WithName(name string) logr.Logger { + return &testLogger{ + name: l.name + "." + name, + keyValues: l.keyValues, + writer: l.writer, + } +} + +func (l *testLogger) WithValues(kvs ...interface{}) logr.Logger { + newMap := make(map[string]interface{}, len(l.keyValues)+len(kvs)/2) + for k, v := range l.keyValues { + newMap[k] = v + } + for i := 0; i < len(kvs); i += 2 { + newMap[kvs[i].(string)] = kvs[i+1] + } + return &testLogger{ + name: l.name, + keyValues: newMap, + writer: l.writer, + } +} diff --git a/controllers/horizontalrunnerautoscaler_controller.go b/controllers/horizontalrunnerautoscaler_controller.go index 5dcbfdc2..594b9607 100644 --- a/controllers/horizontalrunnerautoscaler_controller.go +++ b/controllers/horizontalrunnerautoscaler_controller.go @@ -46,6 +46,8 @@ type HorizontalRunnerAutoscalerReconciler struct { Log logr.Logger Recorder record.EventRecorder Scheme *runtime.Scheme + + CacheDuration time.Duration } // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;update;patch @@ -79,13 +81,23 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl return ctrl.Result{}, nil } - replicas, err := r.computeReplicas(rd, hra) - if err != nil { - r.Recorder.Event(&hra, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error()) + var replicas *int - log.Error(err, "Could not compute replicas") + replicasFromCache := r.getDesiredReplicasFromCache(hra) - return ctrl.Result{}, err + if replicasFromCache != nil { + replicas = replicasFromCache + } else { + var err error + + replicas, err = r.computeReplicas(rd, hra) + if err != nil { + r.Recorder.Event(&hra, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error()) + + log.Error(err, "Could not compute replicas") + + return ctrl.Result{}, err + } } const defaultReplicas = 1 @@ -93,6 +105,18 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl currentDesiredReplicas := getIntOrDefault(rd.Spec.Replicas, defaultReplicas) newDesiredReplicas := getIntOrDefault(replicas, defaultReplicas) + now := time.Now() + + for _, reservation := range hra.Spec.CapacityReservations { + if reservation.ExpirationTime.Time.After(now) { + newDesiredReplicas += reservation.Replicas + } + } + + if hra.Spec.MaxReplicas != nil && *hra.Spec.MaxReplicas < newDesiredReplicas { + newDesiredReplicas = *hra.Spec.MaxReplicas + } + // Please add more conditions that we can in-place update the newest runnerreplicaset without disruption if currentDesiredReplicas != newDesiredReplicas { copy := rd.DeepCopy() @@ -103,12 +127,12 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl return ctrl.Result{}, err } - - return ctrl.Result{}, err } + var updated *v1alpha1.HorizontalRunnerAutoscaler + if hra.Status.DesiredReplicas == nil || *hra.Status.DesiredReplicas != *replicas { - updated := hra.DeepCopy() + updated = hra.DeepCopy() if (hra.Status.DesiredReplicas == nil && *replicas > 1) || (hra.Status.DesiredReplicas != nil && *replicas > *hra.Status.DesiredReplicas) { @@ -117,7 +141,37 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl } updated.Status.DesiredReplicas = replicas + } + if replicasFromCache == nil { + if updated == nil { + updated = hra.DeepCopy() + } + + var cacheEntries []v1alpha1.CacheEntry + + for _, ent := range updated.Status.CacheEntries { + if ent.ExpirationTime.Before(&metav1.Time{Time: now}) { + cacheEntries = append(cacheEntries, ent) + } + } + + var cacheDuration time.Duration + + if r.CacheDuration > 0 { + cacheDuration = r.CacheDuration + } else { + cacheDuration = 10 * time.Minute + } + + updated.Status.CacheEntries = append(updated.Status.CacheEntries, v1alpha1.CacheEntry{ + Key: v1alpha1.CacheEntryKeyDesiredReplicas, + Value: *replicas, + ExpirationTime: metav1.Time{Time: time.Now().Add(cacheDuration)}, + }) + } + + if updated != nil { if err := r.Status().Update(ctx, updated); err != nil { log.Error(err, "Failed to update horizontalrunnerautoscaler status") diff --git a/controllers/integration_test.go b/controllers/integration_test.go index 55552c89..1e761a61 100644 --- a/controllers/integration_test.go +++ b/controllers/integration_test.go @@ -2,6 +2,11 @@ package controllers import ( "context" + "github.com/google/go-github/v33/github" + github3 "github.com/google/go-github/v33/github" + github2 "github.com/summerwind/actions-runner-controller/github" + "net/http" + "net/http/httptest" "time" "github.com/summerwind/actions-runner-controller/github/fake" @@ -30,6 +35,12 @@ var ( workflowRunsFor1Replicas = `{"total_count": 6, "workflow_runs":[{"status":"queued"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}]}"` ) +var webhookServer *httptest.Server + +var ghClient *github2.Client + +var fakeRunnerList *fake.RunnersList + // SetupIntegrationTest will set up a testing environment. // This includes: // * creating a Namespace to be used during the test @@ -41,10 +52,13 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment { ns := &corev1.Namespace{} responses := &fake.FixedResponses{} + responses.ListRunners = fake.DefaultListRunnersHandler() responses.ListRepositoryWorkflowRuns = &fake.Handler{ Status: 200, Body: workflowRunsFor3Replicas, } + fakeRunnerList = fake.NewRunnersList() + responses.ListRunners = fakeRunnerList.HandleList() fakeGithubServer := fake.NewServer(fake.WithFixedResponses(responses)) BeforeEach(func() { @@ -59,9 +73,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment { mgr, err := ctrl.NewManager(cfg, ctrl.Options{}) Expect(err).NotTo(HaveOccurred(), "failed to create manager") - runnersList = fake.NewRunnersList() - server = runnersList.GetServer() - ghClient := newGithubClient(server) + ghClient = newGithubClient(fakeGithubServer) replicasetController := &RunnerReplicaSetReconciler{ Client: mgr.GetClient(), @@ -85,15 +97,30 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment { client := newGithubClient(fakeGithubServer) autoscalerController := &HorizontalRunnerAutoscalerReconciler{ - Client: mgr.GetClient(), - Scheme: scheme.Scheme, - Log: logf.Log, - GitHubClient: client, - Recorder: mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"), + Client: mgr.GetClient(), + Scheme: scheme.Scheme, + Log: logf.Log, + GitHubClient: client, + Recorder: mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"), + CacheDuration: 1 * time.Second, } err = autoscalerController.SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred(), "failed to setup controller") + autoscalerWebhook := &HorizontalRunnerAutoscalerGitHubWebhook{ + Client: mgr.GetClient(), + Scheme: scheme.Scheme, + Log: logf.Log, + Recorder: mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"), + } + err = autoscalerWebhook.SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred(), "failed to setup autoscaler webhook") + + mux := http.NewServeMux() + mux.HandleFunc("/", autoscalerWebhook.Handle) + + webhookServer = httptest.NewServer(mux) + go func() { defer GinkgoRecover() @@ -106,6 +133,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment { close(stopCh) fakeGithubServer.Close() + webhookServer.Close() err := k8sClient.Delete(ctx, ns) Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace") @@ -114,7 +142,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment { return &testEnvironment{Namespace: ns, Responses: responses} } -var _ = Context("Inside of a new namespace", func() { +var _ = Context("INTEGRATION: Inside of a new namespace", func() { ctx := context.TODO() env := SetupIntegrationTest(ctx) ns := env.Namespace @@ -235,8 +263,20 @@ var _ = Context("Inside of a new namespace", func() { }, MinReplicas: intPtr(1), MaxReplicas: intPtr(3), - ScaleDownDelaySecondsAfterScaleUp: nil, + ScaleDownDelaySecondsAfterScaleUp: intPtr(1), Metrics: nil, + ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{ + { + GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{ + PullRequest: &actionsv1alpha1.PullRequestSpec{ + Types: []string{"created"}, + Branches: []string{"main"}, + }, + }, + Amount: 1, + Duration: metav1.Duration{Duration: time.Minute}, + }, + }, }, } @@ -274,8 +314,33 @@ var _ = Context("Inside of a new namespace", func() { time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(3)) } + { + var runnerList actionsv1alpha1.RunnerList + + err := k8sClient.List(ctx, &runnerList, client.InNamespace(ns.Name)) + if err != nil { + logf.Log.Error(err, "list runners") + } + + for i, r := range runnerList.Items { + fakeRunnerList.Add(&github3.Runner{ + ID: github.Int64(int64(i)), + Name: github.String(r.Name), + OS: github.String("linux"), + Status: github.String("online"), + Busy: github.Bool(false), + }) + } + + rs, err := ghClient.ListRunners(context.Background(), "", "", "test/valid") + Expect(err).NotTo(HaveOccurred(), "verifying list fake runners response") + Expect(len(rs)).To(Equal(3), "count of fake list runners") + } + // Scale-down to 1 replica { + time.Sleep(time.Second) + responses.ListRepositoryWorkflowRuns.Body = workflowRunsFor1Replicas var hra actionsv1alpha1.HorizontalRunnerAutoscaler @@ -308,7 +373,60 @@ var _ = Context("Inside of a new namespace", func() { return *runnerSets.Items[0].Spec.Replicas }, - time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1)) + time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1), "runners after HRA force update for scale-down") + } + + { + resp, err := sendWebhook(webhookServer, "pull_request", &github.PullRequestEvent{ + PullRequest: &github.PullRequest{ + Base: &github.PullRequestBranch{ + Ref: github.String("main"), + }, + }, + Repo: &github.Repository{ + Name: github.String("test/valid"), + Organization: &github.Organization{ + Name: github.String("test"), + }, + }, + Action: github.String("created"), + }) + + Expect(err).NotTo(HaveOccurred(), "failed to send pull_request event") + + Expect(resp.StatusCode).To(Equal(200)) + } + + // Scale-up to 2 replicas + { + runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}} + + Eventually( + func() int { + err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name)) + if err != nil { + logf.Log.Error(err, "list runner sets") + } + + return len(runnerSets.Items) + }, + time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1), "runner sets after webhook") + + Eventually( + func() int { + err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name)) + if err != nil { + logf.Log.Error(err, "list runner sets") + } + + if len(runnerSets.Items) == 0 { + logf.Log.Info("No runnerreplicasets exist yet") + return -1 + } + + return *runnerSets.Items[0].Spec.Replicas + }, + time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2), "runners after webhook") } }) }) diff --git a/controllers/runnerdeployment_controller.go b/controllers/runnerdeployment_controller.go index 61fe5f28..1b495055 100644 --- a/controllers/runnerdeployment_controller.go +++ b/controllers/runnerdeployment_controller.go @@ -177,7 +177,7 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e rs := oldSets[i] if err := r.Client.Delete(ctx, &rs); err != nil { - log.Error(err, "Failed to delete runner resource") + log.Error(err, "Failed to delete runnerreplicaset resource") return ctrl.Result{}, err } diff --git a/controllers/runnerreplicaset_controller.go b/controllers/runnerreplicaset_controller.go index 32a2d7db..d070103e 100644 --- a/controllers/runnerreplicaset_controller.go +++ b/controllers/runnerreplicaset_controller.go @@ -117,7 +117,7 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e } for i := 0; i < n; i++ { - if err := r.Client.Delete(ctx, ¬Busy[i]); err != nil { + if err := r.Client.Delete(ctx, ¬Busy[i]); client.IgnoreNotFound(err) != nil { log.Error(err, "Failed to delete runner resource") return ctrl.Result{}, err diff --git a/controllers/suite_test.go b/controllers/suite_test.go index ff75227b..e5dd5771 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -17,6 +17,8 @@ limitations under the License. package controllers import ( + "github.com/onsi/ginkgo/config" + "os" "path/filepath" "testing" @@ -43,6 +45,8 @@ var testEnv *envtest.Environment func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) + config.GinkgoConfig.FocusString = os.Getenv("GINKGO_FOCUS") + RunSpecsWithDefaultAndCustomReporters(t, "Controller Suite", []Reporter{envtest.NewlineReporter{}}) diff --git a/github/fake/fake.go b/github/fake/fake.go index 49cb2305..cdea9cb6 100644 --- a/github/fake/fake.go +++ b/github/fake/fake.go @@ -24,6 +24,16 @@ const ( ` ) +type ListRunnersHandler struct { + Status int + Body string +} + +func (h *ListRunnersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(h.Status) + fmt.Fprintf(w, h.Body) +} + type Handler struct { Status int Body string @@ -94,10 +104,7 @@ func NewServer(opts ...Option) *httptest.Server { }, // For ListRunners - "/repos/test/valid/actions/runners": &Handler{ - Status: http.StatusOK, - Body: RunnersListBody, - }, + "/repos/test/valid/actions/runners": config.FixedResponses.ListRunners, "/repos/test/invalid/actions/runners": &Handler{ Status: http.StatusNoContent, Body: "", @@ -159,3 +166,10 @@ func NewServer(opts ...Option) *httptest.Server { return httptest.NewServer(mux) } + +func DefaultListRunnersHandler() *ListRunnersHandler { + return &ListRunnersHandler{ + Status: http.StatusOK, + Body: RunnersListBody, + } +} diff --git a/github/fake/options.go b/github/fake/options.go index 2b78c888..21cf5fac 100644 --- a/github/fake/options.go +++ b/github/fake/options.go @@ -1,8 +1,11 @@ package fake +import "net/http" + type FixedResponses struct { ListRepositoryWorkflowRuns *Handler ListWorkflowJobs *MapHandler + ListRunners http.Handler } type Option func(*ServerConfig) @@ -25,6 +28,15 @@ func WithListWorkflowJobsResponse(status int, bodies map[int]string) Option { } } +func WithListRunnersResponse(status int, body string) Option { + return func(c *ServerConfig) { + c.FixedResponses.ListRunners = &ListRunnersHandler{ + Status: status, + Body: body, + } + } +} + func WithFixedResponses(responses *FixedResponses) Option { return func(c *ServerConfig) { c.FixedResponses = responses diff --git a/github/fake/runners.go b/github/fake/runners.go index fbae51fe..01f2ac1d 100644 --- a/github/fake/runners.go +++ b/github/fake/runners.go @@ -29,15 +29,15 @@ func (r *RunnersList) Add(runner *github.Runner) { func (r *RunnersList) GetServer() *httptest.Server { router := mux.NewRouter() - router.Handle("/repos/{owner}/{repo}/actions/runners", r.handleList()) + router.Handle("/repos/{owner}/{repo}/actions/runners", r.HandleList()) router.Handle("/repos/{owner}/{repo}/actions/runners/{id}", r.handleRemove()) - router.Handle("/orgs/{org}/actions/runners", r.handleList()) + router.Handle("/orgs/{org}/actions/runners", r.HandleList()) router.Handle("/orgs/{org}/actions/runners/{id}", r.handleRemove()) return httptest.NewServer(router) } -func (r *RunnersList) handleList() http.HandlerFunc { +func (r *RunnersList) HandleList() http.HandlerFunc { return func(w http.ResponseWriter, res *http.Request) { j, err := json.Marshal(github.Runners{ TotalCount: len(r.runners), diff --git a/github/github_test.go b/github/github_test.go index f7852ff2..4672f65e 100644 --- a/github/github_test.go +++ b/github/github_test.go @@ -32,7 +32,10 @@ func newTestClient() *Client { } func TestMain(m *testing.M) { - server = fake.NewServer() + res := &fake.FixedResponses{ + ListRunners: fake.DefaultListRunnersHandler(), + } + server = fake.NewServer(fake.WithFixedResponses(res)) defer server.Close() m.Run() } diff --git a/main.go b/main.go index 63fcff1f..7f22e752 100644 --- a/main.go +++ b/main.go @@ -144,10 +144,11 @@ func main() { } horizontalRunnerAutoscaler := &controllers.HorizontalRunnerAutoscalerReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("HorizontalRunnerAutoscaler"), - Scheme: mgr.GetScheme(), - GitHubClient: ghClient, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("HorizontalRunnerAutoscaler"), + Scheme: mgr.GetScheme(), + GitHubClient: ghClient, + CacheDuration: syncPeriod - 10*time.Second, } if err = horizontalRunnerAutoscaler.SetupWithManager(mgr); err != nil {