From ebc3970b8460728dc8f1863e6713800585551bee Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Fri, 19 Feb 2021 10:33:04 +0900 Subject: [PATCH] Add integration test for autoscaling on check_run webhook event --- .../horizontal_runner_autoscaler_webhook.go | 5 + .../horizontalrunnerautoscaler_controller.go | 5 + controllers/integration_test.go | 249 +++++++++++++++--- controllers/runnerdeployment_controller.go | 5 + controllers/runnerreplicaset_controller.go | 5 + 5 files changed, 228 insertions(+), 41 deletions(-) diff --git a/controllers/horizontal_runner_autoscaler_webhook.go b/controllers/horizontal_runner_autoscaler_webhook.go index 89ad7113..6b1920e9 100644 --- a/controllers/horizontal_runner_autoscaler_webhook.go +++ b/controllers/horizontal_runner_autoscaler_webhook.go @@ -56,6 +56,7 @@ type HorizontalRunnerAutoscalerGitHubWebhook struct { // scaled on Webhook. // Set to empty for letting it watch for all namespaces. WatchNamespace string + Name string } func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Reconcile(request reconcile.Request) (reconcile.Result, error) { @@ -350,6 +351,10 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) tryScaleUp(ctx contex func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) SetupWithManager(mgr ctrl.Manager) error { name := "webhookbasedautoscaler" + if autoscaler.Name != "" { + name = autoscaler.Name + } + autoscaler.Recorder = mgr.GetEventRecorderFor(name) if err := mgr.GetFieldIndexer().IndexField(&v1alpha1.HorizontalRunnerAutoscaler{}, scaleTargetKey, func(rawObj runtime.Object) []string { diff --git a/controllers/horizontalrunnerautoscaler_controller.go b/controllers/horizontalrunnerautoscaler_controller.go index 374841f6..55ee672f 100644 --- a/controllers/horizontalrunnerautoscaler_controller.go +++ b/controllers/horizontalrunnerautoscaler_controller.go @@ -48,6 +48,7 @@ type HorizontalRunnerAutoscalerReconciler struct { Scheme *runtime.Scheme CacheDuration time.Duration + Name string } // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;update;patch @@ -184,6 +185,10 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl func (r *HorizontalRunnerAutoscalerReconciler) SetupWithManager(mgr ctrl.Manager) error { name := "horizontalrunnerautoscaler-controller" + if r.Name != "" { + name = r.Name + } + r.Recorder = mgr.GetEventRecorderFor(name) return ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/integration_test.go b/controllers/integration_test.go index 48e59af2..573eb7f0 100644 --- a/controllers/integration_test.go +++ b/controllers/integration_test.go @@ -31,23 +31,22 @@ import ( type testEnvironment struct { Namespace *corev1.Namespace Responses *fake.FixedResponses + + webhookServer *httptest.Server + ghClient *github2.Client + fakeRunnerList *fake.RunnersList + fakeGithubServer *httptest.Server } var ( workflowRunsFor3Replicas = `{"total_count": 5, "workflow_runs":[{"status":"queued"}, {"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"` workflowRunsFor3Replicas_queued = `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"queued"}]}"` - workflowRunsFor3Replicas_in_progress = `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"` + workflowRunsFor3Replicas_in_progress = `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"` workflowRunsFor1Replicas = `{"total_count": 6, "workflow_runs":[{"status":"queued"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}]}"` workflowRunsFor1Replicas_queued = `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"` workflowRunsFor1Replicas_in_progress = `{"total_count": 0, "workflow_runs":[]}"` ) -var webhookServer *httptest.Server - -var ghClient *github2.Client - -var fakeRunnerList *fake.RunnersList - // SetupIntegrationTest will set up a testing environment. // This includes: // * creating a Namespace to be used during the test @@ -58,19 +57,11 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment { var stopCh chan struct{} ns := &corev1.Namespace{} - responses := &fake.FixedResponses{} - responses.ListRunners = fake.DefaultListRunnersHandler() - responses.ListRepositoryWorkflowRuns = &fake.Handler{ - Status: 200, - Body: workflowRunsFor3Replicas, - Statuses: map[string]string{ - "queued": workflowRunsFor3Replicas_queued, - "in_progress": workflowRunsFor3Replicas_in_progress, - }, + env := &testEnvironment{ + Namespace: ns, + webhookServer: nil, + ghClient: nil, } - fakeRunnerList = fake.NewRunnersList() - responses.ListRunners = fakeRunnerList.HandleList() - fakeGithubServer := fake.NewServer(fake.WithFixedResponses(responses)) BeforeEach(func() { stopCh = make(chan struct{}) @@ -84,14 +75,36 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment { mgr, err := ctrl.NewManager(cfg, ctrl.Options{}) Expect(err).NotTo(HaveOccurred(), "failed to create manager") - ghClient = newGithubClient(fakeGithubServer) + responses := &fake.FixedResponses{} + responses.ListRunners = fake.DefaultListRunnersHandler() + responses.ListRepositoryWorkflowRuns = &fake.Handler{ + Status: 200, + Body: workflowRunsFor3Replicas, + Statuses: map[string]string{ + "queued": workflowRunsFor3Replicas_queued, + "in_progress": workflowRunsFor3Replicas_in_progress, + }, + } + fakeRunnerList := fake.NewRunnersList() + responses.ListRunners = fakeRunnerList.HandleList() + fakeGithubServer := fake.NewServer(fake.WithFixedResponses(responses)) + + env.Responses = responses + env.fakeRunnerList = fakeRunnerList + env.fakeGithubServer = fakeGithubServer + env.ghClient = newGithubClient(fakeGithubServer) + + controllerName := func(name string) string { + return fmt.Sprintf("%s%s", ns.Name, name) + } replicasetController := &RunnerReplicaSetReconciler{ Client: mgr.GetClient(), Scheme: scheme.Scheme, Log: logf.Log, Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"), - GitHubClient: ghClient, + GitHubClient: env.ghClient, + Name: controllerName("runnerreplicaset"), } err = replicasetController.SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred(), "failed to setup controller") @@ -101,19 +114,19 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment { Scheme: scheme.Scheme, Log: logf.Log, Recorder: mgr.GetEventRecorderFor("runnerdeployment-controller"), + Name: controllerName("runnnerdeployment"), } err = deploymentsController.SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred(), "failed to setup controller") - client := newGithubClient(fakeGithubServer) - autoscalerController := &HorizontalRunnerAutoscalerReconciler{ Client: mgr.GetClient(), Scheme: scheme.Scheme, Log: logf.Log, - GitHubClient: client, + GitHubClient: env.ghClient, Recorder: mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"), CacheDuration: 1 * time.Second, + Name: controllerName("horizontalrunnerautoscaler"), } err = autoscalerController.SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred(), "failed to setup controller") @@ -123,6 +136,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment { Scheme: scheme.Scheme, Log: logf.Log, Recorder: mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"), + Name: controllerName("horizontalrunnerautoscalergithubwebhook"), } err = autoscalerWebhook.SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred(), "failed to setup autoscaler webhook") @@ -130,7 +144,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment { mux := http.NewServeMux() mux.HandleFunc("/", autoscalerWebhook.Handle) - webhookServer = httptest.NewServer(mux) + env.webhookServer = httptest.NewServer(mux) go func() { defer GinkgoRecover() @@ -143,25 +157,24 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment { AfterEach(func() { close(stopCh) - fakeGithubServer.Close() - webhookServer.Close() + env.fakeGithubServer.Close() + env.webhookServer.Close() err := k8sClient.Delete(ctx, ns) Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace") }) - return &testEnvironment{Namespace: ns, Responses: responses} + return env } var _ = Context("INTEGRATION: Inside of a new namespace", func() { ctx := context.TODO() env := SetupIntegrationTest(ctx) ns := env.Namespace - responses := env.Responses Describe("when no existing resources exist", func() { - It("should create and scale runners", func() { + It("should create and scale runners on pull_request event", func() { name := "example-runnerdeploy" { @@ -195,7 +208,7 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() { rd.Spec.Replicas = intPtr(2) }) ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1) - ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Namespace, 2) + ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2) } // Scale-up to 3 replicas @@ -243,7 +256,7 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() { } for i, r := range runnerList.Items { - fakeRunnerList.Add(&github3.Runner{ + env.fakeRunnerList.Add(&github3.Runner{ ID: github.Int64(int64(i)), Name: github.String(r.Name), OS: github.String("linux"), @@ -252,7 +265,7 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() { }) } - rs, err := ghClient.ListRunners(context.Background(), "", "", "test/valid") + rs, err := env.ghClient.ListRunners(context.Background(), "", "", "test/valid") Expect(err).NotTo(HaveOccurred(), "verifying list fake runners response") Expect(len(rs)).To(Equal(3), "count of fake list runners") } @@ -261,9 +274,9 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() { { time.Sleep(time.Second) - responses.ListRepositoryWorkflowRuns.Body = workflowRunsFor1Replicas - responses.ListRepositoryWorkflowRuns.Statuses["queued"] = workflowRunsFor1Replicas_queued - responses.ListRepositoryWorkflowRuns.Statuses["in_progress"] = workflowRunsFor1Replicas_in_progress + env.Responses.ListRepositoryWorkflowRuns.Body = workflowRunsFor1Replicas + env.Responses.ListRepositoryWorkflowRuns.Statuses["queued"] = workflowRunsFor1Replicas_queued + env.Responses.ListRepositoryWorkflowRuns.Statuses["in_progress"] = workflowRunsFor1Replicas_in_progress var hra actionsv1alpha1.HorizontalRunnerAutoscaler @@ -284,24 +297,133 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() { // Scale-up to 2 replicas on first pull_request create webhook event { - SendPullRequestEvent("test/valid", "main", "created") + env.SendPullRequestEvent("test/valid", "main", "created") ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook") ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event") } // Scale-up to 3 replicas on second pull_request create webhook event { - SendPullRequestEvent("test/valid", "main", "created") + env.SendPullRequestEvent("test/valid", "main", "created") ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 3, "runners after second webhook event") } }) + + It("should create and scale runners on check_run event", func() { + name := "example-runnerdeploy" + + { + rd := &actionsv1alpha1.RunnerDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns.Name, + }, + Spec: actionsv1alpha1.RunnerDeploymentSpec{ + Replicas: intPtr(1), + Template: actionsv1alpha1.RunnerTemplate{ + Spec: actionsv1alpha1.RunnerSpec{ + Repository: "test/valid", + Image: "bar", + Group: "baz", + Env: []corev1.EnvVar{ + {Name: "FOO", Value: "FOOVALUE"}, + }, + }, + }, + }, + } + + ExpectCreate(ctx, rd, "test RunnerDeployment") + ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1) + ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1) + } + + { + env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners") + } + + // Scale-up to 3 replicas by the default TotalNumberOfQueuedAndInProgressWorkflowRuns-based scaling + // See workflowRunsFor3Replicas_queued and workflowRunsFor3Replicas_in_progress for GitHub List-Runners API responses + // used while testing. + { + hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns.Name, + }, + Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{ + ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{ + Name: name, + }, + MinReplicas: intPtr(1), + MaxReplicas: intPtr(5), + ScaleDownDelaySecondsAfterScaleUp: intPtr(1), + Metrics: nil, + ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{ + { + GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{ + CheckRun: &actionsv1alpha1.CheckRunSpec{ + Types: []string{"created"}, + Status: "pending", + }, + }, + Amount: 1, + Duration: metav1.Duration{Duration: time.Minute}, + }, + }, + }, + } + + ExpectCreate(ctx, hra, "test HorizontalRunnerAutoscaler") + + ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1) + ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 3) + } + + { + env.ExpectRegisteredNumberCountEventuallyEquals(3, "count of fake list runners") + } + + // Scale-up to 4 replicas on first check_run create webhook event + { + env.SendCheckRunEvent("test/valid", "pending", "created") + ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook") + ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 4, "runners after first webhook event") + } + + { + env.ExpectRegisteredNumberCountEventuallyEquals(4, "count of fake list runners") + } + + // Scale-up to 5 replicas on second check_run create webhook event + { + env.SendCheckRunEvent("test/valid", "pending", "created") + ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 5, "runners after second webhook event") + } + + env.ExpectRegisteredNumberCountEventuallyEquals(5, "count of fake list runners") + }) }) }) -func SendPullRequestEvent(repo string, branch string, action string) { +func (env *testEnvironment) ExpectRegisteredNumberCountEventuallyEquals(want int, optionalDescriptions ...interface{}) { + EventuallyWithOffset( + 1, + func() int { + env.SyncRunnerRegistrations() + + rs, err := env.ghClient.ListRunners(context.Background(), "", "", "test/valid") + Expect(err).NotTo(HaveOccurred(), "verifying list fake runners response") + + return len(rs) + }, + time.Second*1, time.Millisecond*500).Should(Equal(want), optionalDescriptions...) +} + +func (env *testEnvironment) SendPullRequestEvent(repo string, branch string, action string) { org := strings.Split(repo, "/")[0] - resp, err := sendWebhook(webhookServer, "pull_request", &github.PullRequestEvent{ + resp, err := sendWebhook(env.webhookServer, "pull_request", &github.PullRequestEvent{ PullRequest: &github.PullRequest{ Base: &github.PullRequestBranch{ Ref: github.String(branch), @@ -321,6 +443,46 @@ func SendPullRequestEvent(repo string, branch string, action string) { ExpectWithOffset(1, resp.StatusCode).To(Equal(200)) } +func (env *testEnvironment) SendCheckRunEvent(repo string, status string, action string) { + org := strings.Split(repo, "/")[0] + + resp, err := sendWebhook(env.webhookServer, "check_run", &github.CheckRunEvent{ + CheckRun: &github.CheckRun{ + Status: github.String(status), + }, + Org: &github.Organization{ + Login: github.String(org), + }, + Repo: &github.Repository{ + Name: github.String(repo), + }, + Action: github.String(action), + }) + + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "failed to send check_run event") + + ExpectWithOffset(1, resp.StatusCode).To(Equal(200)) +} + +func (env *testEnvironment) SyncRunnerRegistrations() { + var runnerList actionsv1alpha1.RunnerList + + err := k8sClient.List(context.TODO(), &runnerList, client.InNamespace(env.Namespace.Name)) + if err != nil { + logf.Log.Error(err, "list runners") + } + + for i, r := range runnerList.Items { + env.fakeRunnerList.Add(&github3.Runner{ + ID: github.Int64(int64(i)), + Name: github.String(r.Name), + OS: github.String("linux"), + Status: github.String("online"), + Busy: github.Bool(false), + }) + } +} + func ExpectCreate(ctx context.Context, rd runtime.Object, s string) { err := k8sClient.Create(ctx, rd) @@ -360,7 +522,7 @@ func ExpectRunnerSetsCountEventuallyEquals(ctx context.Context, ns string, count return len(runnerSets.Items) }, - time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(count), optionalDescription...) + time.Second*10, time.Millisecond*500).Should(BeEquivalentTo(count), optionalDescription...) } func ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx context.Context, ns string, count int, optionalDescription ...interface{}) { @@ -379,6 +541,11 @@ func ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx context.Context, n return -1 } + if len(runnerSets.Items) != 1 { + logf.Log.Info("Too many runnerreplicasets exist", "runnerSets", runnerSets) + return -1 + } + return *runnerSets.Items[0].Spec.Replicas }, time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(count), optionalDescription...) diff --git a/controllers/runnerdeployment_controller.go b/controllers/runnerdeployment_controller.go index a37bbad6..9fc8a79f 100644 --- a/controllers/runnerdeployment_controller.go +++ b/controllers/runnerdeployment_controller.go @@ -52,6 +52,7 @@ type RunnerDeploymentReconciler struct { Recorder record.EventRecorder Scheme *runtime.Scheme CommonRunnerLabels []string + Name string } // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;create;update;patch;delete @@ -291,6 +292,10 @@ func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeplo func (r *RunnerDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { name := "runnerdeployment-controller" + if r.Name != "" { + name = r.Name + } + r.Recorder = mgr.GetEventRecorderFor(name) if err := mgr.GetFieldIndexer().IndexField(&v1alpha1.RunnerReplicaSet{}, runnerSetOwnerKey, func(rawObj runtime.Object) []string { diff --git a/controllers/runnerreplicaset_controller.go b/controllers/runnerreplicaset_controller.go index 7f32d5d4..461fbc83 100644 --- a/controllers/runnerreplicaset_controller.go +++ b/controllers/runnerreplicaset_controller.go @@ -44,6 +44,7 @@ type RunnerReplicaSetReconciler struct { Recorder record.EventRecorder Scheme *runtime.Scheme GitHubClient *github.Client + Name string } // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete @@ -222,6 +223,10 @@ func (r *RunnerReplicaSetReconciler) newRunner(rs v1alpha1.RunnerReplicaSet) (v1 func (r *RunnerReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error { name := "runnerreplicaset-controller" + if r.Name != "" { + name = r.Name + } + r.Recorder = mgr.GetEventRecorderFor(name) return ctrl.NewControllerManagedBy(mgr).