diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d2516cf2..7cd3d96a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,6 +18,8 @@ jobs: curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.2.0/kubebuilder_2.2.0_linux_amd64.tar.gz tar zxvf kubebuilder_2.2.0_linux_amd64.tar.gz sudo mv kubebuilder_2.2.0_linux_amd64 /usr/local/kubebuilder + - name: Run tests + run: make test - name: Build container image run: make docker-build - name: Docker Login diff --git a/api/v1alpha1/runner_types.go b/api/v1alpha1/runner_types.go index d593c0b7..234d9aaa 100644 --- a/api/v1alpha1/runner_types.go +++ b/api/v1alpha1/runner_types.go @@ -88,6 +88,42 @@ type RunnerList struct { Items []Runner `json:"items"` } -func init() { - SchemeBuilder.Register(&Runner{}, &RunnerList{}) +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:JSONPath=".spec.replicas",name=Desired,type=number +// +kubebuilder:printcolumn:JSONPath=".status.availableReplicas",name=Current,type=number +// +kubebuilder:printcolumn:JSONPath=".status.readyReplicas",name=Ready,type=number + +// RunnerSet is the Schema for the runnersets API +type RunnerSet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RunnerSetSpec `json:"spec,omitempty"` + Status RunnerSetStatus `json:"status,omitempty"` +} + +// RunnerSetSpec defines the desired state of RunnerSet +type RunnerSetSpec struct { + Replicas *int `json:"replicas"` + + Template RunnerSpec `json:"template"` +} + +type RunnerSetStatus struct { + AvailableReplicas int `json:"availableReplicas"` + ReadyReplicas int `json:"readyReplicas"` +} + +// +kubebuilder:object:root=true + +// RunnerList contains a list of Runner +type RunnerSetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []RunnerSet `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Runner{}, &RunnerList{}, &RunnerSet{}, &RunnerSetList{}) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 880556f3..db9f191a 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -84,6 +84,101 @@ func (in *RunnerList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RunnerSet) DeepCopyInto(out *RunnerSet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerSet. +func (in *RunnerSet) DeepCopy() *RunnerSet { + if in == nil { + return nil + } + out := new(RunnerSet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RunnerSet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RunnerSetList) DeepCopyInto(out *RunnerSetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RunnerSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerSetList. +func (in *RunnerSetList) DeepCopy() *RunnerSetList { + if in == nil { + return nil + } + out := new(RunnerSetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RunnerSetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RunnerSetSpec) DeepCopyInto(out *RunnerSetSpec) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int) + **out = **in + } + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerSetSpec. +func (in *RunnerSetSpec) DeepCopy() *RunnerSetSpec { + if in == nil { + return nil + } + out := new(RunnerSetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RunnerSetStatus) DeepCopyInto(out *RunnerSetStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerSetStatus. +func (in *RunnerSetStatus) DeepCopy() *RunnerSetStatus { + if in == nil { + return nil + } + out := new(RunnerSetStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RunnerSpec) DeepCopyInto(out *RunnerSpec) { *out = *in diff --git a/config/crd/bases/actions.summerwind.dev_runnersets.yaml b/config/crd/bases/actions.summerwind.dev_runnersets.yaml new file mode 100644 index 00000000..f14232a0 --- /dev/null +++ b/config/crd/bases/actions.summerwind.dev_runnersets.yaml @@ -0,0 +1,189 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: runnersets.actions.summerwind.dev +spec: + additionalPrinterColumns: + - JSONPath: .spec.replicas + name: Desired + type: number + - JSONPath: .status.availableReplicas + name: Current + type: number + - JSONPath: .status.readyReplicas + name: Ready + type: number + group: actions.summerwind.dev + names: + kind: RunnerSet + listKind: RunnerSetList + plural: runnersets + singular: runnerset + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: RunnerSet is the Schema for the runnersets API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RunnerSetSpec defines the desired state of RunnerSet + properties: + replicas: + type: integer + template: + description: RunnerSpec defines the desired state of Runner + properties: + env: + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previous defined environment variables in the + container and any service environment variables. If a variable + cannot be resolved, the reference in the input string will + be unchanged. The $(VAR_NAME) syntax can be escaped with + a double $$, ie: $$(VAR_NAME). Escaped references will never + be expanded, regardless of whether the variable exists or + not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP.' + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + description: Specifies the output format of the exposed + resources, defaults to "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + image: + type: string + repository: + minLength: 3 + pattern: ^[^/]+/[^/]+$ + type: string + required: + - repository + type: object + required: + - replicas + - template + type: object + status: + properties: + availableReplicas: + type: integer + readyReplicas: + type: integer + required: + - availableReplicas + - readyReplicas + type: object + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index a7e150e6..10ba6024 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -26,6 +26,26 @@ rules: - get - patch - update +- apiGroups: + - actions.summerwind.dev + resources: + - runnersets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - actions.summerwind.dev + resources: + - runnersets/status + verbs: + - get + - patch + - update - apiGroups: - "" resources: diff --git a/controllers/runnerset_controller.go b/controllers/runnerset_controller.go new file mode 100644 index 00000000..bb745379 --- /dev/null +++ b/controllers/runnerset_controller.go @@ -0,0 +1,164 @@ +/* +Copyright 2020 The actions-runner-controller authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/summerwind/actions-runner-controller/api/v1alpha1" +) + +// RunnerSetReconciler reconciles a Runner object +type RunnerSetReconciler struct { + client.Client + Log logr.Logger + Recorder record.EventRecorder + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/status,verbs=get;update;patch + +func (r *RunnerSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + ctx := context.Background() + log := r.Log.WithValues("runner", req.NamespacedName) + + var rs v1alpha1.RunnerSet + if err := r.Get(ctx, req.NamespacedName, &rs); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !rs.ObjectMeta.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + var allRunners v1alpha1.RunnerList + if err := r.List(ctx, &allRunners, client.InNamespace(req.Namespace)); err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + + var myRunners []v1alpha1.Runner + + var available, ready int + + for _, r := range allRunners.Items { + if metav1.IsControlledBy(&r, &rs) { + myRunners = append(myRunners, r) + + available += 1 + + if r.Status.Phase == string(corev1.PodRunning) { + ready += 1 + } + } + } + + var desired int + + if rs.Spec.Replicas != nil { + desired = *rs.Spec.Replicas + } else { + desired = 1 + } + + log.V(0).Info("debug", "desired", desired, "available", available) + + if available > desired { + n := available - desired + + for i := 0; i < n; i++ { + if err := r.Client.Delete(ctx, &myRunners[i]); err != nil { + log.Error(err, "Failed to delete runner resource") + + return ctrl.Result{}, err + } + + r.Recorder.Event(&rs, corev1.EventTypeNormal, "RunnerDeleted", fmt.Sprintf("Deleted runner '%s'", myRunners[i].Name)) + log.Info("Deleted runner", "runnerset", rs.ObjectMeta.Name) + } + } else if desired > available { + n := desired - available + + for i := 0; i < n; i++ { + newRunner, err := r.newRunner(rs) + if err != nil { + log.Error(err, "Could not create runner") + + return ctrl.Result{}, err + } + + if err := r.Client.Create(ctx, &newRunner); err != nil { + log.Error(err, "Failed to create runner resource") + + return ctrl.Result{}, err + } + } + } + + if rs.Status.AvailableReplicas != available || rs.Status.ReadyReplicas != ready { + updated := rs.DeepCopy() + updated.Status.AvailableReplicas = available + updated.Status.ReadyReplicas = ready + + if err := r.Status().Update(ctx, updated); err != nil { + log.Error(err, "Failed to update runner status") + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} + +func (r *RunnerSetReconciler) newRunner(rs v1alpha1.RunnerSet) (v1alpha1.Runner, error) { + runner := v1alpha1.Runner{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: rs.ObjectMeta.Name, + Namespace: rs.ObjectMeta.Namespace, + }, + Spec: rs.Spec.Template, + } + + if err := ctrl.SetControllerReference(&rs, &runner, r.Scheme); err != nil { + return runner, err + } + + return runner, nil +} + +func (r *RunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Recorder = mgr.GetEventRecorderFor("runnerset-controller") + + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.RunnerSet{}). + Owns(&v1alpha1.Runner{}). + Complete(r) +} diff --git a/controllers/runnerset_controller_test.go b/controllers/runnerset_controller_test.go new file mode 100644 index 00000000..1fafb570 --- /dev/null +++ b/controllers/runnerset_controller_test.go @@ -0,0 +1,192 @@ +package controllers + +import ( + "context" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "math/rand" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1" +) + +// SetupTest will set up a testing environment. +// This includes: +// * creating a Namespace to be used during the test +// * starting the 'RunnerReconciler' +// * stopping the 'RunnerSetReconciler" after the test ends +// Call this function at the start of each of your tests. +func SetupTest(ctx context.Context) *corev1.Namespace { + var stopCh chan struct{} + ns := &corev1.Namespace{} + + BeforeEach(func() { + stopCh = make(chan struct{}) + *ns = corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "testns-" + randStringRunes(5)}, + } + + err := k8sClient.Create(ctx, ns) + Expect(err).NotTo(HaveOccurred(), "failed to create test namespace") + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{}) + Expect(err).NotTo(HaveOccurred(), "failed to create manager") + + controller := &RunnerSetReconciler{ + Client: mgr.GetClient(), + Scheme: scheme.Scheme, + Log: logf.Log, + Recorder: mgr.GetEventRecorderFor("runnerset-controller"), + } + err = controller.SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred(), "failed to setup controller") + + go func() { + defer GinkgoRecover() + + err := mgr.Start(stopCh) + Expect(err).NotTo(HaveOccurred(), "failed to start manager") + }() + }) + + AfterEach(func() { + close(stopCh) + + err := k8sClient.Delete(ctx, ns) + Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace") + }) + + return ns +} + +var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz1234567890") + +func randStringRunes(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} + +func intPtr(v int) *int { + return &v +} + +var _ = Context("Inside of a new namespace", func() { + ctx := context.TODO() + ns := SetupTest(ctx) + + Describe("when no existing resources exist", func() { + + It("should create a new Runner resource from the specified template, add a another Runner on replicas increased, and removes all the replicas when set to 0", func() { + name := "example-runnerset" + + { + rs := &actionsv1alpha1.RunnerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns.Name, + }, + Spec: actionsv1alpha1.RunnerSetSpec{ + Replicas: intPtr(1), + Template: actionsv1alpha1.RunnerSpec{ + Repository: "foo/bar", + Image: "bar", + Env: []corev1.EnvVar{ + {Name: "FOO", Value: "FOOVALUE"}, + }, + }, + }, + } + + err := k8sClient.Create(ctx, rs) + + Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerSet resource") + + runners := actionsv1alpha1.RunnerList{Items: []actionsv1alpha1.Runner{}} + + Eventually( + func() int { + err := k8sClient.List(ctx, &runners, client.InNamespace(ns.Name)) + if err != nil { + logf.Log.Error(err, "list runners") + } + + return len(runners.Items) + }, + time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1)) + } + + { + // We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification + // made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas + // Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again + Eventually(func() error { + var rs actionsv1alpha1.RunnerSet + + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rs) + + Expect(err).NotTo(HaveOccurred(), "failed to get test RunnerSet resource") + + rs.Spec.Replicas = intPtr(2) + + return k8sClient.Update(ctx, &rs) + }, + time.Second*1, time.Millisecond*500).Should(BeNil()) + + runners := actionsv1alpha1.RunnerList{Items: []actionsv1alpha1.Runner{}} + + Eventually( + func() int { + err := k8sClient.List(ctx, &runners, client.InNamespace(ns.Name)) + if err != nil { + logf.Log.Error(err, "list runners") + } + + return len(runners.Items) + }, + time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2)) + } + + { + // We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification + // made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas + // Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again + Eventually(func() error { + var rs actionsv1alpha1.RunnerSet + + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rs) + + Expect(err).NotTo(HaveOccurred(), "failed to get test RunnerSet resource") + + rs.Spec.Replicas = intPtr(0) + + return k8sClient.Update(ctx, &rs) + }, + time.Second*1, time.Millisecond*500).Should(BeNil()) + + runners := actionsv1alpha1.RunnerList{Items: []actionsv1alpha1.Runner{}} + + Eventually( + func() int { + err := k8sClient.List(ctx, &runners, client.InNamespace(ns.Name)) + if err != nil { + logf.Log.Error(err, "list runners") + } + + return len(runners.Items) + }, + time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(0)) + } + }) + }) +}) diff --git a/main.go b/main.go index 783a98b1..49f9daf6 100644 --- a/main.go +++ b/main.go @@ -110,6 +110,17 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Runner") os.Exit(1) } + + runnerSetReconciler := &controllers.RunnerSetReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("RunnerSet"), + Scheme: mgr.GetScheme(), + } + + if err = runnerSetReconciler.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "RunnerSet") + os.Exit(1) + } // +kubebuilder:scaffold:builder setupLog.Info("starting manager")