public_git/actions-runner-controller
1
0
Fork 0
mirror of https://github.com/actions-runner-controller/actions-runner-controller.git synced 2026-09-30 21:41:04 +02:00
Code Issues Projects Releases Wiki Activity
Files
810127e8d9a1c8dfe6e6bd02c037ee189d08d47b
actions-runner-controller/controllers/actions.github.com/autoscalingrunnerset_controller_test.go
T
Nikola JokicandCopilot App ec793dfe9f Track runner spec updates with an actionable revision 
When the AutoscalingRunnerSet's runner spec changes, the EphemeralRunnerSet
has to delete its idle and pending runners so they are rebuilt from the new
spec. That was detected by hashing Spec.EphemeralRunnerSpec into the
actions.github.com/integrity-hash annotation and comparing the annotation
against a freshly computed hash.

Replace it with a monotonic revision counter split across spec and status.
The AutoscalingRunnerSet controller bumps Spec.ActionableRevision when it
patches a new runner spec across; the EphemeralRunnerSet controller advances
Status.AppliedActionableRevision only after the cleanup has actually
succeeded. Because the applied marker lives in status and is written last, a
controller that crashes part-way through the cleanup comes back with the
applied revision still behind the spec revision and redoes the work, instead
of skipping runners that are still running the old spec.

The drift check uses apiequality.Semantic.DeepEqual rather than cmp.Equal or
reflect.DeepEqual. Most PodSpec collection fields carry omitempty, so a
template containing an explicitly empty value (`env: []`) is dropped when the
EphemeralRunnerSet is written and reads back as nil. A strict comparison would
report drift on every reconcile, bump the revision each time, and delete every
idle and pending runner forever. helpers_drift_test.go pins that behaviour with
a round-trip-through-JSON test.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-10 22:56:48 +02:00

2715 lines
103 KiB
Go
Raw Blame History

package actionsgithubcom
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"sync"
"sync/atomic"
"time"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
logf "sigs.k8s.io/controller-runtime/pkg/log"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
"github.com/actions/actions-runner-controller/build"
"github.com/actions/actions-runner-controller/controllers/actions.github.com/multiclient"
scalefake "github.com/actions/actions-runner-controller/controllers/actions.github.com/multiclient/fake"
"github.com/actions/actions-runner-controller/controllers/actions.github.com/secretresolver"
"github.com/actions/scaleset"
)
const (
autoscalingRunnerSetTestTimeout = time.Second * 20
autoscalingRunnerSetTestInterval = time.Millisecond * 250
)
var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() {
var ctx context.Context
var mgr ctrl.Manager
var controller *AutoscalingRunnerSetReconciler
var autoscalingNS *corev1.Namespace
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
var configSecret *corev1.Secret
var resourceCache *ResourceCache
var originalBuildVersion string
buildVersion := "0.1.0"
BeforeAll(func() {
originalBuildVersion = build.Version
build.Version = buildVersion
})
AfterAll(func() {
build.Version = originalBuildVersion
})
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
// Track runner group mappings for dynamic responses
runnerGroupMap := map[int]string{1: "testgroup"} // ID -> Name mapping
runnerGroupMapLock := &sync.RWMutex{} // Thread-safe access
resourceCache = newTestResourceCache()
controller = &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: resourceCache,
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByNameFunc(func(ctx context.Context, groupName string) (*scaleset.RunnerGroup, error) {
// Support both "testgroup" and "testgroup2"
// Update the mapping when a new group is requested
runnerGroupMapLock.Lock()
runnerGroupMap[1] = groupName
runnerGroupMapLock.Unlock()
return &scaleset.RunnerGroup{ID: 1, Name: groupName}, nil
}),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSetFunc(func(ctx context.Context, rs *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) {
// Return a RunnerScaleSet with name matching the requesting ARS
runnerGroupMapLock.RLock()
groupName := runnerGroupMap[rs.RunnerGroupID]
runnerGroupMapLock.RUnlock()
return &scaleset.RunnerScaleSet{ID: 1, Name: rs.Name, RunnerGroupID: rs.RunnerGroupID, RunnerGroupName: groupName}, nil
}),
scalefake.WithUpdateRunnerScaleSetFunc(func(ctx context.Context, scaleSetID int, rs *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) {
// Return a RunnerScaleSet with the group name corresponding to the runner group ID
runnerGroupMapLock.RLock()
groupName := runnerGroupMap[rs.RunnerGroupID]
runnerGroupMapLock.RUnlock()
return &scaleset.RunnerScaleSet{ID: 1, Name: rs.Name, RunnerGroupID: rs.RunnerGroupID, RunnerGroupName: groupName}, nil
}),
scalefake.WithDeleteRunnerScaleSet(nil),
),
),
)),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
min := 1
max := 10
autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
"arc.test/label-drift": "initial",
"arc.test/spec-update-label": "initial",
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
EphemeralRunnerSetMetadata: &v1alpha1.ResourceMeta{
Annotations: map[string]string{
"arc.test/metadata-annotation": "initial",
},
},
EphemeralRunnerMetadata: &v1alpha1.ResourceMeta{
Labels: map[string]string{
"arc.test/runner-metadata-label": "initial",
},
Annotations: map[string]string{
"arc.test/runner-metadata-annotation": "initial",
},
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
startManagers(GinkgoT(), mgr)
})
Context("When creating a new AutoScalingRunnerSet", func() {
It("It should create/add all required resources for a new AutoScalingRunnerSet (finalizer, runnerscaleset, ephemeralrunnerset, listener)", func() {
// Check if finalizer is added
created := new(v1alpha1.AutoscalingRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
if err != nil {
return "", err
}
if len(created.Finalizers) == 0 {
return "", nil
}
return created.Finalizers[0], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
// Check if runner scale set is created on service
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
if err != nil {
return "", err
}
if _, ok := created.Annotations[runnerScaleSetIDAnnotationKey]; !ok {
return "", nil
}
if _, ok := created.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
return "", nil
}
return fmt.Sprintf(
"%s_%s",
created.Annotations[runnerScaleSetIDAnnotationKey],
created.Annotations[AnnotationKeyGitHubRunnerGroupName],
), nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(
BeEquivalentTo("1_testgroup"),
"RunnerScaleSet should be created/fetched and update the AutoScalingRunnerSet's annotation",
)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
if err != nil {
return "", err
}
if _, ok := created.Labels[LabelKeyGitHubOrganization]; !ok {
return "", nil
}
if _, ok := created.Labels[LabelKeyGitHubRepository]; !ok {
return "", nil
}
return fmt.Sprintf("%s/%s", created.Labels[LabelKeyGitHubOrganization], created.Labels[LabelKeyGitHubRepository]), nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo("owner/repo"), "RunnerScaleSet should be created/fetched and update the AutoScalingRunnerSet's label")
// Check if ephemeral runner set is created
Eventually(
func() (int, error) {
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
if err != nil {
return 0, err
}
return len(runnerSetList.Items), nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created")
// Check if listener is created
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
Eventually(
func() bool {
return resourceCacheStateHasMainObjectEntries(resourceCache.ephemeralRunnerSet, created) &&
resourceCacheStateHasMainObjectEntries(resourceCache.autoscalingListener, created)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "AutoScalingRunnerSet EphemeralRunnerSet and AutoScalingListener resources should be cached after reconciliation")
// Check if status is updated
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
Expect(len(runnerSetList.Items)).To(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created")
})
})
Context("When deleting a new AutoScalingRunnerSet", func() {
It("It should cleanup all resources for a deleting AutoScalingRunnerSet before removing it", func() {
// Wait till the listener is created
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
created := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
Expect(err).NotTo(HaveOccurred(), "failed to get AutoScalingRunnerSet")
Eventually(
func() bool {
return resourceCacheStateHasMainObjectEntries(resourceCache.ephemeralRunnerSet, created) &&
resourceCacheStateHasMainObjectEntries(resourceCache.autoscalingListener, created)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "AutoScalingRunnerSet EphemeralRunnerSet and AutoScalingListener resources should be cached before deletion")
// Delete the AutoScalingRunnerSet
err = k8sClient.Delete(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to delete AutoScalingRunnerSet")
// Check if the listener is deleted
Eventually(
func() error {
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
if err != nil && errors.IsNotFound(err) {
return nil
}
return fmt.Errorf("listener is not deleted")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be deleted")
// Check if all the EphemeralRunnerSet is deleted
Eventually(
func() error {
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
if err != nil {
return err
}
if len(runnerSetList.Items) != 0 {
return fmt.Errorf("EphemeralRunnerSet is not deleted, count=%v", len(runnerSetList.Items))
}
return nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "All EphemeralRunnerSet should be deleted")
// Check if the AutoScalingRunnerSet is deleted
Eventually(
func() error {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingRunnerSet))
if err != nil && errors.IsNotFound(err) {
return nil
}
return fmt.Errorf("AutoScalingRunnerSet is not deleted")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "AutoScalingRunnerSet should be deleted")
Eventually(
func() bool {
return resourceCacheStateHasMainObjectEntries(resourceCache.ephemeralRunnerSet, created) ||
resourceCacheStateHasMainObjectEntries(resourceCache.autoscalingListener, created)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeFalse(), "AutoScalingRunnerSet EphemeralRunnerSet and AutoScalingListener resources should be removed from cache after deletion")
})
})
It("should not churn listener when already referencing latest ERS (no-op stability)", func() {
min := 1
max := 10
testARSName := "test-asrs-no-churn"
testARSNamespace := autoscalingNS.Name
testARS := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: testARSName,
Namespace: testARSNamespace,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, testARS)
Expect(err).NotTo(HaveOccurred(), "failed to create test AutoScalingRunnerSet")
var latestERSName string
Eventually(
func() (string, error) {
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(testARSNamespace))
if err != nil {
return "", err
}
// Filter to only ERS owned by our test ARS
var ownedByTestARS []v1alpha1.EphemeralRunnerSet
for _, ers := range runnerSetList.Items {
for _, owner := range ers.OwnerReferences {
if owner.UID == testARS.UID {
ownedByTestARS = append(ownedByTestARS, ers)
break
}
}
}
if len(ownedByTestARS) != 1 {
return "", fmt.Errorf("expected 1 EphemeralRunnerSet owned by test ARS, got %d", len(ownedByTestARS))
}
return ownedByTestARS[0].Name, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Not(BeEmpty()), "ERS should be created")
// Capture the latest ERS name
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err = k8sClient.List(ctx, runnerSetList, client.InNamespace(testARSNamespace))
Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
var ownedByTestARS []v1alpha1.EphemeralRunnerSet
for _, ers := range runnerSetList.Items {
for _, owner := range ers.OwnerReferences {
if owner.UID == testARS.UID {
ownedByTestARS = append(ownedByTestARS, ers)
break
}
}
}
Expect(len(ownedByTestARS)).To(Equal(1), "should have exactly 1 EphemeralRunnerSet owned by test ARS")
latestERSName = ownedByTestARS[0].Name
listener := new(v1alpha1.AutoscalingListener)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, listener)
if err != nil {
return "", err
}
return listener.Spec.EphemeralRunnerSetName, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Equal(latestERSName), "listener should reference the latest ERS")
// Capture listener identity (UID and ResourceVersion)
err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, listener)
Expect(err).NotTo(HaveOccurred(), "failed to get listener")
originalUID := listener.UID
originalResourceVersion := listener.ResourceVersion
Consistently(
func() (types.UID, error) {
currentListener := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, currentListener)
if err != nil {
return "", err
}
return currentListener.UID, nil
},
time.Second*5,
autoscalingRunnerSetTestInterval,
).Should(Equal(originalUID), "listener UID should remain unchanged (no recreation)")
Consistently(
func() (string, error) {
currentListener := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, currentListener)
if err != nil {
return "", err
}
return currentListener.ResourceVersion, nil
},
time.Second*5,
autoscalingRunnerSetTestInterval,
).Should(Equal(originalResourceVersion), "listener ResourceVersion should remain unchanged (no updates)")
err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, listener)
Expect(err).NotTo(HaveOccurred(), "failed to get listener")
Expect(listener.Spec.EphemeralRunnerSetName).To(Equal(latestERSName), "listener should still reference latest ERS")
})
Context("When updating a new AutoScalingRunnerSet", func() {
It("advances the observed generation once a spec change has been applied", func() {
var settledGeneration int64
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation))
settledGeneration = current.Generation
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "AutoscalingRunnerSet should settle with its generation observed")
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated"
Expect(k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))).To(Succeed(), "failed to patch AutoScalingRunnerSet")
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
g.Expect(current.Generation).To(BeNumerically(">", settledGeneration), "a spec write must bump metadata.generation")
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation), "observed generation must catch up once the change is applied")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
// A label-only edit does not bump metadata.generation, so it never
// advances the observed generation. It does still rebuild the listener,
// because the desired listener labels are derived from the
// AutoscalingRunnerSet's, and the phase has to report that rebuild. So
// the scale set passes through Pending transiently and comes back to
// Running, while the observed generation stays exactly where it was.
It("does not re-observe a generation when only labels change, but still reports the listener rebuild", func() {
listener := new(v1alpha1.AutoscalingListener)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
originalListenerUID := listener.UID
var settledGeneration int64
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation))
settledGeneration = current.Generation
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "AutoscalingRunnerSet should settle with its generation observed")
// Hold the deletion window open. Without this the listener is deleted
// and re-created between two polls, so an implementation that never
// reported Pending would look identical to one that did. The
// AutoscalingListener controller does not run in this suite, so
// nothing else is going to clear this finalizer.
blockDeletion(listener)
defer unblockDeletion(listener)
patched := autoscalingRunnerSet.DeepCopy()
patched.Labels["arc.test/label-drift"] = "updated"
Expect(k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))).To(Succeed(), "failed to patch AutoScalingRunnerSet labels")
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)).To(Succeed())
g.Expect(current.Labels).To(HaveKeyWithValue("arc.test/label-drift", "updated"), "labels should still propagate to the EphemeralRunnerSet")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
// The listener is on its way out, so the scale set must not claim to
// be running. This is the assertion that fails if the Pending update
// before the listener delete is removed.
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingListener)
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)).To(Succeed())
g.Expect(current.DeletionTimestamp).NotTo(BeNil(), "listener should be marked for deletion to pick up the new label")
ars := new(v1alpha1.AutoscalingRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), ars)).To(Succeed())
g.Expect(ars.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhasePending), "scale set must report Pending while its listener is being rebuilt")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
// It stays Pending for as long as the rebuild has not happened, and
// the observed generation does not drift in the meantime.
Consistently(
func(g Gomega) {
ars := new(v1alpha1.AutoscalingRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), ars)).To(Succeed())
g.Expect(ars.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhasePending), "scale set must stay Pending until the listener is back")
g.Expect(ars.Status.ObservedGeneration).To(Equal(settledGeneration))
},
3*time.Second,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
// Let the rebuild finish.
unblockDeletion(listener)
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingListener)
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)).To(Succeed())
g.Expect(current.UID).NotTo(Equal(originalListenerUID), "listener should be re-created to pick up the new label")
g.Expect(current.Labels).To(HaveKeyWithValue("arc.test/label-drift", "updated"), "listener should carry the new label")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
// Once the listener is back, the scale set settles on Running again
// without the observed generation ever moving, because no spec write
// happened.
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "AutoscalingRunnerSet should return to Running once the listener is rebuilt")
Consistently(
func(g Gomega) {
current := new(v1alpha1.AutoscalingRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
g.Expect(current.Generation).To(Equal(settledGeneration), "a label-only edit must not bump metadata.generation")
g.Expect(current.Status.ObservedGeneration).To(Equal(settledGeneration), "a label-only edit must not move the observed generation")
},
3*time.Second,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
// The observed generation is only meant to advance once a change has
// actually been applied. A reconcile that cannot finish must leave it
// behind the live generation, so the work is retried rather than being
// mistaken for settled.
It("leaves the observed generation behind until the reconcile completes", func() {
listener := new(v1alpha1.AutoscalingListener)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
var settledGeneration int64
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation))
settledGeneration = current.Generation
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "AutoscalingRunnerSet should settle with its generation observed")
// Stall the reconcile part way through: the max runner change below
// forces the listener to be rebuilt, and the rebuild cannot complete
// while the listener is held in deletion.
blockDeletion(listener)
defer unblockDeletion(listener)
patched := autoscalingRunnerSet.DeepCopy()
updatedMax := 20
patched.Spec.MaxRunners = &updatedMax
Expect(k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))).To(Succeed(), "failed to patch AutoScalingRunnerSet max runners")
var liveGeneration int64
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
g.Expect(current.Generation).To(BeNumerically(">", settledGeneration), "a spec write must bump metadata.generation")
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhasePending), "the scale set should go Pending while the change is being applied")
liveGeneration = current.Generation
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
Consistently(
func(g Gomega) {
current := new(v1alpha1.AutoscalingRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
g.Expect(current.Status.ObservedGeneration).To(Equal(settledGeneration), "observed generation must not advance while the change is still being applied")
g.Expect(current.Status.ObservedGeneration).To(BeNumerically("<", liveGeneration))
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhasePending), "the scale set must stay Pending until the change is applied")
},
3*time.Second,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
// Let the reconcile complete, and only now should the marker catch up.
unblockDeletion(listener)
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
g.Expect(current.Status.ObservedGeneration).To(Equal(liveGeneration), "observed generation must catch up once the change has been applied")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
It("updates EphemeralRunnerSet when the runner image changes without touching the Listener", func() {
listener := new(v1alpha1.AutoscalingListener)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
originalListenerUID := listener.UID
originalListenerResourceVersion := listener.ResourceVersion
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "EphemeralRunnerSet should be created")
originalRunnerSetUID := runnerSet.UID
originalActionableRevision := runnerSet.Spec.ActionableRevision
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated"
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should be updated in place")
g.Expect(current.Spec.EphemeralRunnerSpec.PodTemplateSpec.Spec.Containers[0].Image).To(Equal("ghcr.io/actions/runner:updated"))
g.Expect(current.Spec.ActionableRevision).To(BeNumerically(">", originalActionableRevision), "ActionableRevision should increment for actionable spec changes")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
Consistently(
func(g Gomega) {
current := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener")
g.Expect(current.UID).To(Equal(originalListenerUID), "Listener should not be recreated")
g.Expect(current.ResourceVersion).To(Equal(originalListenerResourceVersion), "Listener should not be updated")
},
time.Second*5,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
It("recreates only the Listener when max runners changes", func() {
listener := new(v1alpha1.AutoscalingListener)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
originalListenerUID := listener.UID
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "EphemeralRunnerSet should be created")
originalRunnerSetUID := runnerSet.UID
originalActionableRevision := runnerSet.Spec.ActionableRevision
patched := autoscalingRunnerSet.DeepCopy()
max := 20
patched.Spec.MaxRunners = &max
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener")
g.Expect(current.UID).NotTo(Equal(originalListenerUID), "Listener should be recreated")
g.Expect(current.Spec.MaxRunners).To(Equal(max))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
Consistently(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should not be recreated")
g.Expect(current.Spec.ActionableRevision).To(Equal(originalActionableRevision), "ActionableRevision should not change for non-actionable updates")
},
time.Second*5,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
It("updates EphemeralRunnerSet labels when only AutoScalingRunnerSet labels change", func() {
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
if err != nil {
return "", err
}
return runnerSet.Labels["arc.test/label-drift"], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined label")
patched := autoscalingRunnerSet.DeepCopy()
patched.Labels["arc.test/label-drift"] = "updated"
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet labels")
Eventually(
func() (string, error) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
if err != nil {
return "", err
}
return current.Labels["arc.test/label-drift"], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Equal("updated"), "EphemeralRunnerSet should be patched with label-only drift")
})
It("updates EphemeralRunnerSet annotations when only EphemeralRunnerSet metadata annotations change", func() {
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
if err != nil {
return "", err
}
return runnerSet.Annotations["arc.test/metadata-annotation"], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined annotation")
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated"
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/new-metadata-annotation"] = "added"
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunnerSet metadata")
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated"))
g.Expect(current.Annotations["arc.test/new-metadata-annotation"]).To(Equal("added"))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "EphemeralRunnerSet should be patched with annotation-only metadata drift")
})
It("preserves foreign annotations and labels on the EphemeralRunnerSet", func() {
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
if err != nil {
return "", err
}
return runnerSet.Annotations["arc.test/metadata-annotation"], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined annotation")
// Simulate a third party (admission webhook, another controller, a user)
// adding metadata the AutoscalingRunnerSet knows nothing about.
foreign := runnerSet.DeepCopy()
foreign.Annotations["thirdparty.example.com/injected"] = "keep-me"
foreign.Labels["thirdparty.example.com/injected"] = "keep-me"
err := k8sClient.Patch(ctx, foreign, client.MergeFrom(runnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to inject foreign metadata on EphemeralRunnerSet")
// Force the controller through the metadata reconciliation path.
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated"
err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunnerSet metadata")
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated"))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "desired annotation should still propagate")
// The foreign keys must survive, and the controller must be able to get
// past the metadata block. Before the fix the comparison was made
// against the unmerged desired metadata, so a foreign key made the
// block report "modified" on every reconcile and return early, which
// meant nothing after it — including listener reconciliation — ever
// ran again. Deleting the listener makes that stall observable.
listener := new(v1alpha1.AutoscalingListener)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "listener should exist before deletion")
Expect(k8sClient.Delete(ctx, listener)).To(Succeed())
Eventually(
func(g Gomega) {
recreated := new(v1alpha1.AutoscalingListener)
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, recreated)).To(Succeed())
current := new(v1alpha1.EphemeralRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)).To(Succeed())
g.Expect(current.Annotations).To(HaveKeyWithValue("thirdparty.example.com/injected", "keep-me"), "foreign annotation must not be stripped")
g.Expect(current.Labels).To(HaveKeyWithValue("thirdparty.example.com/injected", "keep-me"), "foreign label must not be stripped")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "reconciliation must converge past the metadata block instead of looping on it")
})
It("updates EphemeralRunnerSet runner metadata when only EphemeralRunner metadata changes", func() {
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func(g Gomega) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(runnerSet.Spec.EphemeralRunnerMetadata).NotTo(BeNil())
g.Expect(runnerSet.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"]).To(Equal("initial"))
g.Expect(runnerSet.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"]).To(Equal("initial"))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"] = "updated"
patched.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"] = "updated"
patched.Spec.EphemeralRunnerMetadata.Annotations["arc.test/new-runner-metadata-annotation"] = "added"
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunner metadata")
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.Spec.EphemeralRunnerMetadata).NotTo(BeNil())
g.Expect(current.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"]).To(Equal("updated"))
g.Expect(current.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"]).To(Equal("updated"))
g.Expect(current.Spec.EphemeralRunnerMetadata.Annotations["arc.test/new-runner-metadata-annotation"]).To(Equal("added"))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
It("propagates latest labels while updating EphemeralRunnerSet spec", func() {
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
if err != nil {
return "", err
}
return runnerSet.Labels["arc.test/spec-update-label"], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined label")
patched := autoscalingRunnerSet.DeepCopy()
patched.Labels["arc.test/spec-update-label"] = "updated"
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated"
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/new-spec-update-annotation"] = "added"
patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated-with-label"
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet spec and labels")
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.Spec.EphemeralRunnerSpec.PodTemplateSpec.Spec.Containers[0].Image).To(Equal("ghcr.io/actions/runner:updated-with-label"))
g.Expect(current.Labels["arc.test/spec-update-label"]).To(Equal("updated"))
g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated"))
g.Expect(current.Annotations["arc.test/new-spec-update-annotation"]).To(Equal("added"))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
It("updates EphemeralRunnerSet and Listener when the GitHub config secret changes", func() {
updatedSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "github-config-secret-updated",
Namespace: autoscalingRunnerSet.Namespace,
},
Data: map[string][]byte{
"github_token": []byte(defaultGitHubToken),
},
}
err := k8sClient.Create(ctx, updatedSecret)
Expect(err).NotTo(HaveOccurred(), "failed to create updated GitHub config secret")
listener := new(v1alpha1.AutoscalingListener)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
originalListenerUID := listener.UID
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "EphemeralRunnerSet should be created")
originalRunnerSetUID := runnerSet.UID
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.GitHubConfigSecret = updatedSecret.Name
err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get AutoScalingRunnerSet")
g.Expect(current.Spec.GitHubConfigSecret).To(Equal(updatedSecret.Name))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should be updated in place")
g.Expect(current.Spec.EphemeralRunnerSpec.GitHubConfigSecret).To(Equal(updatedSecret.Name))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener")
g.Expect(current.UID).NotTo(Equal(originalListenerUID), "Listener should be recreated")
g.Expect(current.Spec.GitHubConfigSecret).To(Equal(updatedSecret.Name))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
It("It should update RunnerScaleSet's runner group on service when it changes", func() {
updated := new(v1alpha1.AutoscalingRunnerSet)
// Wait till the listener is created
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.RunnerGroup = "testgroup2"
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
// Check if AutoScalingRunnerSet has the new runner group in its annotation
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
if err != nil {
return "", err
}
if _, ok := updated.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
return "", nil
}
return updated.Annotations[AnnotationKeyGitHubRunnerGroupName], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the new runner group in its annotation")
// delete the annotation and it should be re-added
patched = autoscalingRunnerSet.DeepCopy()
delete(patched.Annotations, AnnotationKeyGitHubRunnerGroupName)
err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
// Check if AutoScalingRunnerSet still has the runner group in its annotation
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
if err != nil {
return "", err
}
if _, ok := updated.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
return "", nil
}
return updated.Annotations[AnnotationKeyGitHubRunnerGroupName], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the runner group in its annotation")
})
})
It("Should update Status on EphemeralRunnerSet status Update", func() {
ars := new(v1alpha1.AutoscalingRunnerSet)
Eventually(
func() (bool, error) {
err := k8sClient.Get(
ctx,
client.ObjectKey{
Name: autoscalingRunnerSet.Name,
Namespace: autoscalingRunnerSet.Namespace,
},
ars,
)
if err != nil {
return false, err
}
return true, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "AutoscalingRunnerSet should be created")
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
Eventually(
func() (int, error) {
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(ars.Namespace))
if err != nil {
return 0, err
}
return len(runnerSetList.Items), nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(1), "Failed to fetch runner set list")
runnerSet := runnerSetList.Items[0]
statusUpdate := runnerSet.DeepCopy()
statusUpdate.Status.Phase = v1alpha1.EphemeralRunnerSetPhaseRunning
err := k8sClient.Status().Patch(ctx, statusUpdate, client.MergeFrom(&runnerSet))
Expect(err).NotTo(HaveOccurred(), "Failed to patch runner set status")
Eventually(
func() (v1alpha1.AutoscalingRunnerSetStatus, error) {
updated := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
if err != nil {
return v1alpha1.AutoscalingRunnerSetStatus{}, fmt.Errorf("failed to get AutoScalingRunnerSet: %w", err)
}
return updated.Status, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(SatisfyAll(
WithTransform(func(s v1alpha1.AutoscalingRunnerSetStatus) v1alpha1.AutoscalingRunnerSetPhase { return s.Phase }, Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning)),
WithTransform(func(s v1alpha1.AutoscalingRunnerSetStatus) int64 { return s.ObservedGeneration }, BeNumerically(">=", ars.Generation)),
), "AutoScalingRunnerSet status should be updated")
})
})
var _ = Describe("Test AutoScalingController updates", Ordered, func() {
var originalBuildVersion string
buildVersion := "0.1.0"
BeforeAll(func() {
originalBuildVersion = build.Version
build.Version = buildVersion
})
AfterAll(func() {
build.Version = originalBuildVersion
})
Context("Creating autoscaling runner set with RunnerScaleSetName set", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
var configSecret *corev1.Secret
BeforeEach(func() {
originalBuildVersion = build.Version
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
multiClient := scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGenerateJitRunnerConfig(
&scaleset.RunnerScaleSetJitRunnerConfig{
Runner: &scaleset.RunnerReference{ID: 1, Name: "test-runner"},
EncodedJITConfig: "fake-jit-config",
},
nil,
),
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "testset", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithUpdateRunnerScaleSet(
&scaleset.RunnerScaleSet{
ID: 1,
Name: "testset_update",
RunnerGroupID: 1,
RunnerGroupName: "testgroup",
Labels: []scaleset.Label{{Type: "test", Name: "test"}},
RunnerSetting: scaleset.RunnerSetting{},
CreatedOn: time.Now(),
RunnerJitConfigURL: "test.test.test",
Statistics: nil,
},
nil,
),
),
),
)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), multiClient),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
})
It("It should be create AutoScalingRunnerSet and has annotation for the RunnerScaleSetName", func() {
min := 1
max := 10
autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerScaleSetName: "testset",
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
// Wait for the AutoScalingRunnerSet to be created with right annotation
ars := new(v1alpha1.AutoscalingRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
if err != nil {
return "", err
}
if val, ok := ars.Annotations[AnnotationKeyGitHubRunnerScaleSetName]; ok {
return val, nil
}
return "", nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(autoscalingRunnerSet.Spec.RunnerScaleSetName), "AutoScalingRunnerSet should have annotation for the RunnerScaleSetName")
update := autoscalingRunnerSet.DeepCopy()
update.Spec.RunnerScaleSetName = "testset_update"
err = k8sClient.Patch(ctx, update, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to update AutoScalingRunnerSet")
// Wait for the AutoScalingRunnerSet to be updated with right annotation
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
if err != nil {
return "", err
}
if val, ok := ars.Annotations[AnnotationKeyGitHubRunnerScaleSetName]; ok {
return val, nil
}
return "", nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(update.Spec.RunnerScaleSetName), "AutoScalingRunnerSet should have a updated annotation for the RunnerScaleSetName")
})
})
})
var _ = Describe("Test AutoscalingController creation failures", Ordered, func() {
var originalBuildVersion string
buildVersion := "0.1.0"
BeforeAll(func() {
originalBuildVersion = build.Version
build.Version = buildVersion
})
AfterAll(func() {
build.Version = originalBuildVersion
})
Context("When autoscaling runner set creation fails on the client", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
})
It("It should be able to clean up if annotation related to scale set id does not exist", func() {
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
// wait for the finalizer to be added
ars := new(v1alpha1.AutoscalingRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
if err != nil {
return "", err
}
if len(ars.Finalizers) == 0 {
return "", nil
}
return ars.Finalizers[0], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
ars.Annotations = make(map[string]string)
err = k8sClient.Update(ctx, ars)
Expect(err).NotTo(HaveOccurred(), "Update autoscaling runner set without annotation should be successful")
Eventually(
func() (bool, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
if err != nil {
return false, err
}
return len(ars.ObjectMeta.Annotations) == 0, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(true), "Autoscaling runner set should be updated with empty annotations")
err = k8sClient.Delete(ctx, ars)
Expect(err).NotTo(HaveOccurred(), "Delete autoscaling runner set should be successful")
Eventually(
func() (bool, error) {
updated := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
if err == nil {
return false, nil
}
if !errors.IsNotFound(err) {
return false, err
}
return !controllerutil.ContainsFinalizer(updated, autoscalingRunnerSetFinalizerName), nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(true), "Finalizer and resource should eventually be deleted")
})
})
})
var _ = Describe("Test client optional configuration", Ordered, func() {
var originalBuildVersion string
buildVersion := "0.1.0"
BeforeAll(func() {
originalBuildVersion = build.Version
build.Version = buildVersion
})
AfterAll(func() {
build.Version = originalBuildVersion
})
Context("When specifying a proxy", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var configSecret *corev1.Secret
var controller *AutoscalingRunnerSetReconciler
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
controller = &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), multiclient.NewScaleset()),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
})
It("should be able to make requests to a server using a proxy", func() {
serverSuccessfullyCalled := false
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
serverSuccessfullyCalled = true
w.WriteHeader(http.StatusOK)
}))
defer proxy.Close()
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "http://example.com/org/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Proxy: &v1alpha1.ProxyConfig{
HTTP: &v1alpha1.ProxyServerConfig{
Url: proxy.URL,
},
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
// wait for server to be called
Eventually(
func() (bool, error) {
return serverSuccessfullyCalled, nil
},
autoscalingRunnerSetTestTimeout,
1*time.Nanosecond,
).Should(BeTrue(), "server was not called")
})
It("should be able to make requests to a server using a proxy with user info", func() {
controller.ResourceBuilder.SecretResolver = secretresolver.New(k8sClient, scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithUpdateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithDeleteRunnerScaleSet(nil),
),
),
))
secretCredentials := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "proxy-credentials",
Namespace: autoscalingNS.Name,
},
Data: map[string][]byte{
"username": []byte("test"),
"password": []byte("password"),
},
}
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer proxy.Close()
err := k8sClient.Create(ctx, secretCredentials)
Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials")
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "http://example.com/org/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Proxy: &v1alpha1.ProxyConfig{
HTTP: &v1alpha1.ProxyServerConfig{
Url: "http://test:password@" + proxy.Listener.Addr().String(),
CredentialSecretRef: "proxy-credentials",
},
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
// Verify proxy config with credentials is propagated to EphemeralRunnerSet
Eventually(
func() (*v1alpha1.EphemeralRunnerSet, error) {
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingNS.Name))
if err != nil || len(runnerSetList.Items) == 0 {
return nil, err
}
return &runnerSetList.Items[0], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(WithTransform(func(ers *v1alpha1.EphemeralRunnerSet) *v1alpha1.ProxyConfig {
if ers != nil {
return ers.Spec.EphemeralRunnerSpec.Proxy
}
return nil
}, Not(BeNil())), "EphemeralRunnerSet should have proxy configuration with credentials")
})
})
Context("When specifying a configmap for root CAs", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var configSecret *corev1.Secret
var rootCAConfigMap *corev1.ConfigMap
var controller *AutoscalingRunnerSetReconciler
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
cert, err := os.ReadFile(filepath.Join(
"../../",
"github",
"actions",
"testdata",
"rootCA.crt",
))
Expect(err).NotTo(HaveOccurred(), "failed to read root CA cert")
rootCAConfigMap = &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "root-ca-configmap",
Namespace: autoscalingNS.Name,
},
Data: map[string]string{
"rootCA.crt": string(cert),
},
}
err = k8sClient.Create(ctx, rootCAConfigMap)
Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs")
controller = &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithUpdateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithDeleteRunnerScaleSet(nil),
),
),
)),
},
}
err = controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
})
It("should be able to make requests to a server using root CAs", func() {
controller.SecretResolver = secretresolver.New(k8sClient, multiclient.NewScaleset())
certsFolder := filepath.Join(
"../../",
"github",
"actions",
"testdata",
)
certPath := filepath.Join(certsFolder, "server.crt")
keyPath := filepath.Join(certsFolder, "server.key")
serverSuccessfullyCalled := false
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
serverSuccessfullyCalled = true
w.WriteHeader(http.StatusOK)
}))
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
Expect(err).NotTo(HaveOccurred(), "failed to load server cert")
server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}}
server.StartTLS()
defer server.Close()
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: server.URL + "/my-org",
GitHubConfigSecret: configSecret.Name,
GitHubServerTLS: &v1alpha1.TLSConfig{
CertificateFrom: &v1alpha1.TLSCertificateSource{
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: rootCAConfigMap.Name,
},
Key: "rootCA.crt",
},
},
},
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
// wait for server to be called
Eventually(
func() (bool, error) {
return serverSuccessfullyCalled, nil
},
autoscalingRunnerSetTestTimeout,
1*time.Nanosecond,
).Should(BeTrue(), "server was not called")
})
It("it creates a listener referencing the right configmap for TLS", func() {
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
GitHubServerTLS: &v1alpha1.TLSConfig{
CertificateFrom: &v1alpha1.TLSCertificateSource{
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: rootCAConfigMap.Name,
},
Key: "rootCA.crt",
},
},
},
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
Eventually(
func(g Gomega) {
listener := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(
ctx,
client.ObjectKey{
Name: scaleSetListenerName(autoscalingRunnerSet),
Namespace: autoscalingRunnerSet.Namespace,
},
listener,
)
g.Expect(err).NotTo(HaveOccurred(), "failed to get listener")
g.Expect(listener.Spec.GitHubServerTLS).NotTo(BeNil(), "listener does not have TLS config")
g.Expect(listener.Spec.GitHubServerTLS).To(BeEquivalentTo(autoscalingRunnerSet.Spec.GitHubServerTLS), "listener does not have TLS config")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "tls config is incorrect")
})
It("it creates an ephemeral runner set referencing the right configmap for TLS", func() {
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
GitHubServerTLS: &v1alpha1.TLSConfig{
CertificateFrom: &v1alpha1.TLSCertificateSource{
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: rootCAConfigMap.Name,
},
Key: "rootCA.crt",
},
},
},
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
Eventually(
func(g Gomega) {
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
g.Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
g.Expect(runnerSetList.Items).To(HaveLen(1), "expected 1 EphemeralRunnerSet to be created")
runnerSet := &runnerSetList.Items[0]
g.Expect(runnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS).NotTo(BeNil(), "expected EphemeralRunnerSpec.GitHubServerTLS to be set")
g.Expect(runnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS).To(BeEquivalentTo(autoscalingRunnerSet.Spec.GitHubServerTLS), "EphemeralRunnerSpec does not have TLS config")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
})
})
var _ = Describe("Test external permissions cleanup", Ordered, func() {
var originalBuildVersion string
buildVersion := "0.1.0"
BeforeAll(func() {
originalBuildVersion = build.Version
build.Version = buildVersion
})
AfterAll(func() {
build.Version = originalBuildVersion
})
It("Should clean up kubernetes mode permissions", func() {
ctx := context.Background()
autoscalingNS, mgr := createNamespace(GinkgoT(), k8sClient)
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
"app.kubernetes.io/name": "gha-runner-scale-set",
LabelKeyKubernetesVersion: buildVersion,
},
Annotations: map[string]string{
AnnotationKeyKubernetesModeRoleBindingName: "kube-mode-role-binding",
AnnotationKeyKubernetesModeRoleName: "kube-mode-role",
AnnotationKeyKubernetesModeServiceAccountName: "kube-mode-service-account",
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
role := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeRoleName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
}
err = k8sClient.Create(ctx, role)
Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes mode role")
serviceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeServiceAccountName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
}
err = k8sClient.Create(ctx, serviceAccount)
Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes mode service account")
roleBinding := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeRoleBindingName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: serviceAccount.Name,
Namespace: serviceAccount.Namespace,
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: rbacv1.GroupName,
// Kind is the type of resource being referenced
Kind: "Role",
Name: role.Name,
},
}
err = k8sClient.Create(ctx, roleBinding)
Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes mode role binding")
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
Eventually(
func() (string, error) {
created := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
if err != nil {
return "", err
}
if len(created.Finalizers) == 0 {
return "", nil
}
return created.Finalizers[0], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
err = k8sClient.Delete(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to delete autoscaling runner set")
err = k8sClient.Delete(ctx, roleBinding)
Expect(err).NotTo(HaveOccurred(), "failed to delete kubernetes mode role binding")
err = k8sClient.Delete(ctx, role)
Expect(err).NotTo(HaveOccurred(), "failed to delete kubernetes mode role")
err = k8sClient.Delete(ctx, serviceAccount)
Expect(err).NotTo(HaveOccurred(), "failed to delete kubernetes mode service account")
Eventually(
func() bool {
r := new(rbacv1.RoleBinding)
err := k8sClient.Get(ctx, types.NamespacedName{
Name: roleBinding.Name,
Namespace: roleBinding.Namespace,
}, r)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "Expected role binding to be cleaned up")
Eventually(
func() bool {
r := new(rbacv1.Role)
err := k8sClient.Get(ctx, types.NamespacedName{
Name: role.Name,
Namespace: role.Namespace,
}, r)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "Expected role to be cleaned up")
})
It("Should clean up manager permissions and no-permission service account", func() {
ctx := context.Background()
autoscalingNS, mgr := createNamespace(GinkgoT(), k8sClient)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
"app.kubernetes.io/name": "gha-runner-scale-set",
LabelKeyKubernetesVersion: buildVersion,
},
Annotations: map[string]string{
AnnotationKeyManagerRoleName: "manager-role",
AnnotationKeyManagerRoleBindingName: "manager-role-binding",
AnnotationKeyGitHubSecretName: "gh-secret-name",
AnnotationKeyNoPermissionServiceAccountName: "no-permission-sa",
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyGitHubSecretName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
Data: map[string][]byte{
"github_token": []byte(defaultGitHubToken),
},
}
err = k8sClient.Create(context.Background(), secret)
Expect(err).NotTo(HaveOccurred(), "failed to create github secret")
autoscalingRunnerSet.Spec.GitHubConfigSecret = secret.Name
role := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyManagerRoleName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
}
err = k8sClient.Create(ctx, role)
Expect(err).NotTo(HaveOccurred(), "failed to create manager role")
roleBinding := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyManagerRoleBindingName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
RoleRef: rbacv1.RoleRef{
APIGroup: rbacv1.GroupName,
Kind: "Role",
Name: role.Name,
},
}
err = k8sClient.Create(ctx, roleBinding)
Expect(err).NotTo(HaveOccurred(), "failed to create manager role binding")
noPermissionServiceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyNoPermissionServiceAccountName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
}
err = k8sClient.Create(ctx, noPermissionServiceAccount)
Expect(err).NotTo(HaveOccurred(), "failed to create no permission service account")
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
Eventually(
func() (string, error) {
created := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
if err != nil {
return "", err
}
if len(created.Finalizers) == 0 {
return "", nil
}
return created.Finalizers[0], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
err = k8sClient.Delete(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to delete autoscaling runner set")
err = k8sClient.Delete(ctx, noPermissionServiceAccount)
Expect(err).NotTo(HaveOccurred(), "failed to delete no permission service account")
err = k8sClient.Delete(ctx, secret)
Expect(err).NotTo(HaveOccurred(), "failed to delete GitHub secret")
err = k8sClient.Delete(ctx, roleBinding)
Expect(err).NotTo(HaveOccurred(), "failed to delete manager role binding")
err = k8sClient.Delete(ctx, role)
Expect(err).NotTo(HaveOccurred(), "failed to delete manager role")
Eventually(
func() bool {
r := new(corev1.ServiceAccount)
err := k8sClient.Get(
ctx,
types.NamespacedName{
Name: noPermissionServiceAccount.Name,
Namespace: noPermissionServiceAccount.Namespace,
},
r,
)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "Expected no permission service account to be cleaned up")
Eventually(
func() bool {
r := new(corev1.Secret)
err := k8sClient.Get(ctx, types.NamespacedName{
Name: secret.Name,
Namespace: secret.Namespace,
}, r)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "Expected role binding to be cleaned up")
Eventually(
func() bool {
r := new(rbacv1.RoleBinding)
err := k8sClient.Get(ctx, types.NamespacedName{
Name: roleBinding.Name,
Namespace: roleBinding.Namespace,
}, r)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "Expected role binding to be cleaned up")
Eventually(
func() bool {
r := new(rbacv1.Role)
err := k8sClient.Get(
ctx,
types.NamespacedName{
Name: role.Name,
Namespace: role.Namespace,
},
r,
)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "Expected role to be cleaned up")
})
})
var _ = Describe("Test resource version and build version mismatch", func() {
It("Should delete and recreate the autoscaling runner set to match the build version", func() {
ctx := context.Background()
autoscalingNS, mgr := createNamespace(GinkgoT(), k8sClient)
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
originalVersion := build.Version
defer func() {
build.Version = originalVersion
}()
build.Version = "0.2.0"
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
"app.kubernetes.io/name": "gha-runner-scale-set",
"app.kubernetes.io/version": "0.1.0",
},
Annotations: map[string]string{
AnnotationKeyKubernetesModeRoleBindingName: "kube-mode-role-binding",
AnnotationKeyKubernetesModeRoleName: "kube-mode-role",
AnnotationKeyKubernetesModeServiceAccountName: "kube-mode-service-account",
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
// create autoscaling runner set before starting a manager
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred())
startManagers(GinkgoT(), mgr)
Eventually(
func() bool {
ars := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, types.NamespacedName{Namespace: autoscalingRunnerSet.Namespace, Name: autoscalingRunnerSet.Name}, ars)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue())
})
})
var _ = Describe("Test AutoscalingRunnerSet with a stale runner scale set", Ordered, func() {
const staleRunnerScaleSetID = 42
const freshRunnerScaleSetID = 1
var originalBuildVersion string
buildVersion := "0.1.0"
BeforeAll(func() {
originalBuildVersion = build.Version
build.Version = buildVersion
})
AfterAll(func() {
build.Version = originalBuildVersion
})
newAutoscalingRunnerSet := func(namespace, configSecretName string, annotations map[string]string) *v1alpha1.AutoscalingRunnerSet {
min := 1
max := 10
return &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: namespace,
Labels: map[string]string{LabelKeyKubernetesVersion: buildVersion},
Annotations: annotations,
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecretName,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
}
registeredAnnotations := func(runnerScaleSetID int) map[string]string {
return map[string]string{
runnerScaleSetIDAnnotationKey: strconv.Itoa(runnerScaleSetID),
AnnotationKeyGitHubRunnerScaleSetName: "test-asrs",
AnnotationKeyGitHubRunnerGroupName: "testgroup",
}
}
Context("When the recorded runner scale set no longer exists on the Actions service", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, runnerScaleSetID int) (*scaleset.RunnerScaleSet, error) {
if runnerScaleSetID == staleRunnerScaleSetID {
return nil, scaleset.NotFoundError
}
return &scaleset.RunnerScaleSet{ID: runnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
}),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: freshRunnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithDeleteRunnerScaleSet(nil),
),
),
)),
},
}
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, registeredAnnotations(staleRunnerScaleSetID))
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
})
It("registers the runner scale set again and creates the listener with the new ID", func() {
Eventually(
func() (string, error) {
updated := new(v1alpha1.AutoscalingRunnerSet)
if err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated); err != nil {
return "", err
}
return updated.Annotations[runnerScaleSetIDAnnotationKey], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(strconv.Itoa(freshRunnerScaleSetID)), "the stale runner scale set ID should be replaced")
Eventually(
func() (int, error) {
listener := new(v1alpha1.AutoscalingListener)
if err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener); err != nil {
return 0, err
}
return listener.Spec.RunnerScaleSetID, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(freshRunnerScaleSetID), "the listener should never be created with the stale runner scale set ID")
})
})
Context("When the Actions service cannot confirm whether the runner scale set exists", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
var createCalls atomic.Int64
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
createCalls.Store(0)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, _ int) (*scaleset.RunnerScaleSet, error) {
return nil, fmt.Errorf("actions service is unavailable")
}),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSetFunc(func(_ context.Context, rs *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) {
createCalls.Add(1)
return &scaleset.RunnerScaleSet{ID: freshRunnerScaleSetID, Name: rs.Name, RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
}),
scalefake.WithDeleteRunnerScaleSet(nil),
),
),
)),
},
}
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, registeredAnnotations(staleRunnerScaleSetID))
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
})
It("leaves the runner scale set alone instead of registering a second one", func() {
Consistently(
func() (string, error) {
updated := new(v1alpha1.AutoscalingRunnerSet)
if err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated); err != nil {
return "", err
}
return updated.Annotations[runnerScaleSetIDAnnotationKey], nil
},
"3s",
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(strconv.Itoa(staleRunnerScaleSetID)), "a transient failure should never re-register the runner scale set")
Expect(createCalls.Load()).To(BeEquivalentTo(0), "no runner scale set should be created while the service is unavailable")
})
})
Context("When the runner scale set is already deleted from the Actions service", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, runnerScaleSetID int) (*scaleset.RunnerScaleSet, error) {
return &scaleset.RunnerScaleSet{ID: runnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
}),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: freshRunnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithDeleteRunnerScaleSet(scaleset.NotFoundError),
),
),
)),
},
}
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, nil)
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
})
It("does not leave the AutoscalingRunnerSet stuck in Terminating", func() {
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
Expect(k8sClient.Delete(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to delete AutoScalingRunnerSet")
Eventually(
func() error {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingRunnerSet))
if err != nil && errors.IsNotFound(err) {
return nil
}
return fmt.Errorf("AutoScalingRunnerSet is not deleted")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "the finalizer should be removed even though the runner scale set is already gone")
})
})
})
// testHoldFinalizer keeps an AutoscalingListener around after it has been
// deleted, so a test can observe the window during which the scale set has no
// usable listener. The AutoscalingListener controller is not running in this
// suite, so nothing else adds or removes finalizers on these objects.
const testHoldFinalizer = "arc.test/hold-deletion"
func blockDeletion(listener *v1alpha1.AutoscalingListener) {
GinkgoHelper()
current := new(v1alpha1.AutoscalingListener)
Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(listener), current)).To(Succeed(), "failed to get listener to block its deletion")
original := current.DeepCopy()
Expect(controllerutil.AddFinalizer(current, testHoldFinalizer)).To(BeTrue(), "listener should not already hold the test finalizer")
Expect(k8sClient.Patch(context.Background(), current, client.MergeFrom(original))).To(Succeed(), "failed to add the test finalizer to the listener")
}
// unblockDeletion is idempotent so it can be deferred as a safety net and still
// be called explicitly at the point a test wants the rebuild to proceed.
func unblockDeletion(listener *v1alpha1.AutoscalingListener) {
GinkgoHelper()
current := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(listener), current)
if errors.IsNotFound(err) {
return
}
Expect(err).NotTo(HaveOccurred(), "failed to get listener to unblock its deletion")
original := current.DeepCopy()
if !controllerutil.RemoveFinalizer(current, testHoldFinalizer) {
return
}
Expect(k8sClient.Patch(context.Background(), current, client.MergeFrom(original))).To(Succeed(), "failed to remove the test finalizer from the listener")
}
Reference in New Issue View Git Blame Copy Permalink