public_git/actions-runner-controller
1
0
Fork 0
mirror of https://github.com/actions-runner-controller/actions-runner-controller.git synced 2026-10-01 05:54:31 +02:00
Code Issues Projects Releases Wiki Activity
Files
e481f69cffcf672215090726075d7b9bf7f20eaf
actions-runner-controller/controllers/actions.github.com/autoscalingrunnerset_controller_test.go
T
Nikola Jokic e481f69cff Replace integrity-hash annotations with explicit API state fields 
The `actions.github.com/integrity-hash` annotation was used as an opaque
fingerprint to detect spec drift across AutoscalingRunnerSet,
EphemeralRunnerSet and the listener resources. Hashes are brittle: they
change whenever unrelated serialization details change, they are invisible
to users, and they are not restart-safe. FNV-32a also carries a real
collision risk, where the consequence is an update silently never applied.

Replace it with explicit, typed state:

- `AutoscalingRunnerSetStatus.ObservedGeneration` drives the Pending phase
  transition via `metadata.generation` instead of an annotation hash.
- `EphemeralRunnerSetSpec.ActionableRevision` and
  `EphemeralRunnerSetStatus.AppliedActionableRevision` form a restart-safe
  applied marker. The revision is bumped by the AutoscalingRunnerSet
  controller when `EphemeralRunnerSpec` changes, and only advanced in status
  after idle/pending runner cleanup succeeds.
- `EphemeralRunnerSetStatus.FinishedRunnerCleanupPatchID` records the
  listener patch ID for which finished runners were reaped, so scale-up is
  suppressed until the listener publishes a fresh desired state. This
  prevents creating a replacement runner for a job that already completed.
- Listener pod recreation compares pod specs semantically instead of
  comparing hash annotations.

Drift detection uses `apiequality.Semantic`, not `cmp` or `reflect`:

- `Semantic.DeepEqual` for the EphemeralRunnerSpec. Most PodSpec collection
  fields carry `omitempty`, so a template containing an explicitly empty
  value (`env: []`) is dropped when the EphemeralRunnerSet is written and
  reads back as nil. A strict comparison reports drift on every reconcile,
  bumping ActionableRevision each time and deleting every idle and pending
  runner, forever. Semantic treats nil and empty as equal, understands
  resource.Quantity, and cannot panic on unexported fields the way cmp can.
  It is also roughly six times cheaper than cmp.Equal on a realistic spec.
- `Semantic.DeepDerivative` for the listener pod, because the live pod
  carries many fields the desired pod never sets (nodeName, dnsPolicy,
  default tolerations, the kube-api-access volume, ...). DeepEqual there
  would spin in a delete/create loop. Container port length is checked
  separately, since ports come from the --listener-metrics-addr flag rather
  than from a resource, so disabling metrics would otherwise leave the port
  on the pod forever.

Drift detection is deliberately not short-circuited on
metadata.generation. Re-registration changes the runner scale set ID
through an annotation, and metadata changes do not bump generation, so a
generation-based shortcut would leave the EphemeralRunnerSet pointing at a
scale set that no longer exists. The measured saving did not justify the
risk.

Additionally:

- Count deleting runners toward the scale-up total so terminating runners
  are not double-replaced.
- Cleanup of finished runners is no longer deferred; failures now surface as
  reconcile errors instead of being logged and swallowed.
- Status patches for the new fields use `RetryOnConflict` against a freshly
  read object.
- Keep merging EphemeralRunnerSet annotations and labels rather than
  overwriting them, so metadata applied by admission webhooks or other
  controllers is preserved. Drift detection compares against the merge
  result so foreign keys cannot cause a permanent patch loop.
- Add unit tests and benchmarks for both drift checks, including a guard
  that fails if the listener comparison is ever tightened to DeepEqual.
- Cover the re-registration path, which previously had no assertion that
  the new runner scale set ID reaches the EphemeralRunnerSet at all.
2026-09-07 22:08:17 +02:00

2581 lines
97 KiB
Go
Raw Blame History

package actionsgithubcom
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"sync"
"sync/atomic"
"time"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
logf "sigs.k8s.io/controller-runtime/pkg/log"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
"github.com/actions/actions-runner-controller/build"
"github.com/actions/actions-runner-controller/controllers/actions.github.com/multiclient"
scalefake "github.com/actions/actions-runner-controller/controllers/actions.github.com/multiclient/fake"
"github.com/actions/actions-runner-controller/controllers/actions.github.com/secretresolver"
"github.com/actions/scaleset"
)
const (
autoscalingRunnerSetTestTimeout = time.Second * 20
autoscalingRunnerSetTestInterval = time.Millisecond * 250
)
var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() {
var ctx context.Context
var mgr ctrl.Manager
var controller *AutoscalingRunnerSetReconciler
var autoscalingNS *corev1.Namespace
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
var configSecret *corev1.Secret
var resourceCache *ResourceCache
var originalBuildVersion string
buildVersion := "0.1.0"
BeforeAll(func() {
originalBuildVersion = build.Version
build.Version = buildVersion
})
AfterAll(func() {
build.Version = originalBuildVersion
})
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
// Track runner group mappings for dynamic responses
runnerGroupMap := map[int]string{1: "testgroup"} // ID -> Name mapping
runnerGroupMapLock := &sync.RWMutex{} // Thread-safe access
resourceCache = newTestResourceCache()
controller = &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: resourceCache,
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByNameFunc(func(ctx context.Context, groupName string) (*scaleset.RunnerGroup, error) {
// Support both "testgroup" and "testgroup2"
// Update the mapping when a new group is requested
runnerGroupMapLock.Lock()
runnerGroupMap[1] = groupName
runnerGroupMapLock.Unlock()
return &scaleset.RunnerGroup{ID: 1, Name: groupName}, nil
}),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSetFunc(func(ctx context.Context, rs *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) {
// Return a RunnerScaleSet with name matching the requesting ARS
runnerGroupMapLock.RLock()
groupName := runnerGroupMap[rs.RunnerGroupID]
runnerGroupMapLock.RUnlock()
return &scaleset.RunnerScaleSet{ID: 1, Name: rs.Name, RunnerGroupID: rs.RunnerGroupID, RunnerGroupName: groupName}, nil
}),
scalefake.WithUpdateRunnerScaleSetFunc(func(ctx context.Context, scaleSetID int, rs *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) {
// Return a RunnerScaleSet with the group name corresponding to the runner group ID
runnerGroupMapLock.RLock()
groupName := runnerGroupMap[rs.RunnerGroupID]
runnerGroupMapLock.RUnlock()
return &scaleset.RunnerScaleSet{ID: 1, Name: rs.Name, RunnerGroupID: rs.RunnerGroupID, RunnerGroupName: groupName}, nil
}),
scalefake.WithDeleteRunnerScaleSet(nil),
),
),
)),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
min := 1
max := 10
autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
"arc.test/label-drift": "initial",
"arc.test/spec-update-label": "initial",
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
EphemeralRunnerSetMetadata: &v1alpha1.ResourceMeta{
Annotations: map[string]string{
"arc.test/metadata-annotation": "initial",
},
},
EphemeralRunnerMetadata: &v1alpha1.ResourceMeta{
Labels: map[string]string{
"arc.test/runner-metadata-label": "initial",
},
Annotations: map[string]string{
"arc.test/runner-metadata-annotation": "initial",
},
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
startManagers(GinkgoT(), mgr)
})
Context("When creating a new AutoScalingRunnerSet", func() {
It("It should create/add all required resources for a new AutoScalingRunnerSet (finalizer, runnerscaleset, ephemeralrunnerset, listener)", func() {
// Check if finalizer is added
created := new(v1alpha1.AutoscalingRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
if err != nil {
return "", err
}
if len(created.Finalizers) == 0 {
return "", nil
}
return created.Finalizers[0], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
// Check if runner scale set is created on service
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
if err != nil {
return "", err
}
if _, ok := created.Annotations[runnerScaleSetIDAnnotationKey]; !ok {
return "", nil
}
if _, ok := created.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
return "", nil
}
return fmt.Sprintf(
"%s_%s",
created.Annotations[runnerScaleSetIDAnnotationKey],
created.Annotations[AnnotationKeyGitHubRunnerGroupName],
), nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(
BeEquivalentTo("1_testgroup"),
"RunnerScaleSet should be created/fetched and update the AutoScalingRunnerSet's annotation",
)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
if err != nil {
return "", err
}
if _, ok := created.Labels[LabelKeyGitHubOrganization]; !ok {
return "", nil
}
if _, ok := created.Labels[LabelKeyGitHubRepository]; !ok {
return "", nil
}
return fmt.Sprintf("%s/%s", created.Labels[LabelKeyGitHubOrganization], created.Labels[LabelKeyGitHubRepository]), nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo("owner/repo"), "RunnerScaleSet should be created/fetched and update the AutoScalingRunnerSet's label")
// Check if ephemeral runner set is created
Eventually(
func() (int, error) {
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
if err != nil {
return 0, err
}
return len(runnerSetList.Items), nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created")
// Check if listener is created
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
Eventually(
func() bool {
return resourceCacheStateHasMainObjectEntries(resourceCache.ephemeralRunnerSet, created) &&
resourceCacheStateHasMainObjectEntries(resourceCache.autoscalingListener, created)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "AutoScalingRunnerSet EphemeralRunnerSet and AutoScalingListener resources should be cached after reconciliation")
// Check if status is updated
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
Expect(len(runnerSetList.Items)).To(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created")
})
})
Context("When deleting a new AutoScalingRunnerSet", func() {
It("It should cleanup all resources for a deleting AutoScalingRunnerSet before removing it", func() {
// Wait till the listener is created
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
created := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
Expect(err).NotTo(HaveOccurred(), "failed to get AutoScalingRunnerSet")
Eventually(
func() bool {
return resourceCacheStateHasMainObjectEntries(resourceCache.ephemeralRunnerSet, created) &&
resourceCacheStateHasMainObjectEntries(resourceCache.autoscalingListener, created)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "AutoScalingRunnerSet EphemeralRunnerSet and AutoScalingListener resources should be cached before deletion")
// Delete the AutoScalingRunnerSet
err = k8sClient.Delete(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to delete AutoScalingRunnerSet")
// Check if the listener is deleted
Eventually(
func() error {
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
if err != nil && errors.IsNotFound(err) {
return nil
}
return fmt.Errorf("listener is not deleted")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be deleted")
// Check if all the EphemeralRunnerSet is deleted
Eventually(
func() error {
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
if err != nil {
return err
}
if len(runnerSetList.Items) != 0 {
return fmt.Errorf("EphemeralRunnerSet is not deleted, count=%v", len(runnerSetList.Items))
}
return nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "All EphemeralRunnerSet should be deleted")
// Check if the AutoScalingRunnerSet is deleted
Eventually(
func() error {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingRunnerSet))
if err != nil && errors.IsNotFound(err) {
return nil
}
return fmt.Errorf("AutoScalingRunnerSet is not deleted")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "AutoScalingRunnerSet should be deleted")
Eventually(
func() bool {
return resourceCacheStateHasMainObjectEntries(resourceCache.ephemeralRunnerSet, created) ||
resourceCacheStateHasMainObjectEntries(resourceCache.autoscalingListener, created)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeFalse(), "AutoScalingRunnerSet EphemeralRunnerSet and AutoScalingListener resources should be removed from cache after deletion")
})
})
It("should not churn listener when already referencing latest ERS (no-op stability)", func() {
min := 1
max := 10
testARSName := "test-asrs-no-churn"
testARSNamespace := autoscalingNS.Name
testARS := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: testARSName,
Namespace: testARSNamespace,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, testARS)
Expect(err).NotTo(HaveOccurred(), "failed to create test AutoScalingRunnerSet")
var latestERSName string
Eventually(
func() (string, error) {
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(testARSNamespace))
if err != nil {
return "", err
}
// Filter to only ERS owned by our test ARS
var ownedByTestARS []v1alpha1.EphemeralRunnerSet
for _, ers := range runnerSetList.Items {
for _, owner := range ers.OwnerReferences {
if owner.UID == testARS.UID {
ownedByTestARS = append(ownedByTestARS, ers)
break
}
}
}
if len(ownedByTestARS) != 1 {
return "", fmt.Errorf("expected 1 EphemeralRunnerSet owned by test ARS, got %d", len(ownedByTestARS))
}
return ownedByTestARS[0].Name, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Not(BeEmpty()), "ERS should be created")
// Capture the latest ERS name
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err = k8sClient.List(ctx, runnerSetList, client.InNamespace(testARSNamespace))
Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
var ownedByTestARS []v1alpha1.EphemeralRunnerSet
for _, ers := range runnerSetList.Items {
for _, owner := range ers.OwnerReferences {
if owner.UID == testARS.UID {
ownedByTestARS = append(ownedByTestARS, ers)
break
}
}
}
Expect(len(ownedByTestARS)).To(Equal(1), "should have exactly 1 EphemeralRunnerSet owned by test ARS")
latestERSName = ownedByTestARS[0].Name
listener := new(v1alpha1.AutoscalingListener)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, listener)
if err != nil {
return "", err
}
return listener.Spec.EphemeralRunnerSetName, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Equal(latestERSName), "listener should reference the latest ERS")
// Capture listener identity (UID and ResourceVersion)
err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, listener)
Expect(err).NotTo(HaveOccurred(), "failed to get listener")
originalUID := listener.UID
originalResourceVersion := listener.ResourceVersion
Consistently(
func() (types.UID, error) {
currentListener := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, currentListener)
if err != nil {
return "", err
}
return currentListener.UID, nil
},
time.Second*5,
autoscalingRunnerSetTestInterval,
).Should(Equal(originalUID), "listener UID should remain unchanged (no recreation)")
Consistently(
func() (string, error) {
currentListener := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, currentListener)
if err != nil {
return "", err
}
return currentListener.ResourceVersion, nil
},
time.Second*5,
autoscalingRunnerSetTestInterval,
).Should(Equal(originalResourceVersion), "listener ResourceVersion should remain unchanged (no updates)")
err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, listener)
Expect(err).NotTo(HaveOccurred(), "failed to get listener")
Expect(listener.Spec.EphemeralRunnerSetName).To(Equal(latestERSName), "listener should still reference latest ERS")
})
Context("When updating a new AutoScalingRunnerSet", func() {
It("updates EphemeralRunnerSet when the runner image changes without touching the Listener", func() {
listener := new(v1alpha1.AutoscalingListener)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
originalListenerUID := listener.UID
originalListenerResourceVersion := listener.ResourceVersion
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "EphemeralRunnerSet should be created")
originalRunnerSetUID := runnerSet.UID
originalActionableRevision := runnerSet.Spec.ActionableRevision
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated"
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should be updated in place")
g.Expect(current.Spec.ActionableRevision).To(BeNumerically(">", originalActionableRevision), "ActionableRevision should increment for actionable spec changes")
g.Expect(current.Spec.EphemeralRunnerSpec.PodTemplateSpec.Spec.Containers[0].Image).To(Equal("ghcr.io/actions/runner:updated"))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
Consistently(
func(g Gomega) {
current := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener")
g.Expect(current.UID).To(Equal(originalListenerUID), "Listener should not be recreated")
g.Expect(current.ResourceVersion).To(Equal(originalListenerResourceVersion), "Listener should not be updated")
},
time.Second*5,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
It("recreates only the Listener when max runners changes", func() {
listener := new(v1alpha1.AutoscalingListener)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
originalListenerUID := listener.UID
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "EphemeralRunnerSet should be created")
originalRunnerSetUID := runnerSet.UID
originalActionableRevision := runnerSet.Spec.ActionableRevision
patched := autoscalingRunnerSet.DeepCopy()
max := 20
patched.Spec.MaxRunners = &max
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener")
g.Expect(current.UID).NotTo(Equal(originalListenerUID), "Listener should be recreated")
g.Expect(current.Spec.MaxRunners).To(Equal(max))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
Consistently(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should not be recreated")
g.Expect(current.Spec.ActionableRevision).To(Equal(originalActionableRevision), "ActionableRevision should not change for non-actionable updates")
},
time.Second*5,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
It("updates EphemeralRunnerSet labels when only AutoScalingRunnerSet labels change", func() {
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
if err != nil {
return "", err
}
return runnerSet.Labels["arc.test/label-drift"], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined label")
patched := autoscalingRunnerSet.DeepCopy()
patched.Labels["arc.test/label-drift"] = "updated"
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet labels")
Eventually(
func() (string, error) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
if err != nil {
return "", err
}
return current.Labels["arc.test/label-drift"], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Equal("updated"), "EphemeralRunnerSet should be patched with label-only drift")
})
It("updates EphemeralRunnerSet annotations when only EphemeralRunnerSet metadata annotations change", func() {
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
if err != nil {
return "", err
}
return runnerSet.Annotations["arc.test/metadata-annotation"], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined annotation")
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated"
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/new-metadata-annotation"] = "added"
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunnerSet metadata")
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated"))
g.Expect(current.Annotations["arc.test/new-metadata-annotation"]).To(Equal("added"))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "EphemeralRunnerSet should be patched with annotation-only metadata drift")
})
It("preserves foreign annotations and labels on the EphemeralRunnerSet", func() {
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
if err != nil {
return "", err
}
return runnerSet.Annotations["arc.test/metadata-annotation"], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined annotation")
// Simulate a third party (admission webhook, another controller, a user)
// adding metadata the AutoscalingRunnerSet knows nothing about.
foreign := runnerSet.DeepCopy()
foreign.Annotations["thirdparty.example.com/injected"] = "keep-me"
foreign.Labels["thirdparty.example.com/injected"] = "keep-me"
err := k8sClient.Patch(ctx, foreign, client.MergeFrom(runnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to inject foreign metadata on EphemeralRunnerSet")
// Force the controller through the metadata reconciliation path.
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated"
err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunnerSet metadata")
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated"))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "desired annotation should still propagate")
// The foreign keys must survive, and the resourceVersion must settle
// instead of the controller fighting over the annotation set forever.
settled := new(v1alpha1.EphemeralRunnerSet)
Expect(k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, settled)).To(Succeed())
lastResourceVersion := settled.ResourceVersion
Consistently(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.Annotations).To(HaveKeyWithValue("thirdparty.example.com/injected", "keep-me"), "foreign annotation must not be stripped")
g.Expect(current.Labels).To(HaveKeyWithValue("thirdparty.example.com/injected", "keep-me"), "foreign label must not be stripped")
g.Expect(current.ResourceVersion).To(Equal(lastResourceVersion), "EphemeralRunnerSet should stop being patched once metadata has converged")
},
3*time.Second,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
It("updates EphemeralRunnerSet runner metadata when only EphemeralRunner metadata changes", func() {
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func(g Gomega) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(runnerSet.Spec.EphemeralRunnerMetadata).NotTo(BeNil())
g.Expect(runnerSet.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"]).To(Equal("initial"))
g.Expect(runnerSet.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"]).To(Equal("initial"))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"] = "updated"
patched.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"] = "updated"
patched.Spec.EphemeralRunnerMetadata.Annotations["arc.test/new-runner-metadata-annotation"] = "added"
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunner metadata")
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.Spec.EphemeralRunnerMetadata).NotTo(BeNil())
g.Expect(current.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"]).To(Equal("updated"))
g.Expect(current.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"]).To(Equal("updated"))
g.Expect(current.Spec.EphemeralRunnerMetadata.Annotations["arc.test/new-runner-metadata-annotation"]).To(Equal("added"))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
It("propagates latest labels while updating EphemeralRunnerSet spec", func() {
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
if err != nil {
return "", err
}
return runnerSet.Labels["arc.test/spec-update-label"], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined label")
patched := autoscalingRunnerSet.DeepCopy()
patched.Labels["arc.test/spec-update-label"] = "updated"
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated"
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/new-spec-update-annotation"] = "added"
patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated-with-label"
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet spec and labels")
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.Spec.EphemeralRunnerSpec.PodTemplateSpec.Spec.Containers[0].Image).To(Equal("ghcr.io/actions/runner:updated-with-label"))
g.Expect(current.Labels["arc.test/spec-update-label"]).To(Equal("updated"))
g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated"))
g.Expect(current.Annotations["arc.test/new-spec-update-annotation"]).To(Equal("added"))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
It("updates EphemeralRunnerSet and Listener when the GitHub config secret changes", func() {
updatedSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "github-config-secret-updated",
Namespace: autoscalingRunnerSet.Namespace,
},
Data: map[string][]byte{
"github_token": []byte(defaultGitHubToken),
},
}
err := k8sClient.Create(ctx, updatedSecret)
Expect(err).NotTo(HaveOccurred(), "failed to create updated GitHub config secret")
listener := new(v1alpha1.AutoscalingListener)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
originalListenerUID := listener.UID
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "EphemeralRunnerSet should be created")
originalRunnerSetUID := runnerSet.UID
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.GitHubConfigSecret = updatedSecret.Name
err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get AutoScalingRunnerSet")
g.Expect(current.Spec.GitHubConfigSecret).To(Equal(updatedSecret.Name))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should be updated in place")
g.Expect(current.Spec.EphemeralRunnerSpec.GitHubConfigSecret).To(Equal(updatedSecret.Name))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)
g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener")
g.Expect(current.UID).NotTo(Equal(originalListenerUID), "Listener should be recreated")
g.Expect(current.Spec.GitHubConfigSecret).To(Equal(updatedSecret.Name))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
It("It should update RunnerScaleSet's runner group on service when it changes", func() {
updated := new(v1alpha1.AutoscalingRunnerSet)
// Wait till the listener is created
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
patched := autoscalingRunnerSet.DeepCopy()
patched.Spec.RunnerGroup = "testgroup2"
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
// Check if AutoScalingRunnerSet has the new runner group in its annotation
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
if err != nil {
return "", err
}
if _, ok := updated.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
return "", nil
}
return updated.Annotations[AnnotationKeyGitHubRunnerGroupName], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the new runner group in its annotation")
// delete the annotation and it should be re-added
patched = autoscalingRunnerSet.DeepCopy()
delete(patched.Annotations, AnnotationKeyGitHubRunnerGroupName)
err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
// Check if AutoScalingRunnerSet still has the runner group in its annotation
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
if err != nil {
return "", err
}
if _, ok := updated.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
return "", nil
}
return updated.Annotations[AnnotationKeyGitHubRunnerGroupName], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the runner group in its annotation")
})
})
It("Should update Status on EphemeralRunnerSet status Update", func() {
ars := new(v1alpha1.AutoscalingRunnerSet)
Eventually(
func() (bool, error) {
err := k8sClient.Get(
ctx,
client.ObjectKey{
Name: autoscalingRunnerSet.Name,
Namespace: autoscalingRunnerSet.Namespace,
},
ars,
)
if err != nil {
return false, err
}
return true, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "AutoscalingRunnerSet should be created")
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
Eventually(
func() (int, error) {
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(ars.Namespace))
if err != nil {
return 0, err
}
return len(runnerSetList.Items), nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(1), "Failed to fetch runner set list")
runnerSet := runnerSetList.Items[0]
statusUpdate := runnerSet.DeepCopy()
statusUpdate.Status.Phase = v1alpha1.EphemeralRunnerSetPhaseRunning
err := k8sClient.Status().Patch(ctx, statusUpdate, client.MergeFrom(&runnerSet))
Expect(err).NotTo(HaveOccurred(), "Failed to patch runner set status")
Eventually(
func() (v1alpha1.AutoscalingRunnerSetStatus, error) {
updated := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
if err != nil {
return v1alpha1.AutoscalingRunnerSetStatus{}, fmt.Errorf("failed to get AutoScalingRunnerSet: %w", err)
}
return updated.Status, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(SatisfyAll(
WithTransform(func(s v1alpha1.AutoscalingRunnerSetStatus) v1alpha1.AutoscalingRunnerSetPhase { return s.Phase }, Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning)),
WithTransform(func(s v1alpha1.AutoscalingRunnerSetStatus) int64 { return s.ObservedGeneration }, BeNumerically(">=", ars.Generation)),
), "AutoScalingRunnerSet status should be updated")
})
})
var _ = Describe("Test AutoScalingController updates", Ordered, func() {
var originalBuildVersion string
buildVersion := "0.1.0"
BeforeAll(func() {
originalBuildVersion = build.Version
build.Version = buildVersion
})
AfterAll(func() {
build.Version = originalBuildVersion
})
Context("Creating autoscaling runner set with RunnerScaleSetName set", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
var configSecret *corev1.Secret
BeforeEach(func() {
originalBuildVersion = build.Version
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
multiClient := scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGenerateJitRunnerConfig(
&scaleset.RunnerScaleSetJitRunnerConfig{
Runner: &scaleset.RunnerReference{ID: 1, Name: "test-runner"},
EncodedJITConfig: "fake-jit-config",
},
nil,
),
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "testset", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithUpdateRunnerScaleSet(
&scaleset.RunnerScaleSet{
ID: 1,
Name: "testset_update",
RunnerGroupID: 1,
RunnerGroupName: "testgroup",
Labels: []scaleset.Label{{Type: "test", Name: "test"}},
RunnerSetting: scaleset.RunnerSetting{},
CreatedOn: time.Now(),
RunnerJitConfigURL: "test.test.test",
Statistics: nil,
},
nil,
),
),
),
)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), multiClient),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
})
It("It should be create AutoScalingRunnerSet and has annotation for the RunnerScaleSetName", func() {
min := 1
max := 10
autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerScaleSetName: "testset",
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
// Wait for the AutoScalingRunnerSet to be created with right annotation
ars := new(v1alpha1.AutoscalingRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
if err != nil {
return "", err
}
if val, ok := ars.Annotations[AnnotationKeyGitHubRunnerScaleSetName]; ok {
return val, nil
}
return "", nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(autoscalingRunnerSet.Spec.RunnerScaleSetName), "AutoScalingRunnerSet should have annotation for the RunnerScaleSetName")
update := autoscalingRunnerSet.DeepCopy()
update.Spec.RunnerScaleSetName = "testset_update"
err = k8sClient.Patch(ctx, update, client.MergeFrom(autoscalingRunnerSet))
Expect(err).NotTo(HaveOccurred(), "failed to update AutoScalingRunnerSet")
// Wait for the AutoScalingRunnerSet to be updated with right annotation
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
if err != nil {
return "", err
}
if val, ok := ars.Annotations[AnnotationKeyGitHubRunnerScaleSetName]; ok {
return val, nil
}
return "", nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(update.Spec.RunnerScaleSetName), "AutoScalingRunnerSet should have a updated annotation for the RunnerScaleSetName")
})
})
})
var _ = Describe("Test AutoscalingController creation failures", Ordered, func() {
var originalBuildVersion string
buildVersion := "0.1.0"
BeforeAll(func() {
originalBuildVersion = build.Version
build.Version = buildVersion
})
AfterAll(func() {
build.Version = originalBuildVersion
})
Context("When autoscaling runner set creation fails on the client", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
})
It("It should be able to clean up if annotation related to scale set id does not exist", func() {
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
// wait for the finalizer to be added
ars := new(v1alpha1.AutoscalingRunnerSet)
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
if err != nil {
return "", err
}
if len(ars.Finalizers) == 0 {
return "", nil
}
return ars.Finalizers[0], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
ars.Annotations = make(map[string]string)
err = k8sClient.Update(ctx, ars)
Expect(err).NotTo(HaveOccurred(), "Update autoscaling runner set without annotation should be successful")
Eventually(
func() (bool, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
if err != nil {
return false, err
}
return len(ars.ObjectMeta.Annotations) == 0, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(true), "Autoscaling runner set should be updated with empty annotations")
err = k8sClient.Delete(ctx, ars)
Expect(err).NotTo(HaveOccurred(), "Delete autoscaling runner set should be successful")
Eventually(
func() (bool, error) {
updated := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
if err == nil {
return false, nil
}
if !errors.IsNotFound(err) {
return false, err
}
return !controllerutil.ContainsFinalizer(updated, autoscalingRunnerSetFinalizerName), nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(true), "Finalizer and resource should eventually be deleted")
})
})
})
var _ = Describe("Test client optional configuration", Ordered, func() {
var originalBuildVersion string
buildVersion := "0.1.0"
BeforeAll(func() {
originalBuildVersion = build.Version
build.Version = buildVersion
})
AfterAll(func() {
build.Version = originalBuildVersion
})
Context("When specifying a proxy", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var configSecret *corev1.Secret
var controller *AutoscalingRunnerSetReconciler
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
controller = &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), multiclient.NewScaleset()),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
})
It("should be able to make requests to a server using a proxy", func() {
serverSuccessfullyCalled := false
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
serverSuccessfullyCalled = true
w.WriteHeader(http.StatusOK)
}))
defer proxy.Close()
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "http://example.com/org/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Proxy: &v1alpha1.ProxyConfig{
HTTP: &v1alpha1.ProxyServerConfig{
Url: proxy.URL,
},
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
// wait for server to be called
Eventually(
func() (bool, error) {
return serverSuccessfullyCalled, nil
},
autoscalingRunnerSetTestTimeout,
1*time.Nanosecond,
).Should(BeTrue(), "server was not called")
})
It("should be able to make requests to a server using a proxy with user info", func() {
controller.ResourceBuilder.SecretResolver = secretresolver.New(k8sClient, scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithUpdateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithDeleteRunnerScaleSet(nil),
),
),
))
secretCredentials := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "proxy-credentials",
Namespace: autoscalingNS.Name,
},
Data: map[string][]byte{
"username": []byte("test"),
"password": []byte("password"),
},
}
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer proxy.Close()
err := k8sClient.Create(ctx, secretCredentials)
Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials")
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "http://example.com/org/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Proxy: &v1alpha1.ProxyConfig{
HTTP: &v1alpha1.ProxyServerConfig{
Url: "http://test:password@" + proxy.Listener.Addr().String(),
CredentialSecretRef: "proxy-credentials",
},
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
// Verify proxy config with credentials is propagated to EphemeralRunnerSet
Eventually(
func() (*v1alpha1.EphemeralRunnerSet, error) {
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingNS.Name))
if err != nil || len(runnerSetList.Items) == 0 {
return nil, err
}
return &runnerSetList.Items[0], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(WithTransform(func(ers *v1alpha1.EphemeralRunnerSet) *v1alpha1.ProxyConfig {
if ers != nil {
return ers.Spec.EphemeralRunnerSpec.Proxy
}
return nil
}, Not(BeNil())), "EphemeralRunnerSet should have proxy configuration with credentials")
})
})
Context("When specifying a configmap for root CAs", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var configSecret *corev1.Secret
var rootCAConfigMap *corev1.ConfigMap
var controller *AutoscalingRunnerSetReconciler
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
cert, err := os.ReadFile(filepath.Join(
"../../",
"github",
"actions",
"testdata",
"rootCA.crt",
))
Expect(err).NotTo(HaveOccurred(), "failed to read root CA cert")
rootCAConfigMap = &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "root-ca-configmap",
Namespace: autoscalingNS.Name,
},
Data: map[string]string{
"rootCA.crt": string(cert),
},
}
err = k8sClient.Create(ctx, rootCAConfigMap)
Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs")
controller = &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithUpdateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithDeleteRunnerScaleSet(nil),
),
),
)),
},
}
err = controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
})
It("should be able to make requests to a server using root CAs", func() {
controller.SecretResolver = secretresolver.New(k8sClient, multiclient.NewScaleset())
certsFolder := filepath.Join(
"../../",
"github",
"actions",
"testdata",
)
certPath := filepath.Join(certsFolder, "server.crt")
keyPath := filepath.Join(certsFolder, "server.key")
serverSuccessfullyCalled := false
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
serverSuccessfullyCalled = true
w.WriteHeader(http.StatusOK)
}))
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
Expect(err).NotTo(HaveOccurred(), "failed to load server cert")
server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}}
server.StartTLS()
defer server.Close()
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: server.URL + "/my-org",
GitHubConfigSecret: configSecret.Name,
GitHubServerTLS: &v1alpha1.TLSConfig{
CertificateFrom: &v1alpha1.TLSCertificateSource{
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: rootCAConfigMap.Name,
},
Key: "rootCA.crt",
},
},
},
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
// wait for server to be called
Eventually(
func() (bool, error) {
return serverSuccessfullyCalled, nil
},
autoscalingRunnerSetTestTimeout,
1*time.Nanosecond,
).Should(BeTrue(), "server was not called")
})
It("it creates a listener referencing the right configmap for TLS", func() {
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
GitHubServerTLS: &v1alpha1.TLSConfig{
CertificateFrom: &v1alpha1.TLSCertificateSource{
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: rootCAConfigMap.Name,
},
Key: "rootCA.crt",
},
},
},
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
Eventually(
func(g Gomega) {
listener := new(v1alpha1.AutoscalingListener)
err := k8sClient.Get(
ctx,
client.ObjectKey{
Name: scaleSetListenerName(autoscalingRunnerSet),
Namespace: autoscalingRunnerSet.Namespace,
},
listener,
)
g.Expect(err).NotTo(HaveOccurred(), "failed to get listener")
g.Expect(listener.Spec.GitHubServerTLS).NotTo(BeNil(), "listener does not have TLS config")
g.Expect(listener.Spec.GitHubServerTLS).To(BeEquivalentTo(autoscalingRunnerSet.Spec.GitHubServerTLS), "listener does not have TLS config")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "tls config is incorrect")
})
It("it creates an ephemeral runner set referencing the right configmap for TLS", func() {
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
GitHubServerTLS: &v1alpha1.TLSConfig{
CertificateFrom: &v1alpha1.TLSCertificateSource{
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: rootCAConfigMap.Name,
},
Key: "rootCA.crt",
},
},
},
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
Eventually(
func(g Gomega) {
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
g.Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
g.Expect(runnerSetList.Items).To(HaveLen(1), "expected 1 EphemeralRunnerSet to be created")
runnerSet := &runnerSetList.Items[0]
g.Expect(runnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS).NotTo(BeNil(), "expected EphemeralRunnerSpec.GitHubServerTLS to be set")
g.Expect(runnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS).To(BeEquivalentTo(autoscalingRunnerSet.Spec.GitHubServerTLS), "EphemeralRunnerSpec does not have TLS config")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
})
})
})
var _ = Describe("Test external permissions cleanup", Ordered, func() {
var originalBuildVersion string
buildVersion := "0.1.0"
BeforeAll(func() {
originalBuildVersion = build.Version
build.Version = buildVersion
})
AfterAll(func() {
build.Version = originalBuildVersion
})
It("Should clean up kubernetes mode permissions", func() {
ctx := context.Background()
autoscalingNS, mgr := createNamespace(GinkgoT(), k8sClient)
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
"app.kubernetes.io/name": "gha-runner-scale-set",
LabelKeyKubernetesVersion: buildVersion,
},
Annotations: map[string]string{
AnnotationKeyKubernetesModeRoleBindingName: "kube-mode-role-binding",
AnnotationKeyKubernetesModeRoleName: "kube-mode-role",
AnnotationKeyKubernetesModeServiceAccountName: "kube-mode-service-account",
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
role := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeRoleName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
}
err = k8sClient.Create(ctx, role)
Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes mode role")
serviceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeServiceAccountName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
}
err = k8sClient.Create(ctx, serviceAccount)
Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes mode service account")
roleBinding := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeRoleBindingName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: serviceAccount.Name,
Namespace: serviceAccount.Namespace,
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: rbacv1.GroupName,
// Kind is the type of resource being referenced
Kind: "Role",
Name: role.Name,
},
}
err = k8sClient.Create(ctx, roleBinding)
Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes mode role binding")
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
Eventually(
func() (string, error) {
created := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
if err != nil {
return "", err
}
if len(created.Finalizers) == 0 {
return "", nil
}
return created.Finalizers[0], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
err = k8sClient.Delete(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to delete autoscaling runner set")
err = k8sClient.Delete(ctx, roleBinding)
Expect(err).NotTo(HaveOccurred(), "failed to delete kubernetes mode role binding")
err = k8sClient.Delete(ctx, role)
Expect(err).NotTo(HaveOccurred(), "failed to delete kubernetes mode role")
err = k8sClient.Delete(ctx, serviceAccount)
Expect(err).NotTo(HaveOccurred(), "failed to delete kubernetes mode service account")
Eventually(
func() bool {
r := new(rbacv1.RoleBinding)
err := k8sClient.Get(ctx, types.NamespacedName{
Name: roleBinding.Name,
Namespace: roleBinding.Namespace,
}, r)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "Expected role binding to be cleaned up")
Eventually(
func() bool {
r := new(rbacv1.Role)
err := k8sClient.Get(ctx, types.NamespacedName{
Name: role.Name,
Namespace: role.Namespace,
}, r)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "Expected role to be cleaned up")
})
It("Should clean up manager permissions and no-permission service account", func() {
ctx := context.Background()
autoscalingNS, mgr := createNamespace(GinkgoT(), k8sClient)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
"app.kubernetes.io/name": "gha-runner-scale-set",
LabelKeyKubernetesVersion: buildVersion,
},
Annotations: map[string]string{
AnnotationKeyManagerRoleName: "manager-role",
AnnotationKeyManagerRoleBindingName: "manager-role-binding",
AnnotationKeyGitHubSecretName: "gh-secret-name",
AnnotationKeyNoPermissionServiceAccountName: "no-permission-sa",
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyGitHubSecretName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
Data: map[string][]byte{
"github_token": []byte(defaultGitHubToken),
},
}
err = k8sClient.Create(context.Background(), secret)
Expect(err).NotTo(HaveOccurred(), "failed to create github secret")
autoscalingRunnerSet.Spec.GitHubConfigSecret = secret.Name
role := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyManagerRoleName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
}
err = k8sClient.Create(ctx, role)
Expect(err).NotTo(HaveOccurred(), "failed to create manager role")
roleBinding := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyManagerRoleBindingName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
RoleRef: rbacv1.RoleRef{
APIGroup: rbacv1.GroupName,
Kind: "Role",
Name: role.Name,
},
}
err = k8sClient.Create(ctx, roleBinding)
Expect(err).NotTo(HaveOccurred(), "failed to create manager role binding")
noPermissionServiceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Annotations[AnnotationKeyNoPermissionServiceAccountName],
Namespace: autoscalingRunnerSet.Namespace,
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
},
}
err = k8sClient.Create(ctx, noPermissionServiceAccount)
Expect(err).NotTo(HaveOccurred(), "failed to create no permission service account")
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
Eventually(
func() (string, error) {
created := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
if err != nil {
return "", err
}
if len(created.Finalizers) == 0 {
return "", nil
}
return created.Finalizers[0], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
err = k8sClient.Delete(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to delete autoscaling runner set")
err = k8sClient.Delete(ctx, noPermissionServiceAccount)
Expect(err).NotTo(HaveOccurred(), "failed to delete no permission service account")
err = k8sClient.Delete(ctx, secret)
Expect(err).NotTo(HaveOccurred(), "failed to delete GitHub secret")
err = k8sClient.Delete(ctx, roleBinding)
Expect(err).NotTo(HaveOccurred(), "failed to delete manager role binding")
err = k8sClient.Delete(ctx, role)
Expect(err).NotTo(HaveOccurred(), "failed to delete manager role")
Eventually(
func() bool {
r := new(corev1.ServiceAccount)
err := k8sClient.Get(
ctx,
types.NamespacedName{
Name: noPermissionServiceAccount.Name,
Namespace: noPermissionServiceAccount.Namespace,
},
r,
)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "Expected no permission service account to be cleaned up")
Eventually(
func() bool {
r := new(corev1.Secret)
err := k8sClient.Get(ctx, types.NamespacedName{
Name: secret.Name,
Namespace: secret.Namespace,
}, r)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "Expected role binding to be cleaned up")
Eventually(
func() bool {
r := new(rbacv1.RoleBinding)
err := k8sClient.Get(ctx, types.NamespacedName{
Name: roleBinding.Name,
Namespace: roleBinding.Namespace,
}, r)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "Expected role binding to be cleaned up")
Eventually(
func() bool {
r := new(rbacv1.Role)
err := k8sClient.Get(
ctx,
types.NamespacedName{
Name: role.Name,
Namespace: role.Namespace,
},
r,
)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue(), "Expected role to be cleaned up")
})
})
var _ = Describe("Test resource version and build version mismatch", func() {
It("Should delete and recreate the autoscaling runner set to match the build version", func() {
ctx := context.Background()
autoscalingNS, mgr := createNamespace(GinkgoT(), k8sClient)
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
originalVersion := build.Version
defer func() {
build.Version = originalVersion
}()
build.Version = "0.2.0"
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
"app.kubernetes.io/name": "gha-runner-scale-set",
"app.kubernetes.io/version": "0.1.0",
},
Annotations: map[string]string{
AnnotationKeyKubernetesModeRoleBindingName: "kube-mode-role-binding",
AnnotationKeyKubernetesModeRoleName: "kube-mode-role",
AnnotationKeyKubernetesModeServiceAccountName: "kube-mode-service-account",
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
// create autoscaling runner set before starting a manager
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred())
startManagers(GinkgoT(), mgr)
Eventually(
func() bool {
ars := new(v1alpha1.AutoscalingRunnerSet)
err := k8sClient.Get(ctx, types.NamespacedName{Namespace: autoscalingRunnerSet.Namespace, Name: autoscalingRunnerSet.Name}, ars)
return errors.IsNotFound(err)
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeTrue())
})
})
var _ = Describe("Test AutoscalingRunnerSet with a stale runner scale set", Ordered, func() {
const staleRunnerScaleSetID = 42
const freshRunnerScaleSetID = 1
var originalBuildVersion string
buildVersion := "0.1.0"
BeforeAll(func() {
originalBuildVersion = build.Version
build.Version = buildVersion
})
AfterAll(func() {
build.Version = originalBuildVersion
})
newAutoscalingRunnerSet := func(namespace, configSecretName string, annotations map[string]string) *v1alpha1.AutoscalingRunnerSet {
min := 1
max := 10
return &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: namespace,
Labels: map[string]string{LabelKeyKubernetesVersion: buildVersion},
Annotations: annotations,
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecretName,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
}
registeredAnnotations := func(runnerScaleSetID int) map[string]string {
return map[string]string{
runnerScaleSetIDAnnotationKey: strconv.Itoa(runnerScaleSetID),
AnnotationKeyGitHubRunnerScaleSetName: "test-asrs",
AnnotationKeyGitHubRunnerGroupName: "testgroup",
}
}
Context("When the recorded runner scale set no longer exists on the Actions service", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, runnerScaleSetID int) (*scaleset.RunnerScaleSet, error) {
if runnerScaleSetID == staleRunnerScaleSetID {
return nil, scaleset.NotFoundError
}
return &scaleset.RunnerScaleSet{ID: runnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
}),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: freshRunnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithDeleteRunnerScaleSet(nil),
),
),
)),
},
}
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, registeredAnnotations(staleRunnerScaleSetID))
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
})
It("registers the runner scale set again and creates the listener with the new ID", func() {
Eventually(
func() (string, error) {
updated := new(v1alpha1.AutoscalingRunnerSet)
if err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated); err != nil {
return "", err
}
return updated.Annotations[runnerScaleSetIDAnnotationKey], nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(strconv.Itoa(freshRunnerScaleSetID)), "the stale runner scale set ID should be replaced")
Eventually(
func() (int, error) {
listener := new(v1alpha1.AutoscalingListener)
if err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener); err != nil {
return 0, err
}
return listener.Spec.RunnerScaleSetID, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(freshRunnerScaleSetID), "the listener should never be created with the stale runner scale set ID")
})
// The listener is not the only thing that has to follow a re-registration:
// the EphemeralRunnerSet carries the scale set ID down to every runner, so
// assert the propagation here rather than only on the listener.
It("propagates the fresh runner scale set ID to the EphemeralRunnerSet", func() {
runnerSet := new(v1alpha1.EphemeralRunnerSet)
Eventually(
func() (int, error) {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet); err != nil {
return 0, err
}
return runnerSet.Spec.EphemeralRunnerSpec.RunnerScaleSetID, nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(freshRunnerScaleSetID),
"the EphemeralRunnerSet must be re-pointed at the newly registered runner scale set")
// The runners themselves are still registered against the dead scale
// set, so the revision has to advance for them to be cleaned up.
Expect(runnerSet.Spec.ActionableRevision).To(BeNumerically(">", 0),
"re-registration must bump ActionableRevision so existing runners are replaced")
})
})
// A scale set can lose its Actions service counterpart long after it has
// settled, and re-registration then changes the runner scale set ID without
// anything in the AutoscalingRunnerSet spec changing. This exercises that
// full transition.
//
// It also pins down why drift detection cannot be keyed on
// metadata.generation: re-registration writes the ID as an annotation, and
// metadata changes do not bump generation. A generation-based shortcut would
// leave the EphemeralRunnerSet pointing at the dead scale set forever.
Context("When a settled runner scale set disappears from the Actions service", func() {
const originalRunnerScaleSetID = 77
const replacementRunnerScaleSetID = 78
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
var scaleSetDeleted atomic.Bool
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
scaleSetDeleted.Store(false)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, runnerScaleSetID int) (*scaleset.RunnerScaleSet, error) {
if runnerScaleSetID == originalRunnerScaleSetID && scaleSetDeleted.Load() {
return nil, scaleset.NotFoundError
}
return &scaleset.RunnerScaleSet{ID: runnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
}),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: replacementRunnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithDeleteRunnerScaleSet(nil),
),
),
)),
},
}
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, registeredAnnotations(originalRunnerScaleSetID))
// Set the scale set name explicitly. createRunnerScaleSet defaults an
// empty Spec.RunnerScaleSetName to the object name, and that spec write
// bumps metadata.generation, which would let a generation-based
// shortcut pass this test for the wrong reason.
autoscalingRunnerSet.Spec.RunnerScaleSetName = "test-asrs"
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
})
It("re-points the EphemeralRunnerSet without any spec change on the AutoscalingRunnerSet", func() {
// Let the scale set settle first, so observedGeneration catches up with
// generation and the re-registration below is the only thing in flight.
var settledGeneration int64
Eventually(
func(g Gomega) {
current := new(v1alpha1.AutoscalingRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation),
"AutoscalingRunnerSet should reach a settled state")
settledGeneration = current.Generation
runnerSet := new(v1alpha1.EphemeralRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)).To(Succeed())
g.Expect(runnerSet.Spec.EphemeralRunnerSpec.RunnerScaleSetID).To(Equal(originalRunnerScaleSetID))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
// The scale set is deleted on the Actions service side. Nothing about
// the AutoscalingRunnerSet spec changes as a result.
scaleSetDeleted.Store(true)
// Re-registration is only considered when the listener has to be
// created, so drop the listener the way an operator or an eviction
// would. This deliberately does not touch the AutoscalingRunnerSet, so
// its generation stays put.
listener := new(v1alpha1.AutoscalingListener)
Expect(k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingNS.Name}, listener)).To(Succeed())
Expect(k8sClient.Delete(ctx, listener)).To(Succeed())
Eventually(
func(g Gomega) {
current := new(v1alpha1.EphemeralRunnerSet)
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)).To(Succeed())
g.Expect(current.Spec.EphemeralRunnerSpec.RunnerScaleSetID).To(Equal(replacementRunnerScaleSetID),
"the EphemeralRunnerSet must be re-pointed at the newly registered runner scale set even though the AutoscalingRunnerSet spec never changed")
g.Expect(current.Spec.ActionableRevision).To(BeNumerically(">", 0),
"re-registration must bump ActionableRevision so runners registered against the dead scale set are replaced")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed())
// Guard the premise of the test: re-registration must reach the
// EphemeralRunnerSet purely through a spec content change. If it ever
// starts writing to the AutoscalingRunnerSet spec, generation would
// bump and this would stop demonstrating that.
settled := new(v1alpha1.AutoscalingRunnerSet)
Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), settled)).To(Succeed())
Expect(settled.Generation).To(Equal(settledGeneration),
"re-registration must not change the AutoscalingRunnerSet spec, otherwise this test proves nothing")
})
})
Context("When the Actions service cannot confirm whether the runner scale set exists", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
var createCalls atomic.Int64
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
createCalls.Store(0)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, _ int) (*scaleset.RunnerScaleSet, error) {
return nil, fmt.Errorf("actions service is unavailable")
}),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSetFunc(func(_ context.Context, rs *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) {
createCalls.Add(1)
return &scaleset.RunnerScaleSet{ID: freshRunnerScaleSetID, Name: rs.Name, RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
}),
scalefake.WithDeleteRunnerScaleSet(nil),
),
),
)),
},
}
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, registeredAnnotations(staleRunnerScaleSetID))
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
})
It("leaves the runner scale set alone instead of registering a second one", func() {
Consistently(
func() (string, error) {
updated := new(v1alpha1.AutoscalingRunnerSet)
if err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated); err != nil {
return "", err
}
return updated.Annotations[runnerScaleSetIDAnnotationKey], nil
},
"3s",
autoscalingRunnerSetTestInterval,
).Should(BeEquivalentTo(strconv.Itoa(staleRunnerScaleSetID)), "a transient failure should never re-register the runner scale set")
Expect(createCalls.Load()).To(BeEquivalentTo(0), "no runner scale set should be created while the service is unavailable")
})
})
Context("When the runner scale set is already deleted from the Actions service", func() {
var ctx context.Context
var mgr ctrl.Manager
var autoscalingNS *corev1.Namespace
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, runnerScaleSetID int) (*scaleset.RunnerScaleSet, error) {
return &scaleset.RunnerScaleSet{ID: runnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
}),
scalefake.WithGetRunnerScaleSet(nil, nil),
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: freshRunnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
scalefake.WithDeleteRunnerScaleSet(scaleset.NotFoundError),
),
),
)),
},
}
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
startManagers(GinkgoT(), mgr)
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, nil)
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
})
It("does not leave the AutoscalingRunnerSet stuck in Terminating", func() {
Eventually(
func() error {
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "Listener should be created")
Expect(k8sClient.Delete(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to delete AutoScalingRunnerSet")
Eventually(
func() error {
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingRunnerSet))
if err != nil && errors.IsNotFound(err) {
return nil
}
return fmt.Errorf("AutoScalingRunnerSet is not deleted")
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "the finalizer should be removed even though the runner scale set is already gone")
})
})
})
Reference in New Issue View Git Blame Copy Permalink