mirror of
https://github.com/actions-runner-controller/actions-runner-controller.git
synced 2026-09-30 19:31:43 +02:00
Nikola JokicandCopilot App
80a0a64f3f
Moving update detection to metadata.generation lost a signal. The desired listener is built from the AutoscalingRunnerSet's labels and annotations as well as its spec, and any difference there deletes the listener so it can be re-created. metadata.generation only moves on spec writes, so a label-only edit still tore the listener down while the scale set kept reporting Running. If the rebuild then failed, it reported Running with no listener indefinitely. Under the old label-inclusive hash the phase did move, so this was a regression. Mark the resource pending at the point the listener is deleted rather than trying to infer it from the generation. That is what the phase already means: its own doc comment says pending is when the listener is not yet started. It also covers the case properly, because it keys off the actual decision to rebuild instead of guessing from the trigger. This does not change when the listener is rebuilt. A label-only edit recreates it today and still does; only the reported phase changes. The earlier claim that labels no longer restart anything was wrong: labels are propagated to the listener, so editing one is a restart. What metadata.generation changes is narrower than that, and the test now covers the listener's identity across a label edit rather than implying it is untouched. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2564 lines
96 KiB
Go
2564 lines
96 KiB
Go
package actionsgithubcom
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
rbacv1 "k8s.io/api/rbac/v1"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
|
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
|
|
"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
|
|
"github.com/actions/actions-runner-controller/build"
|
|
"github.com/actions/actions-runner-controller/controllers/actions.github.com/multiclient"
|
|
scalefake "github.com/actions/actions-runner-controller/controllers/actions.github.com/multiclient/fake"
|
|
"github.com/actions/actions-runner-controller/controllers/actions.github.com/secretresolver"
|
|
"github.com/actions/scaleset"
|
|
)
|
|
|
|
const (
|
|
autoscalingRunnerSetTestTimeout = time.Second * 20
|
|
autoscalingRunnerSetTestInterval = time.Millisecond * 250
|
|
)
|
|
|
|
var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var controller *AutoscalingRunnerSetReconciler
|
|
var autoscalingNS *corev1.Namespace
|
|
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
|
|
var configSecret *corev1.Secret
|
|
var resourceCache *ResourceCache
|
|
|
|
var originalBuildVersion string
|
|
buildVersion := "0.1.0"
|
|
|
|
BeforeAll(func() {
|
|
originalBuildVersion = build.Version
|
|
build.Version = buildVersion
|
|
})
|
|
|
|
AfterAll(func() {
|
|
build.Version = originalBuildVersion
|
|
})
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
// Track runner group mappings for dynamic responses
|
|
runnerGroupMap := map[int]string{1: "testgroup"} // ID -> Name mapping
|
|
runnerGroupMapLock := &sync.RWMutex{} // Thread-safe access
|
|
resourceCache = newTestResourceCache()
|
|
|
|
controller = &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: resourceCache,
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGetRunnerGroupByNameFunc(func(ctx context.Context, groupName string) (*scaleset.RunnerGroup, error) {
|
|
// Support both "testgroup" and "testgroup2"
|
|
// Update the mapping when a new group is requested
|
|
runnerGroupMapLock.Lock()
|
|
runnerGroupMap[1] = groupName
|
|
runnerGroupMapLock.Unlock()
|
|
return &scaleset.RunnerGroup{ID: 1, Name: groupName}, nil
|
|
}),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSetFunc(func(ctx context.Context, rs *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) {
|
|
// Return a RunnerScaleSet with name matching the requesting ARS
|
|
runnerGroupMapLock.RLock()
|
|
groupName := runnerGroupMap[rs.RunnerGroupID]
|
|
runnerGroupMapLock.RUnlock()
|
|
return &scaleset.RunnerScaleSet{ID: 1, Name: rs.Name, RunnerGroupID: rs.RunnerGroupID, RunnerGroupName: groupName}, nil
|
|
}),
|
|
scalefake.WithUpdateRunnerScaleSetFunc(func(ctx context.Context, scaleSetID int, rs *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) {
|
|
// Return a RunnerScaleSet with the group name corresponding to the runner group ID
|
|
runnerGroupMapLock.RLock()
|
|
groupName := runnerGroupMap[rs.RunnerGroupID]
|
|
runnerGroupMapLock.RUnlock()
|
|
return &scaleset.RunnerScaleSet{ID: 1, Name: rs.Name, RunnerGroupID: rs.RunnerGroupID, RunnerGroupName: groupName}, nil
|
|
}),
|
|
scalefake.WithDeleteRunnerScaleSet(nil),
|
|
),
|
|
),
|
|
)),
|
|
},
|
|
}
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
"arc.test/label-drift": "initial",
|
|
"arc.test/spec-update-label": "initial",
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
EphemeralRunnerSetMetadata: &v1alpha1.ResourceMeta{
|
|
Annotations: map[string]string{
|
|
"arc.test/metadata-annotation": "initial",
|
|
},
|
|
},
|
|
EphemeralRunnerMetadata: &v1alpha1.ResourceMeta{
|
|
Labels: map[string]string{
|
|
"arc.test/runner-metadata-label": "initial",
|
|
},
|
|
Annotations: map[string]string{
|
|
"arc.test/runner-metadata-annotation": "initial",
|
|
},
|
|
},
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
})
|
|
|
|
Context("When creating a new AutoScalingRunnerSet", func() {
|
|
It("It should create/add all required resources for a new AutoScalingRunnerSet (finalizer, runnerscaleset, ephemeralrunnerset, listener)", func() {
|
|
// Check if finalizer is added
|
|
created := new(v1alpha1.AutoscalingRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(created.Finalizers) == 0 {
|
|
return "", nil
|
|
}
|
|
return created.Finalizers[0], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
|
|
|
|
// Check if runner scale set is created on service
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if _, ok := created.Annotations[runnerScaleSetIDAnnotationKey]; !ok {
|
|
return "", nil
|
|
}
|
|
|
|
if _, ok := created.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
|
|
return "", nil
|
|
}
|
|
|
|
return fmt.Sprintf(
|
|
"%s_%s",
|
|
created.Annotations[runnerScaleSetIDAnnotationKey],
|
|
created.Annotations[AnnotationKeyGitHubRunnerGroupName],
|
|
), nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(
|
|
BeEquivalentTo("1_testgroup"),
|
|
"RunnerScaleSet should be created/fetched and update the AutoScalingRunnerSet's annotation",
|
|
)
|
|
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if _, ok := created.Labels[LabelKeyGitHubOrganization]; !ok {
|
|
return "", nil
|
|
}
|
|
|
|
if _, ok := created.Labels[LabelKeyGitHubRepository]; !ok {
|
|
return "", nil
|
|
}
|
|
|
|
return fmt.Sprintf("%s/%s", created.Labels[LabelKeyGitHubOrganization], created.Labels[LabelKeyGitHubRepository]), nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo("owner/repo"), "RunnerScaleSet should be created/fetched and update the AutoScalingRunnerSet's label")
|
|
|
|
// Check if ephemeral runner set is created
|
|
Eventually(
|
|
func() (int, error) {
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
return len(runnerSetList.Items), nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created")
|
|
|
|
// Check if listener is created
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
return resourceCacheStateHasMainObjectEntries(resourceCache.ephemeralRunnerSet, created) &&
|
|
resourceCacheStateHasMainObjectEntries(resourceCache.autoscalingListener, created)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "AutoScalingRunnerSet EphemeralRunnerSet and AutoScalingListener resources should be cached after reconciliation")
|
|
|
|
// Check if status is updated
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
|
|
Expect(len(runnerSetList.Items)).To(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created")
|
|
})
|
|
})
|
|
|
|
Context("When deleting a new AutoScalingRunnerSet", func() {
|
|
It("It should cleanup all resources for a deleting AutoScalingRunnerSet before removing it", func() {
|
|
// Wait till the listener is created
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
|
|
created := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get AutoScalingRunnerSet")
|
|
Eventually(
|
|
func() bool {
|
|
return resourceCacheStateHasMainObjectEntries(resourceCache.ephemeralRunnerSet, created) &&
|
|
resourceCacheStateHasMainObjectEntries(resourceCache.autoscalingListener, created)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "AutoScalingRunnerSet EphemeralRunnerSet and AutoScalingListener resources should be cached before deletion")
|
|
|
|
// Delete the AutoScalingRunnerSet
|
|
err = k8sClient.Delete(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete AutoScalingRunnerSet")
|
|
|
|
// Check if the listener is deleted
|
|
Eventually(
|
|
func() error {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
|
|
if err != nil && errors.IsNotFound(err) {
|
|
return nil
|
|
}
|
|
|
|
return fmt.Errorf("listener is not deleted")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be deleted")
|
|
|
|
// Check if all the EphemeralRunnerSet is deleted
|
|
Eventually(
|
|
func() error {
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(runnerSetList.Items) != 0 {
|
|
return fmt.Errorf("EphemeralRunnerSet is not deleted, count=%v", len(runnerSetList.Items))
|
|
}
|
|
|
|
return nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "All EphemeralRunnerSet should be deleted")
|
|
|
|
// Check if the AutoScalingRunnerSet is deleted
|
|
Eventually(
|
|
func() error {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingRunnerSet))
|
|
if err != nil && errors.IsNotFound(err) {
|
|
return nil
|
|
}
|
|
|
|
return fmt.Errorf("AutoScalingRunnerSet is not deleted")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "AutoScalingRunnerSet should be deleted")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
return resourceCacheStateHasMainObjectEntries(resourceCache.ephemeralRunnerSet, created) ||
|
|
resourceCacheStateHasMainObjectEntries(resourceCache.autoscalingListener, created)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeFalse(), "AutoScalingRunnerSet EphemeralRunnerSet and AutoScalingListener resources should be removed from cache after deletion")
|
|
})
|
|
})
|
|
|
|
It("should not churn listener when already referencing latest ERS (no-op stability)", func() {
|
|
min := 1
|
|
max := 10
|
|
testARSName := "test-asrs-no-churn"
|
|
testARSNamespace := autoscalingNS.Name
|
|
|
|
testARS := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: testARSName,
|
|
Namespace: testARSNamespace,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, testARS)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create test AutoScalingRunnerSet")
|
|
|
|
var latestERSName string
|
|
Eventually(
|
|
func() (string, error) {
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(testARSNamespace))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
// Filter to only ERS owned by our test ARS
|
|
var ownedByTestARS []v1alpha1.EphemeralRunnerSet
|
|
for _, ers := range runnerSetList.Items {
|
|
for _, owner := range ers.OwnerReferences {
|
|
if owner.UID == testARS.UID {
|
|
ownedByTestARS = append(ownedByTestARS, ers)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(ownedByTestARS) != 1 {
|
|
return "", fmt.Errorf("expected 1 EphemeralRunnerSet owned by test ARS, got %d", len(ownedByTestARS))
|
|
}
|
|
|
|
return ownedByTestARS[0].Name, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Not(BeEmpty()), "ERS should be created")
|
|
|
|
// Capture the latest ERS name
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err = k8sClient.List(ctx, runnerSetList, client.InNamespace(testARSNamespace))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
|
|
|
|
var ownedByTestARS []v1alpha1.EphemeralRunnerSet
|
|
for _, ers := range runnerSetList.Items {
|
|
for _, owner := range ers.OwnerReferences {
|
|
if owner.UID == testARS.UID {
|
|
ownedByTestARS = append(ownedByTestARS, ers)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
Expect(len(ownedByTestARS)).To(Equal(1), "should have exactly 1 EphemeralRunnerSet owned by test ARS")
|
|
latestERSName = ownedByTestARS[0].Name
|
|
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, listener)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return listener.Spec.EphemeralRunnerSetName, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal(latestERSName), "listener should reference the latest ERS")
|
|
|
|
// Capture listener identity (UID and ResourceVersion)
|
|
err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, listener)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get listener")
|
|
originalUID := listener.UID
|
|
originalResourceVersion := listener.ResourceVersion
|
|
|
|
Consistently(
|
|
func() (types.UID, error) {
|
|
currentListener := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, currentListener)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return currentListener.UID, nil
|
|
},
|
|
time.Second*5,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal(originalUID), "listener UID should remain unchanged (no recreation)")
|
|
|
|
Consistently(
|
|
func() (string, error) {
|
|
currentListener := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, currentListener)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return currentListener.ResourceVersion, nil
|
|
},
|
|
time.Second*5,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal(originalResourceVersion), "listener ResourceVersion should remain unchanged (no updates)")
|
|
|
|
err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, listener)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get listener")
|
|
Expect(listener.Spec.EphemeralRunnerSetName).To(Equal(latestERSName), "listener should still reference latest ERS")
|
|
})
|
|
|
|
Context("When updating a new AutoScalingRunnerSet", func() {
|
|
It("advances the observed generation once a spec change has been applied", func() {
|
|
var settledGeneration int64
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
|
|
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation))
|
|
settledGeneration = current.Generation
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "AutoscalingRunnerSet should settle with its generation observed")
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated"
|
|
Expect(k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))).To(Succeed(), "failed to patch AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Generation).To(BeNumerically(">", settledGeneration), "a spec write must bump metadata.generation")
|
|
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
|
|
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation), "observed generation must catch up once the change is applied")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
// metadata.generation only tracks spec writes, so a label-only edit no
|
|
// longer drags the scale set through the Pending phase the way the old
|
|
// label-inclusive hash did. Labels still propagate to the
|
|
// EphemeralRunnerSet, they just do not count as an update to apply.
|
|
// A label-only edit does not bump metadata.generation, so it is not an
|
|
// update to apply. It does still rebuild the listener, because the
|
|
// desired listener labels are derived from the AutoscalingRunnerSet's,
|
|
// and the phase has to report that.
|
|
It("does not re-observe a generation when only labels change, but still reports the listener rebuild", func() {
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
originalListenerUID := listener.UID
|
|
|
|
var settledGeneration int64
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
|
|
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation))
|
|
settledGeneration = current.Generation
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "AutoscalingRunnerSet should settle with its generation observed")
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Labels["arc.test/label-drift"] = "updated"
|
|
Expect(k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))).To(Succeed(), "failed to patch AutoScalingRunnerSet labels")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)).To(Succeed())
|
|
g.Expect(current.Labels).To(HaveKeyWithValue("arc.test/label-drift", "updated"), "labels should still propagate to the EphemeralRunnerSet")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
// The listener is rebuilt to pick the new label up, so the scale set
|
|
// must not keep claiming to be running while that happens.
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingListener)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)).To(Succeed())
|
|
g.Expect(current.UID).NotTo(Equal(originalListenerUID), "listener should be re-created to pick up the new label")
|
|
g.Expect(current.Labels).To(HaveKeyWithValue("arc.test/label-drift", "updated"), "listener should carry the new label")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
// Once the listener is back, the scale set settles on Running again
|
|
// without the observed generation ever moving, because no spec write
|
|
// happened.
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "AutoscalingRunnerSet should return to Running once the listener is rebuilt")
|
|
|
|
Consistently(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Generation).To(Equal(settledGeneration), "a label-only edit must not bump metadata.generation")
|
|
g.Expect(current.Status.ObservedGeneration).To(Equal(settledGeneration), "a label-only edit must not move the observed generation")
|
|
},
|
|
3*time.Second,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
It("updates EphemeralRunnerSet when the runner image changes without touching the Listener", func() {
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
originalListenerUID := listener.UID
|
|
originalListenerResourceVersion := listener.ResourceVersion
|
|
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "EphemeralRunnerSet should be created")
|
|
originalRunnerSetUID := runnerSet.UID
|
|
originalRunnerSetHash := runnerSet.Annotations[annotationKeyIntegrityHash]
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated"
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should be updated in place")
|
|
g.Expect(current.Spec.EphemeralRunnerSpec.PodTemplateSpec.Spec.Containers[0].Image).To(Equal("ghcr.io/actions/runner:updated"))
|
|
g.Expect(current.Annotations[annotationKeyIntegrityHash]).NotTo(Equal(originalRunnerSetHash), "EphemeralRunnerSet spec hash should change")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
Consistently(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener")
|
|
g.Expect(current.UID).To(Equal(originalListenerUID), "Listener should not be recreated")
|
|
g.Expect(current.ResourceVersion).To(Equal(originalListenerResourceVersion), "Listener should not be updated")
|
|
},
|
|
time.Second*5,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
It("recreates only the Listener when max runners changes", func() {
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
originalListenerUID := listener.UID
|
|
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "EphemeralRunnerSet should be created")
|
|
originalRunnerSetUID := runnerSet.UID
|
|
originalRunnerSetHash := runnerSet.Annotations[annotationKeyIntegrityHash]
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
max := 20
|
|
patched.Spec.MaxRunners = &max
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener")
|
|
g.Expect(current.UID).NotTo(Equal(originalListenerUID), "Listener should be recreated")
|
|
g.Expect(current.Spec.MaxRunners).To(Equal(max))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
Consistently(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should not be recreated")
|
|
g.Expect(current.Annotations[annotationKeyIntegrityHash]).To(Equal(originalRunnerSetHash), "EphemeralRunnerSet spec should not change")
|
|
},
|
|
time.Second*5,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
It("updates EphemeralRunnerSet labels when only AutoScalingRunnerSet labels change", func() {
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return runnerSet.Labels["arc.test/label-drift"], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined label")
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Labels["arc.test/label-drift"] = "updated"
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet labels")
|
|
|
|
Eventually(
|
|
func() (string, error) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return current.Labels["arc.test/label-drift"], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal("updated"), "EphemeralRunnerSet should be patched with label-only drift")
|
|
})
|
|
|
|
It("updates EphemeralRunnerSet annotations when only EphemeralRunnerSet metadata annotations change", func() {
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return runnerSet.Annotations["arc.test/metadata-annotation"], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined annotation")
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated"
|
|
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/new-metadata-annotation"] = "added"
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunnerSet metadata")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated"))
|
|
g.Expect(current.Annotations["arc.test/new-metadata-annotation"]).To(Equal("added"))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "EphemeralRunnerSet should be patched with annotation-only metadata drift")
|
|
})
|
|
|
|
It("preserves foreign annotations and labels on the EphemeralRunnerSet", func() {
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return runnerSet.Annotations["arc.test/metadata-annotation"], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined annotation")
|
|
|
|
// Simulate a third party (admission webhook, another controller, a user)
|
|
// adding metadata the AutoscalingRunnerSet knows nothing about.
|
|
foreign := runnerSet.DeepCopy()
|
|
foreign.Annotations["thirdparty.example.com/injected"] = "keep-me"
|
|
foreign.Labels["thirdparty.example.com/injected"] = "keep-me"
|
|
err := k8sClient.Patch(ctx, foreign, client.MergeFrom(runnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to inject foreign metadata on EphemeralRunnerSet")
|
|
|
|
// Force the controller through the metadata reconciliation path.
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated"
|
|
err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunnerSet metadata")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated"))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "desired annotation should still propagate")
|
|
|
|
// The foreign keys must survive, and the controller must be able to get
|
|
// past the metadata block. Before the fix the comparison was made
|
|
// against the unmerged desired metadata, so a foreign key made the
|
|
// block report "modified" on every reconcile and return early, which
|
|
// meant nothing after it — including listener reconciliation — ever
|
|
// ran again. Deleting the listener makes that stall observable.
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "listener should exist before deletion")
|
|
Expect(k8sClient.Delete(ctx, listener)).To(Succeed())
|
|
Eventually(
|
|
func(g Gomega) {
|
|
recreated := new(v1alpha1.AutoscalingListener)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, recreated)).To(Succeed())
|
|
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)).To(Succeed())
|
|
g.Expect(current.Annotations).To(HaveKeyWithValue("thirdparty.example.com/injected", "keep-me"), "foreign annotation must not be stripped")
|
|
g.Expect(current.Labels).To(HaveKeyWithValue("thirdparty.example.com/injected", "keep-me"), "foreign label must not be stripped")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "reconciliation must converge past the metadata block instead of looping on it")
|
|
})
|
|
|
|
It("updates EphemeralRunnerSet runner metadata when only EphemeralRunner metadata changes", func() {
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func(g Gomega) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(runnerSet.Spec.EphemeralRunnerMetadata).NotTo(BeNil())
|
|
g.Expect(runnerSet.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"]).To(Equal("initial"))
|
|
g.Expect(runnerSet.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"]).To(Equal("initial"))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"] = "updated"
|
|
patched.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"] = "updated"
|
|
patched.Spec.EphemeralRunnerMetadata.Annotations["arc.test/new-runner-metadata-annotation"] = "added"
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunner metadata")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.Spec.EphemeralRunnerMetadata).NotTo(BeNil())
|
|
g.Expect(current.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"]).To(Equal("updated"))
|
|
g.Expect(current.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"]).To(Equal("updated"))
|
|
g.Expect(current.Spec.EphemeralRunnerMetadata.Annotations["arc.test/new-runner-metadata-annotation"]).To(Equal("added"))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
It("propagates latest labels while updating EphemeralRunnerSet spec", func() {
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return runnerSet.Labels["arc.test/spec-update-label"], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined label")
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Labels["arc.test/spec-update-label"] = "updated"
|
|
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated"
|
|
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/new-spec-update-annotation"] = "added"
|
|
patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated-with-label"
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet spec and labels")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.Spec.EphemeralRunnerSpec.PodTemplateSpec.Spec.Containers[0].Image).To(Equal("ghcr.io/actions/runner:updated-with-label"))
|
|
g.Expect(current.Labels["arc.test/spec-update-label"]).To(Equal("updated"))
|
|
g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated"))
|
|
g.Expect(current.Annotations["arc.test/new-spec-update-annotation"]).To(Equal("added"))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
It("updates EphemeralRunnerSet and Listener when the GitHub config secret changes", func() {
|
|
updatedSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "github-config-secret-updated",
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
},
|
|
Data: map[string][]byte{
|
|
"github_token": []byte(defaultGitHubToken),
|
|
},
|
|
}
|
|
err := k8sClient.Create(ctx, updatedSecret)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create updated GitHub config secret")
|
|
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
originalListenerUID := listener.UID
|
|
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "EphemeralRunnerSet should be created")
|
|
originalRunnerSetUID := runnerSet.UID
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.GitHubConfigSecret = updatedSecret.Name
|
|
err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get AutoScalingRunnerSet")
|
|
g.Expect(current.Spec.GitHubConfigSecret).To(Equal(updatedSecret.Name))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should be updated in place")
|
|
g.Expect(current.Spec.EphemeralRunnerSpec.GitHubConfigSecret).To(Equal(updatedSecret.Name))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener")
|
|
g.Expect(current.UID).NotTo(Equal(originalListenerUID), "Listener should be recreated")
|
|
g.Expect(current.Spec.GitHubConfigSecret).To(Equal(updatedSecret.Name))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
It("It should update RunnerScaleSet's runner group on service when it changes", func() {
|
|
updated := new(v1alpha1.AutoscalingRunnerSet)
|
|
// Wait till the listener is created
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.RunnerGroup = "testgroup2"
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
|
|
|
|
// Check if AutoScalingRunnerSet has the new runner group in its annotation
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if _, ok := updated.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
|
|
return "", nil
|
|
}
|
|
|
|
return updated.Annotations[AnnotationKeyGitHubRunnerGroupName], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the new runner group in its annotation")
|
|
|
|
// delete the annotation and it should be re-added
|
|
patched = autoscalingRunnerSet.DeepCopy()
|
|
delete(patched.Annotations, AnnotationKeyGitHubRunnerGroupName)
|
|
err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
|
|
|
|
// Check if AutoScalingRunnerSet still has the runner group in its annotation
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if _, ok := updated.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
|
|
return "", nil
|
|
}
|
|
|
|
return updated.Annotations[AnnotationKeyGitHubRunnerGroupName], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the runner group in its annotation")
|
|
})
|
|
})
|
|
|
|
It("Should update Status on EphemeralRunnerSet status Update", func() {
|
|
ars := new(v1alpha1.AutoscalingRunnerSet)
|
|
Eventually(
|
|
func() (bool, error) {
|
|
err := k8sClient.Get(
|
|
ctx,
|
|
client.ObjectKey{
|
|
Name: autoscalingRunnerSet.Name,
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
},
|
|
ars,
|
|
)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "AutoscalingRunnerSet should be created")
|
|
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
Eventually(
|
|
func() (int, error) {
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(ars.Namespace))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return len(runnerSetList.Items), nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(1), "Failed to fetch runner set list")
|
|
|
|
runnerSet := runnerSetList.Items[0]
|
|
statusUpdate := runnerSet.DeepCopy()
|
|
statusUpdate.Status.Phase = v1alpha1.EphemeralRunnerSetPhaseRunning
|
|
|
|
err := k8sClient.Status().Patch(ctx, statusUpdate, client.MergeFrom(&runnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to patch runner set status")
|
|
|
|
Eventually(
|
|
func() (v1alpha1.AutoscalingRunnerSetStatus, error) {
|
|
updated := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
|
|
if err != nil {
|
|
return v1alpha1.AutoscalingRunnerSetStatus{}, fmt.Errorf("failed to get AutoScalingRunnerSet: %w", err)
|
|
}
|
|
return updated.Status, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(SatisfyAll(
|
|
WithTransform(func(s v1alpha1.AutoscalingRunnerSetStatus) v1alpha1.AutoscalingRunnerSetPhase { return s.Phase }, Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning)),
|
|
WithTransform(func(s v1alpha1.AutoscalingRunnerSetStatus) int64 { return s.ObservedGeneration }, BeNumerically(">=", ars.Generation)),
|
|
), "AutoScalingRunnerSet status should be updated")
|
|
})
|
|
})
|
|
|
|
var _ = Describe("Test AutoScalingController updates", Ordered, func() {
|
|
var originalBuildVersion string
|
|
buildVersion := "0.1.0"
|
|
|
|
BeforeAll(func() {
|
|
originalBuildVersion = build.Version
|
|
build.Version = buildVersion
|
|
})
|
|
|
|
AfterAll(func() {
|
|
build.Version = originalBuildVersion
|
|
})
|
|
|
|
Context("Creating autoscaling runner set with RunnerScaleSetName set", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
|
|
var configSecret *corev1.Secret
|
|
|
|
BeforeEach(func() {
|
|
originalBuildVersion = build.Version
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
multiClient := scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGenerateJitRunnerConfig(
|
|
&scaleset.RunnerScaleSetJitRunnerConfig{
|
|
Runner: &scaleset.RunnerReference{ID: 1, Name: "test-runner"},
|
|
EncodedJITConfig: "fake-jit-config",
|
|
},
|
|
nil,
|
|
),
|
|
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "testset", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithUpdateRunnerScaleSet(
|
|
&scaleset.RunnerScaleSet{
|
|
ID: 1,
|
|
Name: "testset_update",
|
|
RunnerGroupID: 1,
|
|
RunnerGroupName: "testgroup",
|
|
Labels: []scaleset.Label{{Type: "test", Name: "test"}},
|
|
RunnerSetting: scaleset.RunnerSetting{},
|
|
CreatedOn: time.Now(),
|
|
RunnerJitConfigURL: "test.test.test",
|
|
Statistics: nil,
|
|
},
|
|
nil,
|
|
),
|
|
),
|
|
),
|
|
)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), multiClient),
|
|
},
|
|
}
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
})
|
|
|
|
It("It should be create AutoScalingRunnerSet and has annotation for the RunnerScaleSetName", func() {
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerScaleSetName: "testset",
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
// Wait for the AutoScalingRunnerSet to be created with right annotation
|
|
ars := new(v1alpha1.AutoscalingRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if val, ok := ars.Annotations[AnnotationKeyGitHubRunnerScaleSetName]; ok {
|
|
return val, nil
|
|
}
|
|
|
|
return "", nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(autoscalingRunnerSet.Spec.RunnerScaleSetName), "AutoScalingRunnerSet should have annotation for the RunnerScaleSetName")
|
|
|
|
update := autoscalingRunnerSet.DeepCopy()
|
|
update.Spec.RunnerScaleSetName = "testset_update"
|
|
|
|
err = k8sClient.Patch(ctx, update, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to update AutoScalingRunnerSet")
|
|
|
|
// Wait for the AutoScalingRunnerSet to be updated with right annotation
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if val, ok := ars.Annotations[AnnotationKeyGitHubRunnerScaleSetName]; ok {
|
|
return val, nil
|
|
}
|
|
|
|
return "", nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(update.Spec.RunnerScaleSetName), "AutoScalingRunnerSet should have a updated annotation for the RunnerScaleSetName")
|
|
})
|
|
})
|
|
})
|
|
|
|
var _ = Describe("Test AutoscalingController creation failures", Ordered, func() {
|
|
var originalBuildVersion string
|
|
buildVersion := "0.1.0"
|
|
|
|
BeforeAll(func() {
|
|
originalBuildVersion = build.Version
|
|
build.Version = buildVersion
|
|
})
|
|
|
|
AfterAll(func() {
|
|
build.Version = originalBuildVersion
|
|
})
|
|
Context("When autoscaling runner set creation fails on the client", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
|
|
},
|
|
}
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
})
|
|
|
|
It("It should be able to clean up if annotation related to scale set id does not exist", func() {
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
// wait for the finalizer to be added
|
|
ars := new(v1alpha1.AutoscalingRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(ars.Finalizers) == 0 {
|
|
return "", nil
|
|
}
|
|
return ars.Finalizers[0], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
|
|
|
|
ars.Annotations = make(map[string]string)
|
|
err = k8sClient.Update(ctx, ars)
|
|
Expect(err).NotTo(HaveOccurred(), "Update autoscaling runner set without annotation should be successful")
|
|
|
|
Eventually(
|
|
func() (bool, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return len(ars.ObjectMeta.Annotations) == 0, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(true), "Autoscaling runner set should be updated with empty annotations")
|
|
|
|
err = k8sClient.Delete(ctx, ars)
|
|
Expect(err).NotTo(HaveOccurred(), "Delete autoscaling runner set should be successful")
|
|
|
|
Eventually(
|
|
func() (bool, error) {
|
|
updated := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
|
|
if err == nil {
|
|
return false, nil
|
|
}
|
|
if !errors.IsNotFound(err) {
|
|
return false, err
|
|
}
|
|
|
|
return !controllerutil.ContainsFinalizer(updated, autoscalingRunnerSetFinalizerName), nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(true), "Finalizer and resource should eventually be deleted")
|
|
})
|
|
})
|
|
})
|
|
|
|
var _ = Describe("Test client optional configuration", Ordered, func() {
|
|
var originalBuildVersion string
|
|
buildVersion := "0.1.0"
|
|
|
|
BeforeAll(func() {
|
|
originalBuildVersion = build.Version
|
|
build.Version = buildVersion
|
|
})
|
|
|
|
AfterAll(func() {
|
|
build.Version = originalBuildVersion
|
|
})
|
|
Context("When specifying a proxy", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
var configSecret *corev1.Secret
|
|
var controller *AutoscalingRunnerSetReconciler
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
controller = &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), multiclient.NewScaleset()),
|
|
},
|
|
}
|
|
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
})
|
|
|
|
It("should be able to make requests to a server using a proxy", func() {
|
|
serverSuccessfullyCalled := false
|
|
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
serverSuccessfullyCalled = true
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer proxy.Close()
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "http://example.com/org/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Proxy: &v1alpha1.ProxyConfig{
|
|
HTTP: &v1alpha1.ProxyServerConfig{
|
|
Url: proxy.URL,
|
|
},
|
|
},
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
// wait for server to be called
|
|
Eventually(
|
|
func() (bool, error) {
|
|
return serverSuccessfullyCalled, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
1*time.Nanosecond,
|
|
).Should(BeTrue(), "server was not called")
|
|
})
|
|
|
|
It("should be able to make requests to a server using a proxy with user info", func() {
|
|
controller.ResourceBuilder.SecretResolver = secretresolver.New(k8sClient, scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithUpdateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithDeleteRunnerScaleSet(nil),
|
|
),
|
|
),
|
|
))
|
|
secretCredentials := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "proxy-credentials",
|
|
Namespace: autoscalingNS.Name,
|
|
},
|
|
Data: map[string][]byte{
|
|
"username": []byte("test"),
|
|
"password": []byte("password"),
|
|
},
|
|
}
|
|
|
|
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer proxy.Close()
|
|
|
|
err := k8sClient.Create(ctx, secretCredentials)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials")
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "http://example.com/org/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Proxy: &v1alpha1.ProxyConfig{
|
|
HTTP: &v1alpha1.ProxyServerConfig{
|
|
Url: "http://test:password@" + proxy.Listener.Addr().String(),
|
|
CredentialSecretRef: "proxy-credentials",
|
|
},
|
|
},
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
// Verify proxy config with credentials is propagated to EphemeralRunnerSet
|
|
Eventually(
|
|
func() (*v1alpha1.EphemeralRunnerSet, error) {
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingNS.Name))
|
|
if err != nil || len(runnerSetList.Items) == 0 {
|
|
return nil, err
|
|
}
|
|
return &runnerSetList.Items[0], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(WithTransform(func(ers *v1alpha1.EphemeralRunnerSet) *v1alpha1.ProxyConfig {
|
|
if ers != nil {
|
|
return ers.Spec.EphemeralRunnerSpec.Proxy
|
|
}
|
|
return nil
|
|
}, Not(BeNil())), "EphemeralRunnerSet should have proxy configuration with credentials")
|
|
})
|
|
})
|
|
|
|
Context("When specifying a configmap for root CAs", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
var configSecret *corev1.Secret
|
|
var rootCAConfigMap *corev1.ConfigMap
|
|
var controller *AutoscalingRunnerSetReconciler
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
cert, err := os.ReadFile(filepath.Join(
|
|
"../../",
|
|
"github",
|
|
"actions",
|
|
"testdata",
|
|
"rootCA.crt",
|
|
))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to read root CA cert")
|
|
rootCAConfigMap = &corev1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "root-ca-configmap",
|
|
Namespace: autoscalingNS.Name,
|
|
},
|
|
Data: map[string]string{
|
|
"rootCA.crt": string(cert),
|
|
},
|
|
}
|
|
err = k8sClient.Create(ctx, rootCAConfigMap)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs")
|
|
|
|
controller = &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithUpdateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithDeleteRunnerScaleSet(nil),
|
|
),
|
|
),
|
|
)),
|
|
},
|
|
}
|
|
err = controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
})
|
|
|
|
It("should be able to make requests to a server using root CAs", func() {
|
|
controller.SecretResolver = secretresolver.New(k8sClient, multiclient.NewScaleset())
|
|
|
|
certsFolder := filepath.Join(
|
|
"../../",
|
|
"github",
|
|
"actions",
|
|
"testdata",
|
|
)
|
|
certPath := filepath.Join(certsFolder, "server.crt")
|
|
keyPath := filepath.Join(certsFolder, "server.key")
|
|
|
|
serverSuccessfullyCalled := false
|
|
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
serverSuccessfullyCalled = true
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to load server cert")
|
|
|
|
server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}}
|
|
server.StartTLS()
|
|
defer server.Close()
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: server.URL + "/my-org",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
GitHubServerTLS: &v1alpha1.TLSConfig{
|
|
CertificateFrom: &v1alpha1.TLSCertificateSource{
|
|
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
|
|
LocalObjectReference: corev1.LocalObjectReference{
|
|
Name: rootCAConfigMap.Name,
|
|
},
|
|
Key: "rootCA.crt",
|
|
},
|
|
},
|
|
},
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
// wait for server to be called
|
|
Eventually(
|
|
func() (bool, error) {
|
|
return serverSuccessfullyCalled, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
1*time.Nanosecond,
|
|
).Should(BeTrue(), "server was not called")
|
|
})
|
|
|
|
It("it creates a listener referencing the right configmap for TLS", func() {
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
GitHubServerTLS: &v1alpha1.TLSConfig{
|
|
CertificateFrom: &v1alpha1.TLSCertificateSource{
|
|
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
|
|
LocalObjectReference: corev1.LocalObjectReference{
|
|
Name: rootCAConfigMap.Name,
|
|
},
|
|
Key: "rootCA.crt",
|
|
},
|
|
},
|
|
},
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(
|
|
ctx,
|
|
client.ObjectKey{
|
|
Name: scaleSetListenerName(autoscalingRunnerSet),
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
},
|
|
listener,
|
|
)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get listener")
|
|
|
|
g.Expect(listener.Spec.GitHubServerTLS).NotTo(BeNil(), "listener does not have TLS config")
|
|
g.Expect(listener.Spec.GitHubServerTLS).To(BeEquivalentTo(autoscalingRunnerSet.Spec.GitHubServerTLS), "listener does not have TLS config")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "tls config is incorrect")
|
|
})
|
|
|
|
It("it creates an ephemeral runner set referencing the right configmap for TLS", func() {
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
GitHubServerTLS: &v1alpha1.TLSConfig{
|
|
CertificateFrom: &v1alpha1.TLSCertificateSource{
|
|
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
|
|
LocalObjectReference: corev1.LocalObjectReference{
|
|
Name: rootCAConfigMap.Name,
|
|
},
|
|
Key: "rootCA.crt",
|
|
},
|
|
},
|
|
},
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
|
|
g.Expect(runnerSetList.Items).To(HaveLen(1), "expected 1 EphemeralRunnerSet to be created")
|
|
|
|
runnerSet := &runnerSetList.Items[0]
|
|
|
|
g.Expect(runnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS).NotTo(BeNil(), "expected EphemeralRunnerSpec.GitHubServerTLS to be set")
|
|
g.Expect(runnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS).To(BeEquivalentTo(autoscalingRunnerSet.Spec.GitHubServerTLS), "EphemeralRunnerSpec does not have TLS config")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
})
|
|
})
|
|
|
|
var _ = Describe("Test external permissions cleanup", Ordered, func() {
|
|
var originalBuildVersion string
|
|
buildVersion := "0.1.0"
|
|
|
|
BeforeAll(func() {
|
|
originalBuildVersion = build.Version
|
|
build.Version = buildVersion
|
|
})
|
|
|
|
AfterAll(func() {
|
|
build.Version = originalBuildVersion
|
|
})
|
|
|
|
It("Should clean up kubernetes mode permissions", func() {
|
|
ctx := context.Background()
|
|
autoscalingNS, mgr := createNamespace(GinkgoT(), k8sClient)
|
|
|
|
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
|
|
},
|
|
}
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
"app.kubernetes.io/name": "gha-runner-scale-set",
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
Annotations: map[string]string{
|
|
AnnotationKeyKubernetesModeRoleBindingName: "kube-mode-role-binding",
|
|
AnnotationKeyKubernetesModeRoleName: "kube-mode-role",
|
|
AnnotationKeyKubernetesModeServiceAccountName: "kube-mode-service-account",
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
role := &rbacv1.Role{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeRoleName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, role)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes mode role")
|
|
|
|
serviceAccount := &corev1.ServiceAccount{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeServiceAccountName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, serviceAccount)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes mode service account")
|
|
|
|
roleBinding := &rbacv1.RoleBinding{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeRoleBindingName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
Subjects: []rbacv1.Subject{
|
|
{
|
|
Kind: "ServiceAccount",
|
|
Name: serviceAccount.Name,
|
|
Namespace: serviceAccount.Namespace,
|
|
},
|
|
},
|
|
RoleRef: rbacv1.RoleRef{
|
|
APIGroup: rbacv1.GroupName,
|
|
// Kind is the type of resource being referenced
|
|
Kind: "Role",
|
|
Name: role.Name,
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, roleBinding)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes mode role binding")
|
|
|
|
err = k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func() (string, error) {
|
|
created := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(created.Finalizers) == 0 {
|
|
return "", nil
|
|
}
|
|
return created.Finalizers[0], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
|
|
|
|
err = k8sClient.Delete(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete autoscaling runner set")
|
|
|
|
err = k8sClient.Delete(ctx, roleBinding)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete kubernetes mode role binding")
|
|
|
|
err = k8sClient.Delete(ctx, role)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete kubernetes mode role")
|
|
|
|
err = k8sClient.Delete(ctx, serviceAccount)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete kubernetes mode service account")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
r := new(rbacv1.RoleBinding)
|
|
err := k8sClient.Get(ctx, types.NamespacedName{
|
|
Name: roleBinding.Name,
|
|
Namespace: roleBinding.Namespace,
|
|
}, r)
|
|
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "Expected role binding to be cleaned up")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
r := new(rbacv1.Role)
|
|
err := k8sClient.Get(ctx, types.NamespacedName{
|
|
Name: role.Name,
|
|
Namespace: role.Namespace,
|
|
}, r)
|
|
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "Expected role to be cleaned up")
|
|
})
|
|
|
|
It("Should clean up manager permissions and no-permission service account", func() {
|
|
ctx := context.Background()
|
|
autoscalingNS, mgr := createNamespace(GinkgoT(), k8sClient)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
|
|
},
|
|
}
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
"app.kubernetes.io/name": "gha-runner-scale-set",
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
Annotations: map[string]string{
|
|
AnnotationKeyManagerRoleName: "manager-role",
|
|
AnnotationKeyManagerRoleBindingName: "manager-role-binding",
|
|
AnnotationKeyGitHubSecretName: "gh-secret-name",
|
|
AnnotationKeyNoPermissionServiceAccountName: "no-permission-sa",
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
secret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyGitHubSecretName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
Data: map[string][]byte{
|
|
"github_token": []byte(defaultGitHubToken),
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(context.Background(), secret)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create github secret")
|
|
|
|
autoscalingRunnerSet.Spec.GitHubConfigSecret = secret.Name
|
|
|
|
role := &rbacv1.Role{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyManagerRoleName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, role)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create manager role")
|
|
|
|
roleBinding := &rbacv1.RoleBinding{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyManagerRoleBindingName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
RoleRef: rbacv1.RoleRef{
|
|
APIGroup: rbacv1.GroupName,
|
|
Kind: "Role",
|
|
Name: role.Name,
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, roleBinding)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create manager role binding")
|
|
|
|
noPermissionServiceAccount := &corev1.ServiceAccount{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyNoPermissionServiceAccountName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, noPermissionServiceAccount)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create no permission service account")
|
|
|
|
err = k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func() (string, error) {
|
|
created := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(created.Finalizers) == 0 {
|
|
return "", nil
|
|
}
|
|
return created.Finalizers[0], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
|
|
|
|
err = k8sClient.Delete(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete autoscaling runner set")
|
|
|
|
err = k8sClient.Delete(ctx, noPermissionServiceAccount)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete no permission service account")
|
|
|
|
err = k8sClient.Delete(ctx, secret)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete GitHub secret")
|
|
|
|
err = k8sClient.Delete(ctx, roleBinding)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete manager role binding")
|
|
|
|
err = k8sClient.Delete(ctx, role)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete manager role")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
r := new(corev1.ServiceAccount)
|
|
err := k8sClient.Get(
|
|
ctx,
|
|
types.NamespacedName{
|
|
Name: noPermissionServiceAccount.Name,
|
|
Namespace: noPermissionServiceAccount.Namespace,
|
|
},
|
|
r,
|
|
)
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "Expected no permission service account to be cleaned up")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
r := new(corev1.Secret)
|
|
err := k8sClient.Get(ctx, types.NamespacedName{
|
|
Name: secret.Name,
|
|
Namespace: secret.Namespace,
|
|
}, r)
|
|
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "Expected role binding to be cleaned up")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
r := new(rbacv1.RoleBinding)
|
|
err := k8sClient.Get(ctx, types.NamespacedName{
|
|
Name: roleBinding.Name,
|
|
Namespace: roleBinding.Namespace,
|
|
}, r)
|
|
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "Expected role binding to be cleaned up")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
r := new(rbacv1.Role)
|
|
err := k8sClient.Get(
|
|
ctx,
|
|
types.NamespacedName{
|
|
Name: role.Name,
|
|
Namespace: role.Namespace,
|
|
},
|
|
r,
|
|
)
|
|
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "Expected role to be cleaned up")
|
|
})
|
|
})
|
|
|
|
var _ = Describe("Test resource version and build version mismatch", func() {
|
|
It("Should delete and recreate the autoscaling runner set to match the build version", func() {
|
|
ctx := context.Background()
|
|
autoscalingNS, mgr := createNamespace(GinkgoT(), k8sClient)
|
|
|
|
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
|
|
},
|
|
}
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
originalVersion := build.Version
|
|
defer func() {
|
|
build.Version = originalVersion
|
|
}()
|
|
build.Version = "0.2.0"
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
"app.kubernetes.io/name": "gha-runner-scale-set",
|
|
"app.kubernetes.io/version": "0.1.0",
|
|
},
|
|
Annotations: map[string]string{
|
|
AnnotationKeyKubernetesModeRoleBindingName: "kube-mode-role-binding",
|
|
AnnotationKeyKubernetesModeRoleName: "kube-mode-role",
|
|
AnnotationKeyKubernetesModeServiceAccountName: "kube-mode-service-account",
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
// create autoscaling runner set before starting a manager
|
|
err = k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
|
|
Eventually(
|
|
func() bool {
|
|
ars := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, types.NamespacedName{Namespace: autoscalingRunnerSet.Namespace, Name: autoscalingRunnerSet.Name}, ars)
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue())
|
|
})
|
|
})
|
|
|
|
var _ = Describe("Test AutoscalingRunnerSet with a stale runner scale set", Ordered, func() {
|
|
const staleRunnerScaleSetID = 42
|
|
const freshRunnerScaleSetID = 1
|
|
|
|
var originalBuildVersion string
|
|
buildVersion := "0.1.0"
|
|
|
|
BeforeAll(func() {
|
|
originalBuildVersion = build.Version
|
|
build.Version = buildVersion
|
|
})
|
|
|
|
AfterAll(func() {
|
|
build.Version = originalBuildVersion
|
|
})
|
|
|
|
newAutoscalingRunnerSet := func(namespace, configSecretName string, annotations map[string]string) *v1alpha1.AutoscalingRunnerSet {
|
|
min := 1
|
|
max := 10
|
|
return &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: namespace,
|
|
Labels: map[string]string{LabelKeyKubernetesVersion: buildVersion},
|
|
Annotations: annotations,
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecretName,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
registeredAnnotations := func(runnerScaleSetID int) map[string]string {
|
|
return map[string]string{
|
|
runnerScaleSetIDAnnotationKey: strconv.Itoa(runnerScaleSetID),
|
|
AnnotationKeyGitHubRunnerScaleSetName: "test-asrs",
|
|
AnnotationKeyGitHubRunnerGroupName: "testgroup",
|
|
}
|
|
}
|
|
|
|
Context("When the recorded runner scale set no longer exists on the Actions service", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
|
|
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, runnerScaleSetID int) (*scaleset.RunnerScaleSet, error) {
|
|
if runnerScaleSetID == staleRunnerScaleSetID {
|
|
return nil, scaleset.NotFoundError
|
|
}
|
|
return &scaleset.RunnerScaleSet{ID: runnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
|
|
}),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: freshRunnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithDeleteRunnerScaleSet(nil),
|
|
),
|
|
),
|
|
)),
|
|
},
|
|
}
|
|
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
|
|
startManagers(GinkgoT(), mgr)
|
|
|
|
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, registeredAnnotations(staleRunnerScaleSetID))
|
|
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
|
|
})
|
|
|
|
It("registers the runner scale set again and creates the listener with the new ID", func() {
|
|
Eventually(
|
|
func() (string, error) {
|
|
updated := new(v1alpha1.AutoscalingRunnerSet)
|
|
if err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated); err != nil {
|
|
return "", err
|
|
}
|
|
return updated.Annotations[runnerScaleSetIDAnnotationKey], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(strconv.Itoa(freshRunnerScaleSetID)), "the stale runner scale set ID should be replaced")
|
|
|
|
Eventually(
|
|
func() (int, error) {
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
if err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener); err != nil {
|
|
return 0, err
|
|
}
|
|
return listener.Spec.RunnerScaleSetID, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(freshRunnerScaleSetID), "the listener should never be created with the stale runner scale set ID")
|
|
})
|
|
})
|
|
|
|
Context("When the Actions service cannot confirm whether the runner scale set exists", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
|
|
var createCalls atomic.Int64
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
createCalls.Store(0)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
|
|
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, _ int) (*scaleset.RunnerScaleSet, error) {
|
|
return nil, fmt.Errorf("actions service is unavailable")
|
|
}),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSetFunc(func(_ context.Context, rs *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) {
|
|
createCalls.Add(1)
|
|
return &scaleset.RunnerScaleSet{ID: freshRunnerScaleSetID, Name: rs.Name, RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
|
|
}),
|
|
scalefake.WithDeleteRunnerScaleSet(nil),
|
|
),
|
|
),
|
|
)),
|
|
},
|
|
}
|
|
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
|
|
startManagers(GinkgoT(), mgr)
|
|
|
|
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, registeredAnnotations(staleRunnerScaleSetID))
|
|
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
|
|
})
|
|
|
|
It("leaves the runner scale set alone instead of registering a second one", func() {
|
|
Consistently(
|
|
func() (string, error) {
|
|
updated := new(v1alpha1.AutoscalingRunnerSet)
|
|
if err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated); err != nil {
|
|
return "", err
|
|
}
|
|
return updated.Annotations[runnerScaleSetIDAnnotationKey], nil
|
|
},
|
|
"3s",
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(strconv.Itoa(staleRunnerScaleSetID)), "a transient failure should never re-register the runner scale set")
|
|
|
|
Expect(createCalls.Load()).To(BeEquivalentTo(0), "no runner scale set should be created while the service is unavailable")
|
|
})
|
|
})
|
|
|
|
Context("When the runner scale set is already deleted from the Actions service", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
|
|
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, runnerScaleSetID int) (*scaleset.RunnerScaleSet, error) {
|
|
return &scaleset.RunnerScaleSet{ID: runnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
|
|
}),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: freshRunnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithDeleteRunnerScaleSet(scaleset.NotFoundError),
|
|
),
|
|
),
|
|
)),
|
|
},
|
|
}
|
|
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
|
|
startManagers(GinkgoT(), mgr)
|
|
|
|
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, nil)
|
|
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
|
|
})
|
|
|
|
It("does not leave the AutoscalingRunnerSet stuck in Terminating", func() {
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
|
|
Expect(k8sClient.Delete(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to delete AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func() error {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingRunnerSet))
|
|
if err != nil && errors.IsNotFound(err) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("AutoScalingRunnerSet is not deleted")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "the finalizer should be removed even though the runner scale set is already gone")
|
|
})
|
|
})
|
|
})
|