mirror of
https://github.com/actions-runner-controller/actions-runner-controller.git
synced 2026-09-30 03:45:31 +02:00
Nikola JokicandCopilot App
5581cd0c54
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2715 lines
103 KiB
Go
2715 lines
103 KiB
Go
package actionsgithubcom
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
rbacv1 "k8s.io/api/rbac/v1"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
|
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
|
|
"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
|
|
"github.com/actions/actions-runner-controller/build"
|
|
"github.com/actions/actions-runner-controller/controllers/actions.github.com/multiclient"
|
|
scalefake "github.com/actions/actions-runner-controller/controllers/actions.github.com/multiclient/fake"
|
|
"github.com/actions/actions-runner-controller/controllers/actions.github.com/secretresolver"
|
|
"github.com/actions/scaleset"
|
|
)
|
|
|
|
const (
|
|
autoscalingRunnerSetTestTimeout = time.Second * 20
|
|
autoscalingRunnerSetTestInterval = time.Millisecond * 250
|
|
)
|
|
|
|
var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var controller *AutoscalingRunnerSetReconciler
|
|
var autoscalingNS *corev1.Namespace
|
|
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
|
|
var configSecret *corev1.Secret
|
|
var resourceCache *ResourceCache
|
|
|
|
var originalBuildVersion string
|
|
buildVersion := "0.1.0"
|
|
|
|
BeforeAll(func() {
|
|
originalBuildVersion = build.Version
|
|
build.Version = buildVersion
|
|
})
|
|
|
|
AfterAll(func() {
|
|
build.Version = originalBuildVersion
|
|
})
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
// Track runner group mappings for dynamic responses
|
|
runnerGroupMap := map[int]string{1: "testgroup"} // ID -> Name mapping
|
|
runnerGroupMapLock := &sync.RWMutex{} // Thread-safe access
|
|
resourceCache = newTestResourceCache()
|
|
|
|
controller = &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: resourceCache,
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGetRunnerGroupByNameFunc(func(ctx context.Context, groupName string) (*scaleset.RunnerGroup, error) {
|
|
// Support both "testgroup" and "testgroup2"
|
|
// Update the mapping when a new group is requested
|
|
runnerGroupMapLock.Lock()
|
|
runnerGroupMap[1] = groupName
|
|
runnerGroupMapLock.Unlock()
|
|
return &scaleset.RunnerGroup{ID: 1, Name: groupName}, nil
|
|
}),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSetFunc(func(ctx context.Context, rs *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) {
|
|
// Return a RunnerScaleSet with name matching the requesting ARS
|
|
runnerGroupMapLock.RLock()
|
|
groupName := runnerGroupMap[rs.RunnerGroupID]
|
|
runnerGroupMapLock.RUnlock()
|
|
return &scaleset.RunnerScaleSet{ID: 1, Name: rs.Name, RunnerGroupID: rs.RunnerGroupID, RunnerGroupName: groupName}, nil
|
|
}),
|
|
scalefake.WithUpdateRunnerScaleSetFunc(func(ctx context.Context, scaleSetID int, rs *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) {
|
|
// Return a RunnerScaleSet with the group name corresponding to the runner group ID
|
|
runnerGroupMapLock.RLock()
|
|
groupName := runnerGroupMap[rs.RunnerGroupID]
|
|
runnerGroupMapLock.RUnlock()
|
|
return &scaleset.RunnerScaleSet{ID: 1, Name: rs.Name, RunnerGroupID: rs.RunnerGroupID, RunnerGroupName: groupName}, nil
|
|
}),
|
|
scalefake.WithDeleteRunnerScaleSet(nil),
|
|
),
|
|
),
|
|
)),
|
|
},
|
|
}
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
"arc.test/label-drift": "initial",
|
|
"arc.test/spec-update-label": "initial",
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
EphemeralRunnerSetMetadata: &v1alpha1.ResourceMeta{
|
|
Annotations: map[string]string{
|
|
"arc.test/metadata-annotation": "initial",
|
|
},
|
|
},
|
|
EphemeralRunnerMetadata: &v1alpha1.ResourceMeta{
|
|
Labels: map[string]string{
|
|
"arc.test/runner-metadata-label": "initial",
|
|
},
|
|
Annotations: map[string]string{
|
|
"arc.test/runner-metadata-annotation": "initial",
|
|
},
|
|
},
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
})
|
|
|
|
Context("When creating a new AutoScalingRunnerSet", func() {
|
|
It("It should create/add all required resources for a new AutoScalingRunnerSet (finalizer, runnerscaleset, ephemeralrunnerset, listener)", func() {
|
|
// Check if finalizer is added
|
|
created := new(v1alpha1.AutoscalingRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(created.Finalizers) == 0 {
|
|
return "", nil
|
|
}
|
|
return created.Finalizers[0], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
|
|
|
|
// Check if runner scale set is created on service
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if _, ok := created.Annotations[runnerScaleSetIDAnnotationKey]; !ok {
|
|
return "", nil
|
|
}
|
|
|
|
if _, ok := created.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
|
|
return "", nil
|
|
}
|
|
|
|
return fmt.Sprintf(
|
|
"%s_%s",
|
|
created.Annotations[runnerScaleSetIDAnnotationKey],
|
|
created.Annotations[AnnotationKeyGitHubRunnerGroupName],
|
|
), nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(
|
|
BeEquivalentTo("1_testgroup"),
|
|
"RunnerScaleSet should be created/fetched and update the AutoScalingRunnerSet's annotation",
|
|
)
|
|
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if _, ok := created.Labels[LabelKeyGitHubOrganization]; !ok {
|
|
return "", nil
|
|
}
|
|
|
|
if _, ok := created.Labels[LabelKeyGitHubRepository]; !ok {
|
|
return "", nil
|
|
}
|
|
|
|
return fmt.Sprintf("%s/%s", created.Labels[LabelKeyGitHubOrganization], created.Labels[LabelKeyGitHubRepository]), nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo("owner/repo"), "RunnerScaleSet should be created/fetched and update the AutoScalingRunnerSet's label")
|
|
|
|
// Check if ephemeral runner set is created
|
|
Eventually(
|
|
func() (int, error) {
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
return len(runnerSetList.Items), nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created")
|
|
|
|
// Check if listener is created
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
return resourceCacheStateHasMainObjectEntries(resourceCache.ephemeralRunnerSet, created) &&
|
|
resourceCacheStateHasMainObjectEntries(resourceCache.autoscalingListener, created)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "AutoScalingRunnerSet EphemeralRunnerSet and AutoScalingListener resources should be cached after reconciliation")
|
|
|
|
// Check if status is updated
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
|
|
Expect(len(runnerSetList.Items)).To(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created")
|
|
})
|
|
})
|
|
|
|
Context("When deleting a new AutoScalingRunnerSet", func() {
|
|
It("It should cleanup all resources for a deleting AutoScalingRunnerSet before removing it", func() {
|
|
// Wait till the listener is created
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
|
|
created := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get AutoScalingRunnerSet")
|
|
Eventually(
|
|
func() bool {
|
|
return resourceCacheStateHasMainObjectEntries(resourceCache.ephemeralRunnerSet, created) &&
|
|
resourceCacheStateHasMainObjectEntries(resourceCache.autoscalingListener, created)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "AutoScalingRunnerSet EphemeralRunnerSet and AutoScalingListener resources should be cached before deletion")
|
|
|
|
// Delete the AutoScalingRunnerSet
|
|
err = k8sClient.Delete(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete AutoScalingRunnerSet")
|
|
|
|
// Check if the listener is deleted
|
|
Eventually(
|
|
func() error {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
|
|
if err != nil && errors.IsNotFound(err) {
|
|
return nil
|
|
}
|
|
|
|
return fmt.Errorf("listener is not deleted")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be deleted")
|
|
|
|
// Check if all the EphemeralRunnerSet is deleted
|
|
Eventually(
|
|
func() error {
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(runnerSetList.Items) != 0 {
|
|
return fmt.Errorf("EphemeralRunnerSet is not deleted, count=%v", len(runnerSetList.Items))
|
|
}
|
|
|
|
return nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "All EphemeralRunnerSet should be deleted")
|
|
|
|
// Check if the AutoScalingRunnerSet is deleted
|
|
Eventually(
|
|
func() error {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingRunnerSet))
|
|
if err != nil && errors.IsNotFound(err) {
|
|
return nil
|
|
}
|
|
|
|
return fmt.Errorf("AutoScalingRunnerSet is not deleted")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "AutoScalingRunnerSet should be deleted")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
return resourceCacheStateHasMainObjectEntries(resourceCache.ephemeralRunnerSet, created) ||
|
|
resourceCacheStateHasMainObjectEntries(resourceCache.autoscalingListener, created)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeFalse(), "AutoScalingRunnerSet EphemeralRunnerSet and AutoScalingListener resources should be removed from cache after deletion")
|
|
})
|
|
})
|
|
|
|
It("should not churn listener when already referencing latest ERS (no-op stability)", func() {
|
|
min := 1
|
|
max := 10
|
|
testARSName := "test-asrs-no-churn"
|
|
testARSNamespace := autoscalingNS.Name
|
|
|
|
testARS := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: testARSName,
|
|
Namespace: testARSNamespace,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, testARS)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create test AutoScalingRunnerSet")
|
|
|
|
var latestERSName string
|
|
Eventually(
|
|
func() (string, error) {
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(testARSNamespace))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
// Filter to only ERS owned by our test ARS
|
|
var ownedByTestARS []v1alpha1.EphemeralRunnerSet
|
|
for _, ers := range runnerSetList.Items {
|
|
for _, owner := range ers.OwnerReferences {
|
|
if owner.UID == testARS.UID {
|
|
ownedByTestARS = append(ownedByTestARS, ers)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(ownedByTestARS) != 1 {
|
|
return "", fmt.Errorf("expected 1 EphemeralRunnerSet owned by test ARS, got %d", len(ownedByTestARS))
|
|
}
|
|
|
|
return ownedByTestARS[0].Name, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Not(BeEmpty()), "ERS should be created")
|
|
|
|
// Capture the latest ERS name
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err = k8sClient.List(ctx, runnerSetList, client.InNamespace(testARSNamespace))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
|
|
|
|
var ownedByTestARS []v1alpha1.EphemeralRunnerSet
|
|
for _, ers := range runnerSetList.Items {
|
|
for _, owner := range ers.OwnerReferences {
|
|
if owner.UID == testARS.UID {
|
|
ownedByTestARS = append(ownedByTestARS, ers)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
Expect(len(ownedByTestARS)).To(Equal(1), "should have exactly 1 EphemeralRunnerSet owned by test ARS")
|
|
latestERSName = ownedByTestARS[0].Name
|
|
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, listener)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return listener.Spec.EphemeralRunnerSetName, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal(latestERSName), "listener should reference the latest ERS")
|
|
|
|
// Capture listener identity (UID and ResourceVersion)
|
|
err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, listener)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get listener")
|
|
originalUID := listener.UID
|
|
originalResourceVersion := listener.ResourceVersion
|
|
|
|
Consistently(
|
|
func() (types.UID, error) {
|
|
currentListener := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, currentListener)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return currentListener.UID, nil
|
|
},
|
|
time.Second*5,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal(originalUID), "listener UID should remain unchanged (no recreation)")
|
|
|
|
Consistently(
|
|
func() (string, error) {
|
|
currentListener := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, currentListener)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return currentListener.ResourceVersion, nil
|
|
},
|
|
time.Second*5,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal(originalResourceVersion), "listener ResourceVersion should remain unchanged (no updates)")
|
|
|
|
err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(testARS), Namespace: testARSNamespace}, listener)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get listener")
|
|
Expect(listener.Spec.EphemeralRunnerSetName).To(Equal(latestERSName), "listener should still reference latest ERS")
|
|
})
|
|
|
|
Context("When updating a new AutoScalingRunnerSet", func() {
|
|
It("advances the observed generation once a spec change has been applied", func() {
|
|
var settledGeneration int64
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
|
|
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation))
|
|
settledGeneration = current.Generation
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "AutoscalingRunnerSet should settle with its generation observed")
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated"
|
|
Expect(k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))).To(Succeed(), "failed to patch AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Generation).To(BeNumerically(">", settledGeneration), "a spec write must bump metadata.generation")
|
|
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
|
|
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation), "observed generation must catch up once the change is applied")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
// A label-only edit does not bump metadata.generation, so it never
|
|
// advances the observed generation. It does still rebuild the listener,
|
|
// because the desired listener labels are derived from the
|
|
// AutoscalingRunnerSet's, and the phase has to report that rebuild. So
|
|
// the scale set passes through Pending transiently and comes back to
|
|
// Running, while the observed generation stays exactly where it was.
|
|
It("does not re-observe a generation when only labels change, but still reports the listener rebuild", func() {
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
originalListenerUID := listener.UID
|
|
|
|
var settledGeneration int64
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
|
|
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation))
|
|
settledGeneration = current.Generation
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "AutoscalingRunnerSet should settle with its generation observed")
|
|
|
|
// Hold the deletion window open. Without this the listener is deleted
|
|
// and re-created between two polls, so an implementation that never
|
|
// reported Pending would look identical to one that did. The
|
|
// AutoscalingListener controller does not run in this suite, so
|
|
// nothing else is going to clear this finalizer.
|
|
blockDeletion(listener)
|
|
defer unblockDeletion(listener)
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Labels["arc.test/label-drift"] = "updated"
|
|
Expect(k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))).To(Succeed(), "failed to patch AutoScalingRunnerSet labels")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)).To(Succeed())
|
|
g.Expect(current.Labels).To(HaveKeyWithValue("arc.test/label-drift", "updated"), "labels should still propagate to the EphemeralRunnerSet")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
// The listener is on its way out, so the scale set must not claim to
|
|
// be running. This is the assertion that fails if the Pending update
|
|
// before the listener delete is removed.
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingListener)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)).To(Succeed())
|
|
g.Expect(current.DeletionTimestamp).NotTo(BeNil(), "listener should be marked for deletion to pick up the new label")
|
|
|
|
ars := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), ars)).To(Succeed())
|
|
g.Expect(ars.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhasePending), "scale set must report Pending while its listener is being rebuilt")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
// It stays Pending for as long as the rebuild has not happened, and
|
|
// the observed generation does not drift in the meantime.
|
|
Consistently(
|
|
func(g Gomega) {
|
|
ars := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), ars)).To(Succeed())
|
|
g.Expect(ars.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhasePending), "scale set must stay Pending until the listener is back")
|
|
g.Expect(ars.Status.ObservedGeneration).To(Equal(settledGeneration))
|
|
},
|
|
3*time.Second,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
// Let the rebuild finish.
|
|
unblockDeletion(listener)
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingListener)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)).To(Succeed())
|
|
g.Expect(current.UID).NotTo(Equal(originalListenerUID), "listener should be re-created to pick up the new label")
|
|
g.Expect(current.Labels).To(HaveKeyWithValue("arc.test/label-drift", "updated"), "listener should carry the new label")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
// Once the listener is back, the scale set settles on Running again
|
|
// without the observed generation ever moving, because no spec write
|
|
// happened.
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "AutoscalingRunnerSet should return to Running once the listener is rebuilt")
|
|
|
|
Consistently(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Generation).To(Equal(settledGeneration), "a label-only edit must not bump metadata.generation")
|
|
g.Expect(current.Status.ObservedGeneration).To(Equal(settledGeneration), "a label-only edit must not move the observed generation")
|
|
},
|
|
3*time.Second,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
// The observed generation is only meant to advance once a change has
|
|
// actually been applied. A reconcile that cannot finish must leave it
|
|
// behind the live generation, so the work is retried rather than being
|
|
// mistaken for settled.
|
|
It("leaves the observed generation behind until the reconcile completes", func() {
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
|
|
var settledGeneration int64
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
|
|
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation))
|
|
settledGeneration = current.Generation
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "AutoscalingRunnerSet should settle with its generation observed")
|
|
|
|
// Stall the reconcile part way through: the max runner change below
|
|
// forces the listener to be rebuilt, and the rebuild cannot complete
|
|
// while the listener is held in deletion.
|
|
blockDeletion(listener)
|
|
defer unblockDeletion(listener)
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
updatedMax := 20
|
|
patched.Spec.MaxRunners = &updatedMax
|
|
Expect(k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))).To(Succeed(), "failed to patch AutoScalingRunnerSet max runners")
|
|
|
|
var liveGeneration int64
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Generation).To(BeNumerically(">", settledGeneration), "a spec write must bump metadata.generation")
|
|
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhasePending), "the scale set should go Pending while the change is being applied")
|
|
liveGeneration = current.Generation
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
Consistently(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Status.ObservedGeneration).To(Equal(settledGeneration), "observed generation must not advance while the change is still being applied")
|
|
g.Expect(current.Status.ObservedGeneration).To(BeNumerically("<", liveGeneration))
|
|
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhasePending), "the scale set must stay Pending until the change is applied")
|
|
},
|
|
3*time.Second,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
// Let the reconcile complete, and only now should the marker catch up.
|
|
unblockDeletion(listener)
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(autoscalingRunnerSet), current)).To(Succeed())
|
|
g.Expect(current.Status.Phase).To(Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning))
|
|
g.Expect(current.Status.ObservedGeneration).To(Equal(liveGeneration), "observed generation must catch up once the change has been applied")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
It("updates EphemeralRunnerSet when the runner image changes without touching the Listener", func() {
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
originalListenerUID := listener.UID
|
|
originalListenerResourceVersion := listener.ResourceVersion
|
|
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "EphemeralRunnerSet should be created")
|
|
originalRunnerSetUID := runnerSet.UID
|
|
originalRunnerSetHash := runnerSet.Annotations[annotationKeyIntegrityHash]
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated"
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should be updated in place")
|
|
g.Expect(current.Spec.EphemeralRunnerSpec.PodTemplateSpec.Spec.Containers[0].Image).To(Equal("ghcr.io/actions/runner:updated"))
|
|
g.Expect(current.Annotations[annotationKeyIntegrityHash]).NotTo(Equal(originalRunnerSetHash), "EphemeralRunnerSet spec hash should change")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
Consistently(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener")
|
|
g.Expect(current.UID).To(Equal(originalListenerUID), "Listener should not be recreated")
|
|
g.Expect(current.ResourceVersion).To(Equal(originalListenerResourceVersion), "Listener should not be updated")
|
|
},
|
|
time.Second*5,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
It("recreates only the Listener when max runners changes", func() {
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
originalListenerUID := listener.UID
|
|
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "EphemeralRunnerSet should be created")
|
|
originalRunnerSetUID := runnerSet.UID
|
|
originalRunnerSetHash := runnerSet.Annotations[annotationKeyIntegrityHash]
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
max := 20
|
|
patched.Spec.MaxRunners = &max
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener")
|
|
g.Expect(current.UID).NotTo(Equal(originalListenerUID), "Listener should be recreated")
|
|
g.Expect(current.Spec.MaxRunners).To(Equal(max))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
Consistently(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should not be recreated")
|
|
g.Expect(current.Annotations[annotationKeyIntegrityHash]).To(Equal(originalRunnerSetHash), "EphemeralRunnerSet spec should not change")
|
|
},
|
|
time.Second*5,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
It("updates EphemeralRunnerSet labels when only AutoScalingRunnerSet labels change", func() {
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return runnerSet.Labels["arc.test/label-drift"], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined label")
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Labels["arc.test/label-drift"] = "updated"
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet labels")
|
|
|
|
Eventually(
|
|
func() (string, error) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return current.Labels["arc.test/label-drift"], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal("updated"), "EphemeralRunnerSet should be patched with label-only drift")
|
|
})
|
|
|
|
It("updates EphemeralRunnerSet annotations when only EphemeralRunnerSet metadata annotations change", func() {
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return runnerSet.Annotations["arc.test/metadata-annotation"], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined annotation")
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated"
|
|
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/new-metadata-annotation"] = "added"
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunnerSet metadata")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated"))
|
|
g.Expect(current.Annotations["arc.test/new-metadata-annotation"]).To(Equal("added"))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "EphemeralRunnerSet should be patched with annotation-only metadata drift")
|
|
})
|
|
|
|
It("preserves foreign annotations and labels on the EphemeralRunnerSet", func() {
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return runnerSet.Annotations["arc.test/metadata-annotation"], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined annotation")
|
|
|
|
// Simulate a third party (admission webhook, another controller, a user)
|
|
// adding metadata the AutoscalingRunnerSet knows nothing about.
|
|
foreign := runnerSet.DeepCopy()
|
|
foreign.Annotations["thirdparty.example.com/injected"] = "keep-me"
|
|
foreign.Labels["thirdparty.example.com/injected"] = "keep-me"
|
|
err := k8sClient.Patch(ctx, foreign, client.MergeFrom(runnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to inject foreign metadata on EphemeralRunnerSet")
|
|
|
|
// Force the controller through the metadata reconciliation path.
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated"
|
|
err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunnerSet metadata")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated"))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "desired annotation should still propagate")
|
|
|
|
// The foreign keys must survive, and the controller must be able to get
|
|
// past the metadata block. Before the fix the comparison was made
|
|
// against the unmerged desired metadata, so a foreign key made the
|
|
// block report "modified" on every reconcile and return early, which
|
|
// meant nothing after it — including listener reconciliation — ever
|
|
// ran again. Deleting the listener makes that stall observable.
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "listener should exist before deletion")
|
|
Expect(k8sClient.Delete(ctx, listener)).To(Succeed())
|
|
Eventually(
|
|
func(g Gomega) {
|
|
recreated := new(v1alpha1.AutoscalingListener)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, recreated)).To(Succeed())
|
|
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)).To(Succeed())
|
|
g.Expect(current.Annotations).To(HaveKeyWithValue("thirdparty.example.com/injected", "keep-me"), "foreign annotation must not be stripped")
|
|
g.Expect(current.Labels).To(HaveKeyWithValue("thirdparty.example.com/injected", "keep-me"), "foreign label must not be stripped")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "reconciliation must converge past the metadata block instead of looping on it")
|
|
})
|
|
|
|
It("updates EphemeralRunnerSet runner metadata when only EphemeralRunner metadata changes", func() {
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func(g Gomega) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(runnerSet.Spec.EphemeralRunnerMetadata).NotTo(BeNil())
|
|
g.Expect(runnerSet.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"]).To(Equal("initial"))
|
|
g.Expect(runnerSet.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"]).To(Equal("initial"))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"] = "updated"
|
|
patched.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"] = "updated"
|
|
patched.Spec.EphemeralRunnerMetadata.Annotations["arc.test/new-runner-metadata-annotation"] = "added"
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet EphemeralRunner metadata")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.Spec.EphemeralRunnerMetadata).NotTo(BeNil())
|
|
g.Expect(current.Spec.EphemeralRunnerMetadata.Labels["arc.test/runner-metadata-label"]).To(Equal("updated"))
|
|
g.Expect(current.Spec.EphemeralRunnerMetadata.Annotations["arc.test/runner-metadata-annotation"]).To(Equal("updated"))
|
|
g.Expect(current.Spec.EphemeralRunnerMetadata.Annotations["arc.test/new-runner-metadata-annotation"]).To(Equal("added"))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
It("propagates latest labels while updating EphemeralRunnerSet spec", func() {
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return runnerSet.Labels["arc.test/spec-update-label"], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Equal("initial"), "EphemeralRunnerSet should start with the predefined label")
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Labels["arc.test/spec-update-label"] = "updated"
|
|
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/metadata-annotation"] = "updated"
|
|
patched.Spec.EphemeralRunnerSetMetadata.Annotations["arc.test/new-spec-update-annotation"] = "added"
|
|
patched.Spec.Template.Spec.Containers[0].Image = "ghcr.io/actions/runner:updated-with-label"
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet spec and labels")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.Spec.EphemeralRunnerSpec.PodTemplateSpec.Spec.Containers[0].Image).To(Equal("ghcr.io/actions/runner:updated-with-label"))
|
|
g.Expect(current.Labels["arc.test/spec-update-label"]).To(Equal("updated"))
|
|
g.Expect(current.Annotations["arc.test/metadata-annotation"]).To(Equal("updated"))
|
|
g.Expect(current.Annotations["arc.test/new-spec-update-annotation"]).To(Equal("added"))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
It("updates EphemeralRunnerSet and Listener when the GitHub config secret changes", func() {
|
|
updatedSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "github-config-secret-updated",
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
},
|
|
Data: map[string][]byte{
|
|
"github_token": []byte(defaultGitHubToken),
|
|
},
|
|
}
|
|
err := k8sClient.Create(ctx, updatedSecret)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create updated GitHub config secret")
|
|
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
originalListenerUID := listener.UID
|
|
|
|
runnerSet := new(v1alpha1.EphemeralRunnerSet)
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, runnerSet)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "EphemeralRunnerSet should be created")
|
|
originalRunnerSetUID := runnerSet.UID
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.GitHubConfigSecret = updatedSecret.Name
|
|
err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get AutoScalingRunnerSet")
|
|
g.Expect(current.Spec.GitHubConfigSecret).To(Equal(updatedSecret.Name))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.EphemeralRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
|
|
g.Expect(current.UID).To(Equal(originalRunnerSetUID), "EphemeralRunnerSet should be updated in place")
|
|
g.Expect(current.Spec.EphemeralRunnerSpec.GitHubConfigSecret).To(Equal(updatedSecret.Name))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
current := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, current)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get Listener")
|
|
g.Expect(current.UID).NotTo(Equal(originalListenerUID), "Listener should be recreated")
|
|
g.Expect(current.Spec.GitHubConfigSecret).To(Equal(updatedSecret.Name))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
|
|
It("It should update RunnerScaleSet's runner group on service when it changes", func() {
|
|
updated := new(v1alpha1.AutoscalingRunnerSet)
|
|
// Wait till the listener is created
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
|
|
patched := autoscalingRunnerSet.DeepCopy()
|
|
patched.Spec.RunnerGroup = "testgroup2"
|
|
err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
|
|
|
|
// Check if AutoScalingRunnerSet has the new runner group in its annotation
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if _, ok := updated.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
|
|
return "", nil
|
|
}
|
|
|
|
return updated.Annotations[AnnotationKeyGitHubRunnerGroupName], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the new runner group in its annotation")
|
|
|
|
// delete the annotation and it should be re-added
|
|
patched = autoscalingRunnerSet.DeepCopy()
|
|
delete(patched.Annotations, AnnotationKeyGitHubRunnerGroupName)
|
|
err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet")
|
|
|
|
// Check if AutoScalingRunnerSet still has the runner group in its annotation
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if _, ok := updated.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
|
|
return "", nil
|
|
}
|
|
|
|
return updated.Annotations[AnnotationKeyGitHubRunnerGroupName], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the runner group in its annotation")
|
|
})
|
|
})
|
|
|
|
It("Should update Status on EphemeralRunnerSet status Update", func() {
|
|
ars := new(v1alpha1.AutoscalingRunnerSet)
|
|
Eventually(
|
|
func() (bool, error) {
|
|
err := k8sClient.Get(
|
|
ctx,
|
|
client.ObjectKey{
|
|
Name: autoscalingRunnerSet.Name,
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
},
|
|
ars,
|
|
)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "AutoscalingRunnerSet should be created")
|
|
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
Eventually(
|
|
func() (int, error) {
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(ars.Namespace))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return len(runnerSetList.Items), nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(1), "Failed to fetch runner set list")
|
|
|
|
runnerSet := runnerSetList.Items[0]
|
|
statusUpdate := runnerSet.DeepCopy()
|
|
statusUpdate.Status.Phase = v1alpha1.EphemeralRunnerSetPhaseRunning
|
|
|
|
err := k8sClient.Status().Patch(ctx, statusUpdate, client.MergeFrom(&runnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to patch runner set status")
|
|
|
|
Eventually(
|
|
func() (v1alpha1.AutoscalingRunnerSetStatus, error) {
|
|
updated := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
|
|
if err != nil {
|
|
return v1alpha1.AutoscalingRunnerSetStatus{}, fmt.Errorf("failed to get AutoScalingRunnerSet: %w", err)
|
|
}
|
|
return updated.Status, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(SatisfyAll(
|
|
WithTransform(func(s v1alpha1.AutoscalingRunnerSetStatus) v1alpha1.AutoscalingRunnerSetPhase { return s.Phase }, Equal(v1alpha1.AutoscalingRunnerSetPhaseRunning)),
|
|
WithTransform(func(s v1alpha1.AutoscalingRunnerSetStatus) int64 { return s.ObservedGeneration }, BeNumerically(">=", ars.Generation)),
|
|
), "AutoScalingRunnerSet status should be updated")
|
|
})
|
|
})
|
|
|
|
var _ = Describe("Test AutoScalingController updates", Ordered, func() {
|
|
var originalBuildVersion string
|
|
buildVersion := "0.1.0"
|
|
|
|
BeforeAll(func() {
|
|
originalBuildVersion = build.Version
|
|
build.Version = buildVersion
|
|
})
|
|
|
|
AfterAll(func() {
|
|
build.Version = originalBuildVersion
|
|
})
|
|
|
|
Context("Creating autoscaling runner set with RunnerScaleSetName set", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
|
|
var configSecret *corev1.Secret
|
|
|
|
BeforeEach(func() {
|
|
originalBuildVersion = build.Version
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
multiClient := scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGenerateJitRunnerConfig(
|
|
&scaleset.RunnerScaleSetJitRunnerConfig{
|
|
Runner: &scaleset.RunnerReference{ID: 1, Name: "test-runner"},
|
|
EncodedJITConfig: "fake-jit-config",
|
|
},
|
|
nil,
|
|
),
|
|
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "testset", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithUpdateRunnerScaleSet(
|
|
&scaleset.RunnerScaleSet{
|
|
ID: 1,
|
|
Name: "testset_update",
|
|
RunnerGroupID: 1,
|
|
RunnerGroupName: "testgroup",
|
|
Labels: []scaleset.Label{{Type: "test", Name: "test"}},
|
|
RunnerSetting: scaleset.RunnerSetting{},
|
|
CreatedOn: time.Now(),
|
|
RunnerJitConfigURL: "test.test.test",
|
|
Statistics: nil,
|
|
},
|
|
nil,
|
|
),
|
|
),
|
|
),
|
|
)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), multiClient),
|
|
},
|
|
}
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
})
|
|
|
|
It("It should be create AutoScalingRunnerSet and has annotation for the RunnerScaleSetName", func() {
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerScaleSetName: "testset",
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
// Wait for the AutoScalingRunnerSet to be created with right annotation
|
|
ars := new(v1alpha1.AutoscalingRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if val, ok := ars.Annotations[AnnotationKeyGitHubRunnerScaleSetName]; ok {
|
|
return val, nil
|
|
}
|
|
|
|
return "", nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(autoscalingRunnerSet.Spec.RunnerScaleSetName), "AutoScalingRunnerSet should have annotation for the RunnerScaleSetName")
|
|
|
|
update := autoscalingRunnerSet.DeepCopy()
|
|
update.Spec.RunnerScaleSetName = "testset_update"
|
|
|
|
err = k8sClient.Patch(ctx, update, client.MergeFrom(autoscalingRunnerSet))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to update AutoScalingRunnerSet")
|
|
|
|
// Wait for the AutoScalingRunnerSet to be updated with right annotation
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if val, ok := ars.Annotations[AnnotationKeyGitHubRunnerScaleSetName]; ok {
|
|
return val, nil
|
|
}
|
|
|
|
return "", nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(update.Spec.RunnerScaleSetName), "AutoScalingRunnerSet should have a updated annotation for the RunnerScaleSetName")
|
|
})
|
|
})
|
|
})
|
|
|
|
var _ = Describe("Test AutoscalingController creation failures", Ordered, func() {
|
|
var originalBuildVersion string
|
|
buildVersion := "0.1.0"
|
|
|
|
BeforeAll(func() {
|
|
originalBuildVersion = build.Version
|
|
build.Version = buildVersion
|
|
})
|
|
|
|
AfterAll(func() {
|
|
build.Version = originalBuildVersion
|
|
})
|
|
Context("When autoscaling runner set creation fails on the client", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
|
|
},
|
|
}
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
})
|
|
|
|
It("It should be able to clean up if annotation related to scale set id does not exist", func() {
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
// wait for the finalizer to be added
|
|
ars := new(v1alpha1.AutoscalingRunnerSet)
|
|
Eventually(
|
|
func() (string, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(ars.Finalizers) == 0 {
|
|
return "", nil
|
|
}
|
|
return ars.Finalizers[0], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
|
|
|
|
ars.Annotations = make(map[string]string)
|
|
err = k8sClient.Update(ctx, ars)
|
|
Expect(err).NotTo(HaveOccurred(), "Update autoscaling runner set without annotation should be successful")
|
|
|
|
Eventually(
|
|
func() (bool, error) {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return len(ars.ObjectMeta.Annotations) == 0, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(true), "Autoscaling runner set should be updated with empty annotations")
|
|
|
|
err = k8sClient.Delete(ctx, ars)
|
|
Expect(err).NotTo(HaveOccurred(), "Delete autoscaling runner set should be successful")
|
|
|
|
Eventually(
|
|
func() (bool, error) {
|
|
updated := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated)
|
|
if err == nil {
|
|
return false, nil
|
|
}
|
|
if !errors.IsNotFound(err) {
|
|
return false, err
|
|
}
|
|
|
|
return !controllerutil.ContainsFinalizer(updated, autoscalingRunnerSetFinalizerName), nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(true), "Finalizer and resource should eventually be deleted")
|
|
})
|
|
})
|
|
})
|
|
|
|
var _ = Describe("Test client optional configuration", Ordered, func() {
|
|
var originalBuildVersion string
|
|
buildVersion := "0.1.0"
|
|
|
|
BeforeAll(func() {
|
|
originalBuildVersion = build.Version
|
|
build.Version = buildVersion
|
|
})
|
|
|
|
AfterAll(func() {
|
|
build.Version = originalBuildVersion
|
|
})
|
|
Context("When specifying a proxy", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
var configSecret *corev1.Secret
|
|
var controller *AutoscalingRunnerSetReconciler
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
controller = &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), multiclient.NewScaleset()),
|
|
},
|
|
}
|
|
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
})
|
|
|
|
It("should be able to make requests to a server using a proxy", func() {
|
|
serverSuccessfullyCalled := false
|
|
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
serverSuccessfullyCalled = true
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer proxy.Close()
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "http://example.com/org/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Proxy: &v1alpha1.ProxyConfig{
|
|
HTTP: &v1alpha1.ProxyServerConfig{
|
|
Url: proxy.URL,
|
|
},
|
|
},
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
// wait for server to be called
|
|
Eventually(
|
|
func() (bool, error) {
|
|
return serverSuccessfullyCalled, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
1*time.Nanosecond,
|
|
).Should(BeTrue(), "server was not called")
|
|
})
|
|
|
|
It("should be able to make requests to a server using a proxy with user info", func() {
|
|
controller.ResourceBuilder.SecretResolver = secretresolver.New(k8sClient, scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithUpdateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithDeleteRunnerScaleSet(nil),
|
|
),
|
|
),
|
|
))
|
|
secretCredentials := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "proxy-credentials",
|
|
Namespace: autoscalingNS.Name,
|
|
},
|
|
Data: map[string][]byte{
|
|
"username": []byte("test"),
|
|
"password": []byte("password"),
|
|
},
|
|
}
|
|
|
|
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer proxy.Close()
|
|
|
|
err := k8sClient.Create(ctx, secretCredentials)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials")
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "http://example.com/org/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Proxy: &v1alpha1.ProxyConfig{
|
|
HTTP: &v1alpha1.ProxyServerConfig{
|
|
Url: "http://test:password@" + proxy.Listener.Addr().String(),
|
|
CredentialSecretRef: "proxy-credentials",
|
|
},
|
|
},
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
// Verify proxy config with credentials is propagated to EphemeralRunnerSet
|
|
Eventually(
|
|
func() (*v1alpha1.EphemeralRunnerSet, error) {
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingNS.Name))
|
|
if err != nil || len(runnerSetList.Items) == 0 {
|
|
return nil, err
|
|
}
|
|
return &runnerSetList.Items[0], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(WithTransform(func(ers *v1alpha1.EphemeralRunnerSet) *v1alpha1.ProxyConfig {
|
|
if ers != nil {
|
|
return ers.Spec.EphemeralRunnerSpec.Proxy
|
|
}
|
|
return nil
|
|
}, Not(BeNil())), "EphemeralRunnerSet should have proxy configuration with credentials")
|
|
})
|
|
})
|
|
|
|
Context("When specifying a configmap for root CAs", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
var configSecret *corev1.Secret
|
|
var rootCAConfigMap *corev1.ConfigMap
|
|
var controller *AutoscalingRunnerSetReconciler
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
cert, err := os.ReadFile(filepath.Join(
|
|
"../../",
|
|
"github",
|
|
"actions",
|
|
"testdata",
|
|
"rootCA.crt",
|
|
))
|
|
Expect(err).NotTo(HaveOccurred(), "failed to read root CA cert")
|
|
rootCAConfigMap = &corev1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "root-ca-configmap",
|
|
Namespace: autoscalingNS.Name,
|
|
},
|
|
Data: map[string]string{
|
|
"rootCA.crt": string(cert),
|
|
},
|
|
}
|
|
err = k8sClient.Create(ctx, rootCAConfigMap)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs")
|
|
|
|
controller = &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithUpdateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: 1, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithDeleteRunnerScaleSet(nil),
|
|
),
|
|
),
|
|
)),
|
|
},
|
|
}
|
|
err = controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
})
|
|
|
|
It("should be able to make requests to a server using root CAs", func() {
|
|
controller.SecretResolver = secretresolver.New(k8sClient, multiclient.NewScaleset())
|
|
|
|
certsFolder := filepath.Join(
|
|
"../../",
|
|
"github",
|
|
"actions",
|
|
"testdata",
|
|
)
|
|
certPath := filepath.Join(certsFolder, "server.crt")
|
|
keyPath := filepath.Join(certsFolder, "server.key")
|
|
|
|
serverSuccessfullyCalled := false
|
|
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
serverSuccessfullyCalled = true
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to load server cert")
|
|
|
|
server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}}
|
|
server.StartTLS()
|
|
defer server.Close()
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: server.URL + "/my-org",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
GitHubServerTLS: &v1alpha1.TLSConfig{
|
|
CertificateFrom: &v1alpha1.TLSCertificateSource{
|
|
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
|
|
LocalObjectReference: corev1.LocalObjectReference{
|
|
Name: rootCAConfigMap.Name,
|
|
},
|
|
Key: "rootCA.crt",
|
|
},
|
|
},
|
|
},
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
// wait for server to be called
|
|
Eventually(
|
|
func() (bool, error) {
|
|
return serverSuccessfullyCalled, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
1*time.Nanosecond,
|
|
).Should(BeTrue(), "server was not called")
|
|
})
|
|
|
|
It("it creates a listener referencing the right configmap for TLS", func() {
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
GitHubServerTLS: &v1alpha1.TLSConfig{
|
|
CertificateFrom: &v1alpha1.TLSCertificateSource{
|
|
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
|
|
LocalObjectReference: corev1.LocalObjectReference{
|
|
Name: rootCAConfigMap.Name,
|
|
},
|
|
Key: "rootCA.crt",
|
|
},
|
|
},
|
|
},
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(
|
|
ctx,
|
|
client.ObjectKey{
|
|
Name: scaleSetListenerName(autoscalingRunnerSet),
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
},
|
|
listener,
|
|
)
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to get listener")
|
|
|
|
g.Expect(listener.Spec.GitHubServerTLS).NotTo(BeNil(), "listener does not have TLS config")
|
|
g.Expect(listener.Spec.GitHubServerTLS).To(BeEquivalentTo(autoscalingRunnerSet.Spec.GitHubServerTLS), "listener does not have TLS config")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "tls config is incorrect")
|
|
})
|
|
|
|
It("it creates an ephemeral runner set referencing the right configmap for TLS", func() {
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
GitHubServerTLS: &v1alpha1.TLSConfig{
|
|
CertificateFrom: &v1alpha1.TLSCertificateSource{
|
|
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
|
|
LocalObjectReference: corev1.LocalObjectReference{
|
|
Name: rootCAConfigMap.Name,
|
|
},
|
|
Key: "rootCA.crt",
|
|
},
|
|
},
|
|
},
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func(g Gomega) {
|
|
runnerSetList := new(v1alpha1.EphemeralRunnerSetList)
|
|
err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace))
|
|
g.Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
|
|
g.Expect(runnerSetList.Items).To(HaveLen(1), "expected 1 EphemeralRunnerSet to be created")
|
|
|
|
runnerSet := &runnerSetList.Items[0]
|
|
|
|
g.Expect(runnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS).NotTo(BeNil(), "expected EphemeralRunnerSpec.GitHubServerTLS to be set")
|
|
g.Expect(runnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS).To(BeEquivalentTo(autoscalingRunnerSet.Spec.GitHubServerTLS), "EphemeralRunnerSpec does not have TLS config")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed())
|
|
})
|
|
})
|
|
})
|
|
|
|
var _ = Describe("Test external permissions cleanup", Ordered, func() {
|
|
var originalBuildVersion string
|
|
buildVersion := "0.1.0"
|
|
|
|
BeforeAll(func() {
|
|
originalBuildVersion = build.Version
|
|
build.Version = buildVersion
|
|
})
|
|
|
|
AfterAll(func() {
|
|
build.Version = originalBuildVersion
|
|
})
|
|
|
|
It("Should clean up kubernetes mode permissions", func() {
|
|
ctx := context.Background()
|
|
autoscalingNS, mgr := createNamespace(GinkgoT(), k8sClient)
|
|
|
|
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
|
|
},
|
|
}
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
"app.kubernetes.io/name": "gha-runner-scale-set",
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
Annotations: map[string]string{
|
|
AnnotationKeyKubernetesModeRoleBindingName: "kube-mode-role-binding",
|
|
AnnotationKeyKubernetesModeRoleName: "kube-mode-role",
|
|
AnnotationKeyKubernetesModeServiceAccountName: "kube-mode-service-account",
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
role := &rbacv1.Role{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeRoleName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, role)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes mode role")
|
|
|
|
serviceAccount := &corev1.ServiceAccount{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeServiceAccountName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, serviceAccount)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes mode service account")
|
|
|
|
roleBinding := &rbacv1.RoleBinding{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeRoleBindingName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
Subjects: []rbacv1.Subject{
|
|
{
|
|
Kind: "ServiceAccount",
|
|
Name: serviceAccount.Name,
|
|
Namespace: serviceAccount.Namespace,
|
|
},
|
|
},
|
|
RoleRef: rbacv1.RoleRef{
|
|
APIGroup: rbacv1.GroupName,
|
|
// Kind is the type of resource being referenced
|
|
Kind: "Role",
|
|
Name: role.Name,
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, roleBinding)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes mode role binding")
|
|
|
|
err = k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func() (string, error) {
|
|
created := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(created.Finalizers) == 0 {
|
|
return "", nil
|
|
}
|
|
return created.Finalizers[0], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
|
|
|
|
err = k8sClient.Delete(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete autoscaling runner set")
|
|
|
|
err = k8sClient.Delete(ctx, roleBinding)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete kubernetes mode role binding")
|
|
|
|
err = k8sClient.Delete(ctx, role)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete kubernetes mode role")
|
|
|
|
err = k8sClient.Delete(ctx, serviceAccount)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete kubernetes mode service account")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
r := new(rbacv1.RoleBinding)
|
|
err := k8sClient.Get(ctx, types.NamespacedName{
|
|
Name: roleBinding.Name,
|
|
Namespace: roleBinding.Namespace,
|
|
}, r)
|
|
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "Expected role binding to be cleaned up")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
r := new(rbacv1.Role)
|
|
err := k8sClient.Get(ctx, types.NamespacedName{
|
|
Name: role.Name,
|
|
Namespace: role.Namespace,
|
|
}, r)
|
|
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "Expected role to be cleaned up")
|
|
})
|
|
|
|
It("Should clean up manager permissions and no-permission service account", func() {
|
|
ctx := context.Background()
|
|
autoscalingNS, mgr := createNamespace(GinkgoT(), k8sClient)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
|
|
},
|
|
}
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
"app.kubernetes.io/name": "gha-runner-scale-set",
|
|
LabelKeyKubernetesVersion: buildVersion,
|
|
},
|
|
Annotations: map[string]string{
|
|
AnnotationKeyManagerRoleName: "manager-role",
|
|
AnnotationKeyManagerRoleBindingName: "manager-role-binding",
|
|
AnnotationKeyGitHubSecretName: "gh-secret-name",
|
|
AnnotationKeyNoPermissionServiceAccountName: "no-permission-sa",
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
secret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyGitHubSecretName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
Data: map[string][]byte{
|
|
"github_token": []byte(defaultGitHubToken),
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(context.Background(), secret)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create github secret")
|
|
|
|
autoscalingRunnerSet.Spec.GitHubConfigSecret = secret.Name
|
|
|
|
role := &rbacv1.Role{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyManagerRoleName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, role)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create manager role")
|
|
|
|
roleBinding := &rbacv1.RoleBinding{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyManagerRoleBindingName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
RoleRef: rbacv1.RoleRef{
|
|
APIGroup: rbacv1.GroupName,
|
|
Kind: "Role",
|
|
Name: role.Name,
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, roleBinding)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create manager role binding")
|
|
|
|
noPermissionServiceAccount := &corev1.ServiceAccount{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: autoscalingRunnerSet.Annotations[AnnotationKeyNoPermissionServiceAccountName],
|
|
Namespace: autoscalingRunnerSet.Namespace,
|
|
Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName},
|
|
},
|
|
}
|
|
|
|
err = k8sClient.Create(ctx, noPermissionServiceAccount)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create no permission service account")
|
|
|
|
err = k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func() (string, error) {
|
|
created := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(created.Finalizers) == 0 {
|
|
return "", nil
|
|
}
|
|
return created.Finalizers[0], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer")
|
|
|
|
err = k8sClient.Delete(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete autoscaling runner set")
|
|
|
|
err = k8sClient.Delete(ctx, noPermissionServiceAccount)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete no permission service account")
|
|
|
|
err = k8sClient.Delete(ctx, secret)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete GitHub secret")
|
|
|
|
err = k8sClient.Delete(ctx, roleBinding)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete manager role binding")
|
|
|
|
err = k8sClient.Delete(ctx, role)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to delete manager role")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
r := new(corev1.ServiceAccount)
|
|
err := k8sClient.Get(
|
|
ctx,
|
|
types.NamespacedName{
|
|
Name: noPermissionServiceAccount.Name,
|
|
Namespace: noPermissionServiceAccount.Namespace,
|
|
},
|
|
r,
|
|
)
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "Expected no permission service account to be cleaned up")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
r := new(corev1.Secret)
|
|
err := k8sClient.Get(ctx, types.NamespacedName{
|
|
Name: secret.Name,
|
|
Namespace: secret.Namespace,
|
|
}, r)
|
|
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "Expected role binding to be cleaned up")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
r := new(rbacv1.RoleBinding)
|
|
err := k8sClient.Get(ctx, types.NamespacedName{
|
|
Name: roleBinding.Name,
|
|
Namespace: roleBinding.Namespace,
|
|
}, r)
|
|
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "Expected role binding to be cleaned up")
|
|
|
|
Eventually(
|
|
func() bool {
|
|
r := new(rbacv1.Role)
|
|
err := k8sClient.Get(
|
|
ctx,
|
|
types.NamespacedName{
|
|
Name: role.Name,
|
|
Namespace: role.Namespace,
|
|
},
|
|
r,
|
|
)
|
|
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue(), "Expected role to be cleaned up")
|
|
})
|
|
})
|
|
|
|
var _ = Describe("Test resource version and build version mismatch", func() {
|
|
It("Should delete and recreate the autoscaling runner set to match the build version", func() {
|
|
ctx := context.Background()
|
|
autoscalingNS, mgr := createNamespace(GinkgoT(), k8sClient)
|
|
|
|
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
|
|
},
|
|
}
|
|
err := controller.SetupWithManager(mgr)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
|
|
|
|
originalVersion := build.Version
|
|
defer func() {
|
|
build.Version = originalVersion
|
|
}()
|
|
build.Version = "0.2.0"
|
|
|
|
min := 1
|
|
max := 10
|
|
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: autoscalingNS.Name,
|
|
Labels: map[string]string{
|
|
"app.kubernetes.io/name": "gha-runner-scale-set",
|
|
"app.kubernetes.io/version": "0.1.0",
|
|
},
|
|
Annotations: map[string]string{
|
|
AnnotationKeyKubernetesModeRoleBindingName: "kube-mode-role-binding",
|
|
AnnotationKeyKubernetesModeRoleName: "kube-mode-role",
|
|
AnnotationKeyKubernetesModeServiceAccountName: "kube-mode-service-account",
|
|
},
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecret.Name,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
// create autoscaling runner set before starting a manager
|
|
err = k8sClient.Create(ctx, autoscalingRunnerSet)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
startManagers(GinkgoT(), mgr)
|
|
|
|
Eventually(
|
|
func() bool {
|
|
ars := new(v1alpha1.AutoscalingRunnerSet)
|
|
err := k8sClient.Get(ctx, types.NamespacedName{Namespace: autoscalingRunnerSet.Namespace, Name: autoscalingRunnerSet.Name}, ars)
|
|
return errors.IsNotFound(err)
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeTrue())
|
|
})
|
|
})
|
|
|
|
var _ = Describe("Test AutoscalingRunnerSet with a stale runner scale set", Ordered, func() {
|
|
const staleRunnerScaleSetID = 42
|
|
const freshRunnerScaleSetID = 1
|
|
|
|
var originalBuildVersion string
|
|
buildVersion := "0.1.0"
|
|
|
|
BeforeAll(func() {
|
|
originalBuildVersion = build.Version
|
|
build.Version = buildVersion
|
|
})
|
|
|
|
AfterAll(func() {
|
|
build.Version = originalBuildVersion
|
|
})
|
|
|
|
newAutoscalingRunnerSet := func(namespace, configSecretName string, annotations map[string]string) *v1alpha1.AutoscalingRunnerSet {
|
|
min := 1
|
|
max := 10
|
|
return &v1alpha1.AutoscalingRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-asrs",
|
|
Namespace: namespace,
|
|
Labels: map[string]string{LabelKeyKubernetesVersion: buildVersion},
|
|
Annotations: annotations,
|
|
},
|
|
Spec: v1alpha1.AutoscalingRunnerSetSpec{
|
|
GitHubConfigUrl: "https://github.com/owner/repo",
|
|
GitHubConfigSecret: configSecretName,
|
|
MaxRunners: &max,
|
|
MinRunners: &min,
|
|
RunnerGroup: "testgroup",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "ghcr.io/actions/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
registeredAnnotations := func(runnerScaleSetID int) map[string]string {
|
|
return map[string]string{
|
|
runnerScaleSetIDAnnotationKey: strconv.Itoa(runnerScaleSetID),
|
|
AnnotationKeyGitHubRunnerScaleSetName: "test-asrs",
|
|
AnnotationKeyGitHubRunnerGroupName: "testgroup",
|
|
}
|
|
}
|
|
|
|
Context("When the recorded runner scale set no longer exists on the Actions service", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
|
|
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, runnerScaleSetID int) (*scaleset.RunnerScaleSet, error) {
|
|
if runnerScaleSetID == staleRunnerScaleSetID {
|
|
return nil, scaleset.NotFoundError
|
|
}
|
|
return &scaleset.RunnerScaleSet{ID: runnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
|
|
}),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: freshRunnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithDeleteRunnerScaleSet(nil),
|
|
),
|
|
),
|
|
)),
|
|
},
|
|
}
|
|
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
|
|
startManagers(GinkgoT(), mgr)
|
|
|
|
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, registeredAnnotations(staleRunnerScaleSetID))
|
|
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
|
|
})
|
|
|
|
It("registers the runner scale set again and creates the listener with the new ID", func() {
|
|
Eventually(
|
|
func() (string, error) {
|
|
updated := new(v1alpha1.AutoscalingRunnerSet)
|
|
if err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated); err != nil {
|
|
return "", err
|
|
}
|
|
return updated.Annotations[runnerScaleSetIDAnnotationKey], nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(strconv.Itoa(freshRunnerScaleSetID)), "the stale runner scale set ID should be replaced")
|
|
|
|
Eventually(
|
|
func() (int, error) {
|
|
listener := new(v1alpha1.AutoscalingListener)
|
|
if err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener); err != nil {
|
|
return 0, err
|
|
}
|
|
return listener.Spec.RunnerScaleSetID, nil
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(freshRunnerScaleSetID), "the listener should never be created with the stale runner scale set ID")
|
|
})
|
|
})
|
|
|
|
Context("When the Actions service cannot confirm whether the runner scale set exists", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
|
|
var createCalls atomic.Int64
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
createCalls.Store(0)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
|
|
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, _ int) (*scaleset.RunnerScaleSet, error) {
|
|
return nil, fmt.Errorf("actions service is unavailable")
|
|
}),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSetFunc(func(_ context.Context, rs *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) {
|
|
createCalls.Add(1)
|
|
return &scaleset.RunnerScaleSet{ID: freshRunnerScaleSetID, Name: rs.Name, RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
|
|
}),
|
|
scalefake.WithDeleteRunnerScaleSet(nil),
|
|
),
|
|
),
|
|
)),
|
|
},
|
|
}
|
|
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
|
|
startManagers(GinkgoT(), mgr)
|
|
|
|
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, registeredAnnotations(staleRunnerScaleSetID))
|
|
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
|
|
})
|
|
|
|
It("leaves the runner scale set alone instead of registering a second one", func() {
|
|
Consistently(
|
|
func() (string, error) {
|
|
updated := new(v1alpha1.AutoscalingRunnerSet)
|
|
if err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated); err != nil {
|
|
return "", err
|
|
}
|
|
return updated.Annotations[runnerScaleSetIDAnnotationKey], nil
|
|
},
|
|
"3s",
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(BeEquivalentTo(strconv.Itoa(staleRunnerScaleSetID)), "a transient failure should never re-register the runner scale set")
|
|
|
|
Expect(createCalls.Load()).To(BeEquivalentTo(0), "no runner scale set should be created while the service is unavailable")
|
|
})
|
|
})
|
|
|
|
Context("When the runner scale set is already deleted from the Actions service", func() {
|
|
var ctx context.Context
|
|
var mgr ctrl.Manager
|
|
var autoscalingNS *corev1.Namespace
|
|
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
|
|
configSecret := createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
|
|
|
|
controller := &AutoscalingRunnerSetReconciler{
|
|
Client: mgr.GetClient(),
|
|
Scheme: mgr.GetScheme(),
|
|
Log: logf.Log,
|
|
ControllerNamespace: autoscalingNS.Name,
|
|
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
|
|
ResourceBuilder: ResourceBuilder{
|
|
ResourceCache: newTestResourceCache(),
|
|
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
|
|
scalefake.WithClient(
|
|
scalefake.NewClient(
|
|
scalefake.WithGetRunnerGroupByName(&scaleset.RunnerGroup{ID: 1, Name: "testgroup"}, nil),
|
|
scalefake.WithGetRunnerScaleSetByIDFunc(func(_ context.Context, runnerScaleSetID int) (*scaleset.RunnerScaleSet, error) {
|
|
return &scaleset.RunnerScaleSet{ID: runnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil
|
|
}),
|
|
scalefake.WithGetRunnerScaleSet(nil, nil),
|
|
scalefake.WithCreateRunnerScaleSet(&scaleset.RunnerScaleSet{ID: freshRunnerScaleSetID, Name: "test-asrs", RunnerGroupID: 1, RunnerGroupName: "testgroup"}, nil),
|
|
scalefake.WithDeleteRunnerScaleSet(scaleset.NotFoundError),
|
|
),
|
|
),
|
|
)),
|
|
},
|
|
}
|
|
Expect(controller.SetupWithManager(mgr)).To(Succeed(), "failed to setup controller")
|
|
startManagers(GinkgoT(), mgr)
|
|
|
|
autoscalingRunnerSet = newAutoscalingRunnerSet(autoscalingNS.Name, configSecret.Name, nil)
|
|
Expect(k8sClient.Create(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to create AutoScalingRunnerSet")
|
|
})
|
|
|
|
It("does not leave the AutoscalingRunnerSet stuck in Terminating", func() {
|
|
Eventually(
|
|
func() error {
|
|
return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener))
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "Listener should be created")
|
|
|
|
Expect(k8sClient.Delete(ctx, autoscalingRunnerSet)).To(Succeed(), "failed to delete AutoScalingRunnerSet")
|
|
|
|
Eventually(
|
|
func() error {
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingRunnerSet))
|
|
if err != nil && errors.IsNotFound(err) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("AutoScalingRunnerSet is not deleted")
|
|
},
|
|
autoscalingRunnerSetTestTimeout,
|
|
autoscalingRunnerSetTestInterval,
|
|
).Should(Succeed(), "the finalizer should be removed even though the runner scale set is already gone")
|
|
})
|
|
})
|
|
})
|
|
|
|
// testHoldFinalizer keeps an AutoscalingListener around after it has been
|
|
// deleted, so a test can observe the window during which the scale set has no
|
|
// usable listener. The AutoscalingListener controller is not running in this
|
|
// suite, so nothing else adds or removes finalizers on these objects.
|
|
const testHoldFinalizer = "arc.test/hold-deletion"
|
|
|
|
func blockDeletion(listener *v1alpha1.AutoscalingListener) {
|
|
GinkgoHelper()
|
|
|
|
current := new(v1alpha1.AutoscalingListener)
|
|
Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(listener), current)).To(Succeed(), "failed to get listener to block its deletion")
|
|
|
|
original := current.DeepCopy()
|
|
Expect(controllerutil.AddFinalizer(current, testHoldFinalizer)).To(BeTrue(), "listener should not already hold the test finalizer")
|
|
Expect(k8sClient.Patch(context.Background(), current, client.MergeFrom(original))).To(Succeed(), "failed to add the test finalizer to the listener")
|
|
}
|
|
|
|
// unblockDeletion is idempotent so it can be deferred as a safety net and still
|
|
// be called explicitly at the point a test wants the rebuild to proceed.
|
|
func unblockDeletion(listener *v1alpha1.AutoscalingListener) {
|
|
GinkgoHelper()
|
|
|
|
current := new(v1alpha1.AutoscalingListener)
|
|
err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(listener), current)
|
|
if errors.IsNotFound(err) {
|
|
return
|
|
}
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get listener to unblock its deletion")
|
|
|
|
original := current.DeepCopy()
|
|
if !controllerutil.RemoveFinalizer(current, testHoldFinalizer) {
|
|
return
|
|
}
|
|
Expect(k8sClient.Patch(context.Background(), current, client.MergeFrom(original))).To(Succeed(), "failed to remove the test finalizer from the listener")
|
|
}
|