Include resource cache for desired resources

This commit is contained in:
Nikola Jokic
2026-07-14 19:57:17 +02:00
parent 9ad6faacb3
commit 76027003c7
17 changed files with 805 additions and 2 deletions
@@ -36,7 +36,12 @@ import (
// +kubebuilder:subresource:status
// +kubebuilder:printcolumn:JSONPath=".spec.minRunners",name=Minimum Runners,type=integer
// +kubebuilder:printcolumn:JSONPath=".spec.maxRunners",name=Maximum Runners,type=integer
// +kubebuilder:printcolumn:JSONPath=".status.currentRunners",name=Current Runners,type=integer
// +kubebuilder:printcolumn:JSONPath=".status.phase",name=Phase,type=string
// +kubebuilder:printcolumn:JSONPath=".status.pendingEphemeralRunners",name=Pending Runners,type=integer
// +kubebuilder:printcolumn:JSONPath=".status.runningEphemeralRunners",name=Running Runners,type=integer
// +kubebuilder:printcolumn:JSONPath=".status.finishedEphemeralRunners",name=Finished Runners,type=integer
// +kubebuilder:printcolumn:JSONPath=".status.deletingEphemeralRunners",name=Deleting Runners,type=integer
// AutoscalingRunnerSet is the Schema for the autoscalingrunnersets API
type AutoscalingRunnerSet struct {
@@ -21,9 +21,24 @@ spec:
- jsonPath: .spec.maxRunners
name: Maximum Runners
type: integer
- jsonPath: .status.currentRunners
name: Current Runners
type: integer
- jsonPath: .status.phase
name: Phase
type: string
- jsonPath: .status.pendingEphemeralRunners
name: Pending Runners
type: integer
- jsonPath: .status.runningEphemeralRunners
name: Running Runners
type: integer
- jsonPath: .status.finishedEphemeralRunners
name: Finished Runners
type: integer
- jsonPath: .status.deletingEphemeralRunners
name: Deleting Runners
type: integer
name: v1alpha1
schema:
openAPIV3Schema:
@@ -21,9 +21,24 @@ spec:
- jsonPath: .spec.maxRunners
name: Maximum Runners
type: integer
- jsonPath: .status.currentRunners
name: Current Runners
type: integer
- jsonPath: .status.phase
name: Phase
type: string
- jsonPath: .status.pendingEphemeralRunners
name: Pending Runners
type: integer
- jsonPath: .status.runningEphemeralRunners
name: Running Runners
type: integer
- jsonPath: .status.finishedEphemeralRunners
name: Finished Runners
type: integer
- jsonPath: .status.deletingEphemeralRunners
name: Deleting Runners
type: integer
name: v1alpha1
schema:
openAPIV3Schema:
@@ -21,9 +21,24 @@ spec:
- jsonPath: .spec.maxRunners
name: Maximum Runners
type: integer
- jsonPath: .status.currentRunners
name: Current Runners
type: integer
- jsonPath: .status.phase
name: Phase
type: string
- jsonPath: .status.pendingEphemeralRunners
name: Pending Runners
type: integer
- jsonPath: .status.runningEphemeralRunners
name: Running Runners
type: integer
- jsonPath: .status.finishedEphemeralRunners
name: Finished Runners
type: integer
- jsonPath: .status.deletingEphemeralRunners
name: Deleting Runners
type: integer
name: v1alpha1
schema:
openAPIV3Schema:
@@ -105,6 +105,7 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl.
}
log.Info("Successfully removed finalizer after cleanup")
r.ResourceCache.Delete(&autoscalingListener)
return ctrl.Result{}, nil
}
@@ -50,6 +50,7 @@ var _ = Describe("Test AutoScalingListener controller", func() {
)
rb := ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretResolver,
}
@@ -593,6 +594,7 @@ var _ = Describe("Test AutoScalingListener customization", func() {
secretResolver := secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient())
rb := ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretResolver,
}
@@ -922,6 +924,7 @@ var _ = Describe("Test AutoScalingListener controller with proxy", func() {
secretResolver := secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient())
rb := ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretResolver,
}
@@ -1127,6 +1130,7 @@ var _ = Describe("Test AutoScalingListener controller with template modification
secretResolver := secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient())
rb := ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretResolver,
}
@@ -1232,6 +1236,7 @@ var _ = Describe("Test GitHub Server TLS configuration", func() {
secretResolver := secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient())
rb := ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretResolver,
}
@@ -108,6 +108,7 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
}
log.Info("Successfully removed finalizer after cleanup")
r.ResourceCache.Delete(&autoscalingRunnerSet)
return ctrl.Result{}, nil
}
@@ -73,6 +73,7 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() {
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
@@ -836,6 +837,7 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() {
).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the runner group in its annotation")
})
})
It("Should update Status on EphemeralRunnerSet status Update", func() {
ars := new(v1alpha1.AutoscalingRunnerSet)
Eventually(
@@ -960,6 +962,7 @@ var _ = Describe("Test AutoScalingController updates", Ordered, func() {
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), multiClient),
},
}
@@ -1077,6 +1080,7 @@ var _ = Describe("Test AutoscalingController creation failures", Ordered, func()
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
@@ -1204,6 +1208,7 @@ var _ = Describe("Test client optional configuration", Ordered, func() {
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), multiclient.NewScaleset()),
},
}
@@ -1399,6 +1404,7 @@ var _ = Describe("Test client optional configuration", Ordered, func() {
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
@@ -1646,6 +1652,7 @@ var _ = Describe("Test external permissions cleanup", Ordered, func() {
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
@@ -1806,6 +1813,7 @@ var _ = Describe("Test external permissions cleanup", Ordered, func() {
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
@@ -2016,6 +2024,7 @@ var _ = Describe("Test resource version and build version mismatch", func() {
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
@@ -151,7 +151,7 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
}
}
log.Info("Successfully removed finalizer after cleanup")
r.ResourceCache.Delete(&ephemeralRunner)
return ctrl.Result{}, nil
}
@@ -111,6 +111,7 @@ var _ = Describe("EphemeralRunner", func() {
Scheme: mgr.GetScheme(),
Log: logf.Log,
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
@@ -1216,6 +1217,7 @@ var _ = Describe("EphemeralRunner", func() {
Scheme: mgr.GetScheme(),
Log: logf.Log,
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(
mgr.GetClient(),
scalefake.NewMultiClient(
@@ -1302,6 +1304,7 @@ var _ = Describe("EphemeralRunner", func() {
Scheme: mgr.GetScheme(),
Log: logf.Log,
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient(
scalefake.WithClient(
scalefake.NewClient(
@@ -1326,6 +1329,7 @@ var _ = Describe("EphemeralRunner", func() {
It("uses an actions client with proxy transport", func() {
// Use an actual client
controller.ResourceBuilder = ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(
mgr.GetClient(),
multiclient.NewScaleset(),
@@ -1485,6 +1489,7 @@ var _ = Describe("EphemeralRunner", func() {
Scheme: mgr.GetScheme(),
Log: logf.Log,
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), scalefake.NewMultiClient()),
},
}
@@ -1519,6 +1524,7 @@ var _ = Describe("EphemeralRunner", func() {
// Use an actual client
controller.ResourceBuilder = ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(
mgr.GetClient(),
multiclient.NewScaleset(),
@@ -117,6 +117,7 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
}
log.Info("Successfully removed finalizer after cleanup")
r.ResourceCache.Delete(&ephemeralRunnerSet)
return ctrl.Result{}, nil
}
@@ -144,6 +144,7 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() {
Scheme: mgr.GetScheme(),
Log: logf.Log,
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), fake.NewMultiClient(
fake.WithClient(
fake.NewClient(
@@ -1363,6 +1364,7 @@ var _ = Describe("EphemeralRunner phase metrics", func() {
Log: logf.Log,
PublishMetrics: true,
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(k8sClient, fake.NewMultiClient(
fake.WithClient(
fake.NewClient(
@@ -1494,6 +1496,7 @@ var _ = Describe("Test EphemeralRunnerSet controller with proxy settings", func(
Scheme: mgr.GetScheme(),
Log: logf.Log,
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), multiclient.NewScaleset()),
},
}
@@ -1812,6 +1815,7 @@ var _ = Describe("Test EphemeralRunnerSet controller with custom root CA", func(
Scheme: mgr.GetScheme(),
Log: logf.Log,
ResourceBuilder: ResourceBuilder{
ResourceCache: newTestResourceCache(),
SecretResolver: secretresolver.New(mgr.GetClient(), multiclient.NewScaleset()),
},
}
@@ -95,7 +95,8 @@ type SecretResolver interface {
type ResourceBuilder struct {
ExcludeLabelPropagationPrefixes []string
SecretResolver
Scheme *runtime.Scheme
Scheme *runtime.Scheme
ResourceCache *ResourceCache
}
func (b *ResourceBuilder) setSchemeIfUnset(scheme *runtime.Scheme) {
@@ -121,6 +122,27 @@ func (b *ResourceBuilder) newAutoscalingListener(autoscalingRunnerSet *v1alpha1.
return nil, err
}
cacheKeyObject := &v1alpha1.AutoscalingListener{
ObjectMeta: metav1.ObjectMeta{
Name: scaleSetListenerName(autoscalingRunnerSet),
Namespace: namespace,
},
}
inputDependency := resourceCacheInputObject("autoscaling-listener-inputs", struct {
Namespace string
Image string
ImagePullSecrets []corev1.LocalObjectReference
}{
Namespace: namespace,
Image: image,
ImagePullSecrets: imagePullSecrets,
})
if b.ResourceCache != nil {
if cached, ok := b.ResourceCache.autoscalingListener.Get(autoscalingRunnerSet, cacheKeyObject, ephemeralRunnerSet, inputDependency); ok {
return cached, nil
}
}
effectiveMinRunners := 0
effectiveMaxRunners := math.MaxInt32
if autoscalingRunnerSet.Spec.MaxRunners != nil {
@@ -182,10 +204,22 @@ func (b *ResourceBuilder) newAutoscalingListener(autoscalingRunnerSet *v1alpha1.
},
Spec: spec,
}
if b.ResourceCache != nil {
b.ResourceCache.autoscalingListener.Upsert(autoscalingRunnerSet, autoscalingListener, ephemeralRunnerSet, inputDependency)
}
return autoscalingListener, nil
}
func resourceCacheInputObject(name string, value any) client.Object {
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: name,
ResourceVersion: hash.ComputeTemplateHash(value),
},
}
}
type listenerMetricsServerConfig struct {
addr string
endpoint string
@@ -307,6 +341,18 @@ func (b *ResourceBuilder) newScaleSetListenerPod(
roleBinding *rbacv1.RoleBinding,
metricsConfig *listenerMetricsServerConfig,
) (*corev1.Pod, error) {
cacheKeyObject := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingListener.Name,
Namespace: autoscalingListener.Namespace,
},
}
if b.ResourceCache != nil {
if cached, ok := b.ResourceCache.listenerPod.Get(autoscalingListener, cacheKeyObject, podConfig, serviceAccount, role, roleBinding); ok {
return cached, nil
}
}
envs := []corev1.EnvVar{
{
Name: "LISTENER_CONFIG_PATH",
@@ -443,6 +489,9 @@ func (b *ResourceBuilder) newScaleSetListenerPod(
if autoscalingListener.Spec.Template != nil {
mergeListenerPodWithTemplate(newRunnerScaleSetListenerPod, autoscalingListener.Spec.Template)
}
if b.ResourceCache != nil {
b.ResourceCache.listenerPod.Upsert(autoscalingListener, newRunnerScaleSetListenerPod, podConfig, serviceAccount, role, roleBinding)
}
return newRunnerScaleSetListenerPod, nil
}
@@ -597,6 +646,18 @@ func mergeListenerContainer(base, from *corev1.Container) {
}
func (b *ResourceBuilder) newScaleSetListenerServiceAccount(autoscalingListener *v1alpha1.AutoscalingListener) (*corev1.ServiceAccount, error) {
cacheKeyObject := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingListener.Name,
Namespace: autoscalingListener.Namespace,
},
}
if b.ResourceCache != nil {
if cached, ok := b.ResourceCache.listenerServiceAccount.Get(autoscalingListener, cacheKeyObject); ok {
return cached, nil
}
}
base := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingListener.Name,
@@ -619,6 +680,9 @@ func (b *ResourceBuilder) newScaleSetListenerServiceAccount(autoscalingListener
if err := b.setControllerReference(autoscalingListener, base); err != nil {
return nil, fmt.Errorf("failed to set controller reference for listener service account: %w", err)
}
if b.ResourceCache != nil {
b.ResourceCache.listenerServiceAccount.Upsert(autoscalingListener, base)
}
return base, nil
}
@@ -640,6 +704,18 @@ func scaleSetListenerServiceAccountIntegrityHash(sa *corev1.ServiceAccount) stri
}
func (b *ResourceBuilder) newScaleSetListenerRole(autoscalingListener *v1alpha1.AutoscalingListener) *rbacv1.Role {
cacheKeyObject := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingListener.Name,
Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
},
}
if b.ResourceCache != nil {
if cached, ok := b.ResourceCache.listenerRole.Get(autoscalingListener, cacheKeyObject); ok {
return cached
}
}
labels := b.filterAndMergeLabels(autoscalingListener.Labels, map[string]string{
LabelKeyGitHubScaleSetNamespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
LabelKeyGitHubScaleSetName: autoscalingListener.Spec.AutoscalingRunnerSetName,
@@ -664,6 +740,9 @@ func (b *ResourceBuilder) newScaleSetListenerRole(autoscalingListener *v1alpha1.
}
newRole.Annotations[annotationKeyIntegrityHash] = scaleSetRoleIntegrityHash(newRole)
if b.ResourceCache != nil {
b.ResourceCache.listenerRole.Upsert(autoscalingListener, newRole)
}
return newRole
}
@@ -681,6 +760,18 @@ func scaleSetRoleIntegrityHash(role *rbacv1.Role) string {
}
func (b *ResourceBuilder) newScaleSetListenerRoleBinding(autoscalingListener *v1alpha1.AutoscalingListener, listenerRole *rbacv1.Role, serviceAccount *corev1.ServiceAccount) *rbacv1.RoleBinding {
cacheKeyObject := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingListener.Name,
Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
},
}
if b.ResourceCache != nil {
if cached, ok := b.ResourceCache.listenerRoleBinding.Get(autoscalingListener, cacheKeyObject, listenerRole, serviceAccount); ok {
return cached
}
}
roleRef := rbacv1.RoleRef{
Kind: "Role",
Name: listenerRole.Name,
@@ -719,6 +810,9 @@ func (b *ResourceBuilder) newScaleSetListenerRoleBinding(autoscalingListener *v1
}
newRoleBinding.Annotations[annotationKeyIntegrityHash] = scaleSetListenerRoleBindingIntegrityHash(newRoleBinding)
if b.ResourceCache != nil {
b.ResourceCache.listenerRoleBinding.Upsert(autoscalingListener, newRoleBinding, listenerRole, serviceAccount)
}
return newRoleBinding
}
@@ -743,6 +837,18 @@ func (b *ResourceBuilder) newEphemeralRunnerSet(autoscalingRunnerSet *v1alpha1.A
return nil, err
}
cacheKeyObject := &v1alpha1.EphemeralRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: autoscalingRunnerSet.Name,
Namespace: autoscalingRunnerSet.Namespace,
},
}
if b.ResourceCache != nil {
if cached, ok := b.ResourceCache.ephemeralRunnerSet.Get(autoscalingRunnerSet, cacheKeyObject); ok {
return cached, nil
}
}
spec := v1alpha1.EphemeralRunnerSetSpec{
Replicas: 0,
EphemeralRunnerSpec: v1alpha1.EphemeralRunnerSpec{
@@ -796,6 +902,9 @@ func (b *ResourceBuilder) newEphemeralRunnerSet(autoscalingRunnerSet *v1alpha1.A
if err := b.setControllerReference(autoscalingRunnerSet, newEphemeralRunnerSet); err != nil {
return nil, fmt.Errorf("failed to set controller reference for ephemeral runner set: %w", err)
}
if b.ResourceCache != nil {
b.ResourceCache.ephemeralRunnerSet.Upsert(autoscalingRunnerSet, newEphemeralRunnerSet)
}
return newEphemeralRunnerSet, nil
}
@@ -171,6 +171,7 @@ func TestMetadataPropagation(t *testing.T) {
ephemeralRunner, err := b.newEphemeralRunner(ephemeralRunnerSet)
require.NoError(t, err)
assert.ElementsMatch(t, []string{ephemeralRunnerFinalizerName, ephemeralRunnerActionsFinalizerName}, ephemeralRunner.Finalizers)
for _, key := range commonLabelKeys {
if key == LabelKeyKubernetesComponent {
@@ -0,0 +1,263 @@
package actionsgithubcom
import (
"reflect"
"slices"
"strings"
"sync"
"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
"github.com/actions/actions-runner-controller/hash"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)
var resourceCacheObjectTypes sync.Map
type ResourceCacheObjectRef struct {
ObjectType string
Namespace string
Name string
UID types.UID
ResourceVersion string
}
type ResourceCacheKey struct {
MainUID types.UID
Namespace string
Name string
}
type ResourceCacheValue[T client.Object] struct {
MainObject ResourceCacheObjectRef
ResourceVersion string
Dependencies []ResourceCacheObjectRef
Object T
}
type ResourceCache struct {
autoscalingListener *resourceCacheState[*v1alpha1.AutoscalingListener]
ephemeralRunnerSet *resourceCacheState[*v1alpha1.EphemeralRunnerSet]
listenerPod *resourceCacheState[*corev1.Pod]
listenerServiceAccount *resourceCacheState[*corev1.ServiceAccount]
listenerRole *resourceCacheState[*rbacv1.Role]
listenerRoleBinding *resourceCacheState[*rbacv1.RoleBinding]
}
func NewResourceCache() ResourceCache {
return ResourceCache{
autoscalingListener: newResourceCacheState[*v1alpha1.AutoscalingListener](),
ephemeralRunnerSet: newResourceCacheState[*v1alpha1.EphemeralRunnerSet](),
listenerPod: newResourceCacheState[*corev1.Pod](),
listenerServiceAccount: newResourceCacheState[*corev1.ServiceAccount](),
listenerRole: newResourceCacheState[*rbacv1.Role](),
listenerRoleBinding: newResourceCacheState[*rbacv1.RoleBinding](),
}
}
type resourceCacheState[T client.Object] struct {
mu sync.RWMutex
entries map[ResourceCacheKey]ResourceCacheValue[T]
}
func newResourceCacheState[T client.Object]() *resourceCacheState[T] {
return &resourceCacheState[T]{
entries: make(map[ResourceCacheKey]ResourceCacheValue[T], 512),
}
}
func (s *resourceCacheState[T]) Get(
mainObject client.Object,
desiredObject T,
dependencies ...client.Object,
) (T, bool) {
key := newResourceCacheKey(mainObject, desiredObject)
s.mu.RLock()
value, ok := s.entries[key]
s.mu.RUnlock()
if !ok || !value.Matches(mainObject, dependencies...) {
var zero T
return zero, false
}
return cloneResourceCacheObject(value.Object), true
}
func (s *resourceCacheState[T]) Upsert(
mainObject client.Object,
desiredObject T,
dependencies ...client.Object,
) (ResourceCacheValue[T], bool) {
key := newResourceCacheKey(mainObject, desiredObject)
mainObjectRef := newResourceCacheObjectRef(mainObject)
resourceVersion := desiredObject.GetResourceVersion()
s.mu.RLock()
previous, ok := s.entries[key]
if ok && previous.MainObject == mainObjectRef && previous.ResourceVersion == resourceVersion && previous.dependenciesMatch(dependencies...) {
s.mu.RUnlock()
return previous, false
}
s.mu.RUnlock()
s.mu.Lock()
defer s.mu.Unlock()
previous, ok = s.entries[key]
if ok && previous.MainObject == mainObjectRef && previous.ResourceVersion == resourceVersion && previous.dependenciesMatch(dependencies...) {
return previous, false
}
dependencyRefs := newResourceCacheObjectRefs(dependencies...)
value := newResourceCacheValue(mainObjectRef, resourceVersion, dependencyRefs, cloneResourceCacheObject(desiredObject))
s.entries[key] = value
return value, true
}
func (c *ResourceCache) Delete(mainObject client.Object) {
if mainObject == nil {
return
}
c.autoscalingListener.Delete(mainObject)
c.ephemeralRunnerSet.Delete(mainObject)
c.listenerPod.Delete(mainObject)
c.listenerServiceAccount.Delete(mainObject)
c.listenerRole.Delete(mainObject)
c.listenerRoleBinding.Delete(mainObject)
}
func (s *resourceCacheState[T]) Delete(mainObject client.Object) {
if mainObject == nil {
return
}
uid := mainObject.GetUID()
if uid == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
for key := range s.entries {
if key.MainUID == uid {
delete(s.entries, key)
}
}
}
func (v ResourceCacheValue[T]) Matches(mainObject client.Object, dependencies ...client.Object) bool {
if v.MainObject != newResourceCacheObjectRef(mainObject) {
return false
}
return v.dependenciesMatch(dependencies...)
}
func newResourceCacheKey(mainObject client.Object, desiredObject client.Object) ResourceCacheKey {
return ResourceCacheKey{
MainUID: mainObject.GetUID(),
Namespace: desiredObject.GetNamespace(),
Name: resourceCacheObjectName(desiredObject),
}
}
func newResourceCacheValue[T client.Object](
mainObjectRef ResourceCacheObjectRef,
resourceVersion string,
dependencyRefs []ResourceCacheObjectRef,
object T,
) ResourceCacheValue[T] {
return ResourceCacheValue[T]{
MainObject: mainObjectRef,
ResourceVersion: resourceVersion,
Dependencies: dependencyRefs,
Object: object,
}
}
func cloneResourceCacheObject[T client.Object](object T) T {
return object.DeepCopyObject().(T)
}
func newResourceCacheObjectRefs(objects ...client.Object) []ResourceCacheObjectRef {
refs := make([]ResourceCacheObjectRef, 0, len(objects))
for _, object := range objects {
refs = append(refs, newResourceCacheObjectRef(object))
}
slices.SortFunc(refs, func(a, b ResourceCacheObjectRef) int {
return compareResourceCacheObjectRefs(a, b)
})
return refs
}
func (v ResourceCacheValue[T]) dependenciesMatch(objects ...client.Object) bool {
if len(v.Dependencies) != len(objects) {
return false
}
for _, object := range objects {
ref := newResourceCacheObjectRef(object)
if !slices.Contains(v.Dependencies, ref) {
return false
}
}
return true
}
func newResourceCacheObjectRef(object client.Object) ResourceCacheObjectRef {
resourceVersion := object.GetResourceVersion()
if resourceVersion == "" {
resourceVersion = hash.ComputeTemplateHash(object)
}
return ResourceCacheObjectRef{
ObjectType: resourceCacheObjectType(object),
Namespace: object.GetNamespace(),
Name: resourceCacheObjectName(object),
UID: object.GetUID(),
ResourceVersion: resourceVersion,
}
}
func compareResourceCacheObjectRefs(a, b ResourceCacheObjectRef) int {
if c := strings.Compare(a.ObjectType, b.ObjectType); c != 0 {
return c
}
if c := strings.Compare(a.Namespace, b.Namespace); c != 0 {
return c
}
if c := strings.Compare(a.Name, b.Name); c != 0 {
return c
}
if c := strings.Compare(string(a.UID), string(b.UID)); c != 0 {
return c
}
return strings.Compare(a.ResourceVersion, b.ResourceVersion)
}
func resourceCacheObjectType(object client.Object) string {
t := reflect.TypeOf(object)
if t.Kind() == reflect.Pointer {
t = t.Elem()
}
if objectType, ok := resourceCacheObjectTypes.Load(t); ok {
return objectType.(string)
}
objectType := t.PkgPath() + "." + t.Name()
actual, _ := resourceCacheObjectTypes.LoadOrStore(t, objectType)
return actual.(string)
}
func resourceCacheObjectName(object client.Object) string {
if object.GetName() != "" {
return object.GetName()
}
return object.GetGenerateName()
}
@@ -0,0 +1,351 @@
package actionsgithubcom
import (
"fmt"
"testing"
"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
var benchmarkEphemeralRunnerSetSink *v1alpha1.EphemeralRunnerSet
func newTestResourceCache() *ResourceCache {
cache := NewResourceCache()
return &cache
}
func TestResourceCacheUpsertReplacesByDependencyResourceVersion(t *testing.T) {
mainObject := &v1alpha1.AutoscalingListener{
ObjectMeta: metav1.ObjectMeta{
Name: "listener",
Namespace: "controller-ns",
UID: "listener-uid",
ResourceVersion: "10",
},
}
desiredPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "listener",
Namespace: "controller-ns",
ResourceVersion: "1",
Labels: map[string]string{
"app": "listener",
},
},
}
configSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "listener-config",
Namespace: "controller-ns",
UID: "config-secret-uid",
ResourceVersion: "1",
},
}
serviceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "listener",
Namespace: "controller-ns",
UID: "service-account-uid",
ResourceVersion: "1",
},
}
role := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: "listener",
Namespace: "scale-set-ns",
UID: "role-uid",
ResourceVersion: "1",
},
}
cache := NewResourceCache()
value, replaced := cache.listenerPod.Upsert(mainObject, desiredPod, configSecret, serviceAccount, role)
assert.True(t, replaced)
_, ok := cache.listenerPod.Get(mainObject, desiredPod, configSecret, serviceAccount, role)
assert.True(t, ok)
assert.Equal(t, "1", value.ResourceVersion)
_, replaced = cache.listenerPod.Upsert(mainObject, desiredPod, role, configSecret, serviceAccount)
assert.False(t, replaced, "dependency ordering should not affect the cache value")
_, ok = cache.listenerPod.Get(mainObject, desiredPod, configSecret, serviceAccount, role)
assert.True(t, ok)
configSecret.ResourceVersion = "2"
value, replaced = cache.listenerPod.Upsert(mainObject, desiredPod, configSecret, serviceAccount, role)
assert.True(t, replaced)
assert.Contains(t, value.Dependencies, ResourceCacheObjectRef{
ObjectType: resourceCacheObjectType(configSecret),
Namespace: "controller-ns",
Name: "listener-config",
UID: "config-secret-uid",
ResourceVersion: "2",
})
desiredPod.Labels["mutated"] = "after-cache"
cachedPod := value.Object
assert.NotContains(t, cachedPod.Labels, "mutated")
}
func TestResourceCacheDeleteRemovesMainObjectEntries(t *testing.T) {
mainObject := &v1alpha1.AutoscalingListener{
ObjectMeta: metav1.ObjectMeta{
Name: "listener",
Namespace: "controller-ns",
UID: "listener-uid",
},
}
otherMainObject := &v1alpha1.AutoscalingListener{
ObjectMeta: metav1.ObjectMeta{
Name: "other-listener",
Namespace: "controller-ns",
UID: "other-listener-uid",
},
}
listenerPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "listener", Namespace: "controller-ns"}}
listenerServiceAccount := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "listener", Namespace: "controller-ns"}}
otherListenerPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "other-listener", Namespace: "controller-ns"}}
cache := NewResourceCache()
cache.listenerPod.Upsert(mainObject, listenerPod)
cache.listenerServiceAccount.Upsert(mainObject, listenerServiceAccount)
cache.listenerPod.Upsert(otherMainObject, otherListenerPod)
cache.Delete(mainObject)
_, ok := cache.listenerPod.Get(mainObject, listenerPod)
assert.False(t, ok)
_, ok = cache.listenerServiceAccount.Get(mainObject, listenerServiceAccount)
assert.False(t, ok)
_, ok = cache.listenerPod.Get(otherMainObject, otherListenerPod)
assert.True(t, ok)
}
func TestResourceBuilderCachesListenerPodDependencies(t *testing.T) {
listener := &v1alpha1.AutoscalingListener{
ObjectMeta: metav1.ObjectMeta{
Name: "listener",
Namespace: "controller-ns",
UID: "listener-uid",
Annotations: map[string]string{
annotationKeyIntegrityHash: "listener-hash",
},
},
Spec: v1alpha1.AutoscalingListenerSpec{
Image: "listener:latest",
AutoscalingRunnerSetName: "scale-set",
AutoscalingRunnerSetNamespace: "scale-set-ns",
EphemeralRunnerSetName: "scale-set",
},
}
podConfig := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "listener-config",
Namespace: "controller-ns",
UID: "config-secret-uid",
ResourceVersion: "11",
Annotations: map[string]string{
annotationKeyIntegrityHash: "config-hash",
},
},
}
serviceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "listener",
Namespace: "controller-ns",
UID: "service-account-uid",
ResourceVersion: "12",
Annotations: map[string]string{
annotationKeyIntegrityHash: "service-account-hash",
},
},
}
role := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: "listener",
Namespace: "scale-set-ns",
UID: "role-uid",
ResourceVersion: "13",
Annotations: map[string]string{
annotationKeyIntegrityHash: "role-hash",
},
},
}
roleBinding := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "listener",
Namespace: "scale-set-ns",
UID: "role-binding-uid",
ResourceVersion: "14",
Annotations: map[string]string{
annotationKeyIntegrityHash: "role-binding-hash",
},
},
}
cache := NewResourceCache()
b := ResourceBuilder{ResourceCache: &cache}
listenerPod, err := b.newScaleSetListenerPod(listener, podConfig, serviceAccount, role, roleBinding, nil)
require.NoError(t, err)
cachedPod, ok := b.ResourceCache.listenerPod.Get(listener, listenerPod, podConfig, serviceAccount, role, roleBinding)
require.True(t, ok)
assert.IsType(t, &corev1.Pod{}, cachedPod)
role.ResourceVersion = "changed"
_, ok = b.ResourceCache.listenerPod.Get(listener, listenerPod, podConfig, serviceAccount, role, roleBinding)
assert.False(t, ok)
}
func TestResourceBuilderCachesEphemeralRunnerSet(t *testing.T) {
autoscalingRunnerSet := v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "scale-set",
Namespace: "default",
UID: "scale-set-uid",
Annotations: map[string]string{
runnerScaleSetIDAnnotationKey: "1",
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/actions/actions-runner-controller",
},
}
cache := NewResourceCache()
b := ResourceBuilder{ResourceCache: &cache}
runnerSet, err := b.newEphemeralRunnerSet(&autoscalingRunnerSet)
require.NoError(t, err)
cachedRunnerSet, ok := b.ResourceCache.ephemeralRunnerSet.Get(&autoscalingRunnerSet, runnerSet)
require.True(t, ok)
assert.Equal(t, runnerSet.Spec, cachedRunnerSet.Spec)
runnerSet.Labels["mutated"] = "after-cache"
assert.NotContains(t, cachedRunnerSet.Labels, "mutated")
fromBuilder, err := b.newEphemeralRunnerSet(&autoscalingRunnerSet)
require.NoError(t, err)
assert.NotContains(t, fromBuilder.Labels, "mutated")
autoscalingRunnerSet.Annotations[runnerScaleSetIDAnnotationKey] = "2"
_, ok = b.ResourceCache.ephemeralRunnerSet.Get(&autoscalingRunnerSet, runnerSet)
assert.False(t, ok)
}
func BenchmarkNewEphemeralRunnerSetResourceCache(b *testing.B) {
autoscalingRunnerSet := newBenchmarkAutoscalingRunnerSet()
b.Run("no_cache", func(b *testing.B) {
builder := ResourceBuilder{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
runnerSet, err := builder.newEphemeralRunnerSet(autoscalingRunnerSet)
if err != nil {
b.Fatal(err)
}
benchmarkEphemeralRunnerSetSink = runnerSet
}
})
b.Run("cache_hit", func(b *testing.B) {
cache := NewResourceCache()
builder := ResourceBuilder{ResourceCache: &cache}
if _, err := builder.newEphemeralRunnerSet(autoscalingRunnerSet); err != nil {
b.Fatal(err)
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
runnerSet, err := builder.newEphemeralRunnerSet(autoscalingRunnerSet)
if err != nil {
b.Fatal(err)
}
benchmarkEphemeralRunnerSetSink = runnerSet
}
})
b.Run("cache_miss", func(b *testing.B) {
cache := NewResourceCache()
builder := ResourceBuilder{ResourceCache: &cache}
autoscalingRunnerSet := autoscalingRunnerSet.DeepCopy()
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
autoscalingRunnerSet.ResourceVersion = fmt.Sprint(i)
runnerSet, err := builder.newEphemeralRunnerSet(autoscalingRunnerSet)
if err != nil {
b.Fatal(err)
}
benchmarkEphemeralRunnerSetSink = runnerSet
}
})
}
func newBenchmarkAutoscalingRunnerSet() *v1alpha1.AutoscalingRunnerSet {
return &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "benchmark-scale-set",
Namespace: "benchmark-namespace",
UID: "benchmark-scale-set-uid",
ResourceVersion: "1",
Labels: map[string]string{
LabelKeyKubernetesVersion: "0.12.0",
"example.com/label-1": "value-1",
"example.com/label-2": "value-2",
},
Annotations: map[string]string{
runnerScaleSetIDAnnotationKey: "123",
AnnotationKeyGitHubRunnerGroupName: "benchmark-runner-group",
AnnotationKeyGitHubRunnerScaleSetName: "benchmark-scale-set",
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/actions/actions-runner-controller",
EphemeralRunnerSetMetadata: &v1alpha1.ResourceMeta{
Labels: map[string]string{
"example.com/runner-set-label": "runner-set-value",
},
Annotations: map[string]string{
"example.com/runner-set-annotation": "runner-set-value",
},
},
EphemeralRunnerMetadata: &v1alpha1.ResourceMeta{
Labels: map[string]string{
"example.com/runner-label": "runner-value",
},
Annotations: map[string]string{
"example.com/runner-annotation": "runner-value",
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"example.com/template-label": "template-value",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: v1alpha1.EphemeralRunnerContainerName,
Image: "ghcr.io/actions/actions-runner:latest",
Env: []corev1.EnvVar{
{Name: "ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER", Value: "false"},
},
},
},
},
},
},
}
}
+2
View File
@@ -299,10 +299,12 @@ func main() {
secretresolver.WithLogger(slogLogger),
)
resourceCache := actionsgithubcom.NewResourceCache()
rb := actionsgithubcom.ResourceBuilder{
ExcludeLabelPropagationPrefixes: excludeLabelPropagationPrefixes,
SecretResolver: secretResolver,
Scheme: mgr.GetScheme(),
ResourceCache: &resourceCache,
}
log.Info("Resource builder initializing")