mirror of
https://github.com/actions-runner-controller/actions-runner-controller.git
synced 2026-10-08 09:06:43 +02:00
The scale-up suppression read Status.FinishedRunnerCleanupPatchID off the EphemeralRunnerSet that Reconcile fetched through the manager's cached client. That marker is written by the cleanup reconcile, and it is the runner deletions performed by that same reconcile which trigger the next one, so the follow-up reconcile is regularly served from an informer cache that has not yet observed the controller's own status write. The marker read as 0, the guard did not fire, and the controller created a replacement runner for a job that had already finished. That is the exact spurious scale up the guard exists to prevent, so it is a correctness problem in a cluster and not only a flaky test. Make the decision from authoritative state. A cached hit still short circuits, because nothing ever clears the marker, so a hit cannot be a false positive. Only a miss falls through to an uncached Get via a new APIReader, which SetupWithManager fills in from mgr.GetAPIReader() so no construction site can forget it. The extra read is confined to scale-up decisions, where the controller is about to issue creates anyway. The window was transient: updateStatus copies the marker from the in-memory object and patches with MergeFrom, so a stale reconcile produces no diff for that field and cannot clobber the recorded value. The envtest spec that caught this only fails under CI load, so the regression is pinned down directly instead. TestScaleUpServicedByFinished RunnerCleanup drives the decision with a lagging cached client and an API reader that already has the write, and asserts the suppression still holds. Pointing the read back at the cached client fails that case and only that case. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
108 lines
4.5 KiB
Go
108 lines
4.5 KiB
Go
package actionsgithubcom
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
)
|
|
|
|
// TestScaleUpServicedByFinishedRunnerCleanup pins down the decision that
|
|
// suppresses scale up after finished runners were cleaned up.
|
|
//
|
|
// The interesting case is the third one. The marker is written by one reconcile
|
|
// and read back by the next, and the deletions performed by the first reconcile
|
|
// are themselves what triggers the second. That next reconcile is regularly
|
|
// served from an informer cache that has not yet observed the controller's own
|
|
// status write, so the decision must not be made from the cached copy alone. An
|
|
// envtest spec only hits that window under load, which is why this is asserted
|
|
// directly instead.
|
|
func TestScaleUpServicedByFinishedRunnerCleanup(t *testing.T) {
|
|
scheme := runtime.NewScheme()
|
|
require.NoError(t, v1alpha1.AddToScheme(scheme))
|
|
|
|
key := types.NamespacedName{Namespace: "test-ns", Name: "test-ers"}
|
|
|
|
newSet := func(specPatchID, statusCleanupPatchID int) *v1alpha1.EphemeralRunnerSet {
|
|
return &v1alpha1.EphemeralRunnerSet{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: key.Namespace, Name: key.Name},
|
|
Spec: v1alpha1.EphemeralRunnerSetSpec{
|
|
Replicas: 2,
|
|
PatchID: specPatchID,
|
|
},
|
|
Status: v1alpha1.EphemeralRunnerSetStatus{
|
|
FinishedRunnerCleanupPatchID: statusCleanupPatchID,
|
|
},
|
|
}
|
|
}
|
|
|
|
newReader := func(objects ...client.Object) client.Client {
|
|
return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build()
|
|
}
|
|
|
|
t.Run("suppresses when the cached marker already matches the spec patch ID", func(t *testing.T) {
|
|
// A cached hit needs no confirmation, so the reconciler is deliberately
|
|
// left without an APIReader: reaching for one here would be a bug.
|
|
r := &EphemeralRunnerSetReconciler{}
|
|
|
|
suppressed, err := r.scaleUpServicedByFinishedRunnerCleanup(context.Background(), key, newSet(2, 2))
|
|
require.NoError(t, err)
|
|
assert.True(t, suppressed, "the cleanup for this patch ID is already recorded, so the shortfall is not new demand")
|
|
})
|
|
|
|
t.Run("does not suppress when neither the cache nor the API server records the patch ID", func(t *testing.T) {
|
|
r := &EphemeralRunnerSetReconciler{
|
|
Client: newReader(newSet(2, 0)),
|
|
APIReader: newReader(newSet(2, 0)),
|
|
}
|
|
|
|
suppressed, err := r.scaleUpServicedByFinishedRunnerCleanup(context.Background(), key, newSet(2, 0))
|
|
require.NoError(t, err)
|
|
assert.False(t, suppressed, "no cleanup was performed for this patch ID, so the shortfall is genuine demand")
|
|
})
|
|
|
|
t.Run("suppresses when the cache is stale but the API server records the patch ID", func(t *testing.T) {
|
|
// This is the race: the cleanup reconcile wrote the marker and returned,
|
|
// and the reconcile triggered by its own deletions is still reading a
|
|
// pre-write cache. Client stands in for that lagging cache, APIReader for
|
|
// the API server that already has the write.
|
|
stale := newSet(2, 0)
|
|
r := &EphemeralRunnerSetReconciler{
|
|
Client: newReader(newSet(2, 0)),
|
|
APIReader: newReader(newSet(2, 2)),
|
|
}
|
|
|
|
suppressed, err := r.scaleUpServicedByFinishedRunnerCleanup(context.Background(), key, stale)
|
|
require.NoError(t, err)
|
|
assert.True(t, suppressed, "the decision must come from the API server, not from a cache that has not caught up")
|
|
})
|
|
|
|
t.Run("never suppresses when the listener published patch ID zero", func(t *testing.T) {
|
|
// Patch ID 0 is the idle-at-minimum state the listener republishes
|
|
// without incrementing. Suppressing on it would let a scale set sit
|
|
// below its minimum indefinitely.
|
|
r := &EphemeralRunnerSetReconciler{
|
|
Client: newReader(newSet(0, 0)),
|
|
APIReader: newReader(newSet(0, 0)),
|
|
}
|
|
|
|
suppressed, err := r.scaleUpServicedByFinishedRunnerCleanup(context.Background(), key, newSet(0, 0))
|
|
require.NoError(t, err)
|
|
assert.False(t, suppressed, "patch ID 0 must always be free to scale up")
|
|
})
|
|
|
|
t.Run("fails loudly when it cannot read past the cache", func(t *testing.T) {
|
|
r := &EphemeralRunnerSetReconciler{Client: newReader(newSet(2, 2))}
|
|
|
|
_, err := r.scaleUpServicedByFinishedRunnerCleanup(context.Background(), key, newSet(2, 0))
|
|
assert.Error(t, err, "a missing APIReader must surface rather than silently fall back to the cached marker")
|
|
})
|
|
}
|