Let the listener own the EphemeralRunner Running phase transition

The controller derived Status.Phase directly from the pod phase, so a runner
became Running as soon as its pod started, whether or not it had picked up a
job. That made Running mean "the pod is up" instead of "the runner is busy",
and it left the EphemeralRunnerSet scale-down path unable to tell an idle
runner from one that is executing a job.

The listener already knows when a job is assigned to a specific runner, so
move the transition there. HandleJobStarted now reads the runner first and
only promotes it to Running when it is not terminal (Failed, Succeeded or
Outdated) and not being deleted, then patches the phase alongside the job
fields it already writes. The listener role gains "get" on ephemeralrunners
for that read.

On the controller side updateRunStatusFromPod keeps publishing the initial
Pending phase while the pod is starting, and no longer promotes to Running.
Runners waiting for work now stay Pending, so scale-down picks them before
runners that are actually executing a job.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

controllers/actions.github.com/ephemeralrunner_controller.go

@@ -834,6 +834,7 @@ func (r *EphemeralRunnerReconciler) createSecret(ctx context.Context, runner *v1
 
 // updateRunStatusFromPod is responsible for updating non-exiting statuses.
 // It should never update phase to Failed or Succeeded
+// It should never update phase to Running (the listener owns that transition)
 //
 // The event should not be re-queued since the termination status should be set
 // before proceeding with reconciliation logic
@@ -851,8 +852,15 @@ func (r *EphemeralRunnerReconciler) updateRunStatusFromPod(ctx context.Context,
 		}
 	}
 
-	phase := v1alpha1.EphemeralRunnerPhase(pod.Status.Phase)
-	phaseChanged := ephemeralRunner.Status.Phase != phase
+	phase := ephemeralRunner.Status.Phase
+	if pod.Status.Phase == corev1.PodPending && phase == "" {
+		phase = v1alpha1.EphemeralRunnerPhasePending
+	}
+
+	// The controller no longer promotes the runner to Running. The listener owns that
+	// transition and applies it when a job is assigned to this runner. The controller
+	// still publishes the initial Pending phase while the runner pod is starting.
+	phaseChanged := phase != ephemeralRunner.Status.Phase
 	readyChanged := ready != ephemeralRunner.Status.Ready
 
 	if !phaseChanged && !readyChanged {

controllers/actions.github.com/ephemeralrunner_controller_test.go

@@ -825,32 +825,47 @@ var _ = Describe("EphemeralRunner", func() {
 				ephemeralRunnerInterval,
 			).Should(BeEquivalentTo(true))
 
-			for _, phase := range []corev1.PodPhase{corev1.PodRunning, corev1.PodPending} {
-				podCopy := pod.DeepCopy()
-				pod.Status.Phase = phase
-				// set container state to force status update
-				pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{
-					Name:  v1alpha1.EphemeralRunnerContainerName,
-					State: corev1.ContainerState{},
-				})
+			podCopy := pod.DeepCopy()
+			pod.Status.Phase = corev1.PodPending
+			// set container state to force status update
+			pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{
+				Name:  v1alpha1.EphemeralRunnerContainerName,
+				State: corev1.ContainerState{},
+			})
 
-				err := k8sClient.Status().Patch(ctx, pod, client.MergeFrom(podCopy))
-				Expect(err).To(BeNil(), "failed to patch pod status")
+			err := k8sClient.Status().Patch(ctx, pod, client.MergeFrom(podCopy))
+			Expect(err).To(BeNil(), "failed to patch pod status")
 
-				var updated *v1alpha1.EphemeralRunner
-				Eventually(
-					func() (v1alpha1.EphemeralRunnerPhase, error) {
-						updated = new(v1alpha1.EphemeralRunner)
-						err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated)
-						if err != nil {
-							return "", err
-						}
-						return updated.Status.Phase, nil
-					},
-					ephemeralRunnerTimeout,
-					ephemeralRunnerInterval,
-				).Should(BeEquivalentTo(phase))
-			}
+			Eventually(
+				func() (v1alpha1.EphemeralRunnerPhase, error) {
+					updated := new(v1alpha1.EphemeralRunner)
+					err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated)
+					if err != nil {
+						return "", err
+					}
+					return updated.Status.Phase, nil
+				},
+				ephemeralRunnerTimeout,
+				ephemeralRunnerInterval,
+			).Should(BeEquivalentTo(v1alpha1.EphemeralRunnerPhasePending))
+
+			podCopy = pod.DeepCopy()
+			pod.Status.Phase = corev1.PodRunning
+			err = k8sClient.Status().Patch(ctx, pod, client.MergeFrom(podCopy))
+			Expect(err).To(BeNil(), "failed to patch pod status")
+
+			Consistently(
+				func() (v1alpha1.EphemeralRunnerPhase, error) {
+					updated := new(v1alpha1.EphemeralRunner)
+					err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated)
+					if err != nil {
+						return "", err
+					}
+					return updated.Status.Phase, nil
+				},
+				ephemeralRunnerInterval*3,
+				ephemeralRunnerInterval,
+			).Should(BeEquivalentTo(v1alpha1.EphemeralRunnerPhasePending), "controller should not set Running from pod status")
 		})
 
 		It("It should update ready based on the latest condition", func() {
@@ -1173,7 +1188,6 @@ var _ = Describe("EphemeralRunner", func() {
 				ephemeralRunnerInterval,
 			).Should(BeEquivalentTo(true))
 
-			// first set phase to running
 			pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{
 				Name: v1alpha1.EphemeralRunnerContainerName,
 				State: corev1.ContainerState{
@@ -1186,19 +1200,15 @@ var _ = Describe("EphemeralRunner", func() {
 			err := k8sClient.Status().Update(ctx, pod)
 			Expect(err).To(BeNil())
 
-			Eventually(
-				func() (v1alpha1.EphemeralRunnerPhase, error) {
-					updated := new(v1alpha1.EphemeralRunner)
-					if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated); err != nil {
-						return "", err
-					}
-					return updated.Status.Phase, nil
-				},
-				ephemeralRunnerTimeout,
-				ephemeralRunnerInterval,
-			).Should(BeEquivalentTo(v1alpha1.EphemeralRunnerPhaseRunning))
+			updated := new(v1alpha1.EphemeralRunner)
+			err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated)
+			Expect(err).To(BeNil())
+
+			original := updated.DeepCopy()
+			updated.Status.Phase = v1alpha1.EphemeralRunnerPhaseRunning
+			err = k8sClient.Status().Patch(ctx, updated, client.MergeFrom(original))
+			Expect(err).To(BeNil())
 
-			// set phase to succeeded
 			pod.Status.Phase = corev1.PodSucceeded
 			err = k8sClient.Status().Update(ctx, pod)
 			Expect(err).To(BeNil())
@@ -1214,6 +1224,60 @@ var _ = Describe("EphemeralRunner", func() {
 				ephemeralRunnerTimeout,
 			).Should(BeEquivalentTo(v1alpha1.EphemeralRunnerPhaseRunning))
 		})
+
+		It("Controller should not set Running phase from pod status - listener owns Running transition", func() {
+			pod := new(corev1.Pod)
+			Eventually(
+				func() (bool, error) {
+					if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil {
+						return false, err
+					}
+					return true, nil
+				},
+				ephemeralRunnerTimeout,
+				ephemeralRunnerInterval,
+			).Should(BeEquivalentTo(true))
+
+			pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{
+				Name: v1alpha1.EphemeralRunnerContainerName,
+				State: corev1.ContainerState{
+					Running: &corev1.ContainerStateRunning{
+						StartedAt: metav1.Now(),
+					},
+				},
+			})
+			pod.Status.Phase = corev1.PodRunning
+			pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
+				Type:               corev1.PodReady,
+				Status:             corev1.ConditionTrue,
+				LastTransitionTime: metav1.Now(),
+			})
+			err := k8sClient.Status().Update(ctx, pod)
+			Expect(err).To(BeNil())
+
+			Consistently(
+				func() (v1alpha1.EphemeralRunnerPhase, error) {
+					updated := new(v1alpha1.EphemeralRunner)
+					if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated); err != nil {
+						return "Unknown", err
+					}
+					return updated.Status.Phase, nil
+				},
+				ephemeralRunnerTimeout,
+			).Should(BeEquivalentTo(""))
+
+			updated := new(v1alpha1.EphemeralRunner)
+			Eventually(
+				func() (bool, error) {
+					if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated); err != nil {
+						return false, err
+					}
+					return updated.Status.Ready, nil
+				},
+				ephemeralRunnerTimeout,
+				ephemeralRunnerInterval,
+			).Should(BeEquivalentTo(true))
+		})
 	})
 
 	Describe("Checking the API", func() {

controllers/actions.github.com/ephemeralrunnerset_controller_test.go

@@ -1840,6 +1840,14 @@ var _ = Describe("EphemeralRunner phase metrics", func() {
 		err = k8sClient.Status().Patch(ctx, podRunning, client.MergeFrom(podPending))
 		Expect(err).NotTo(HaveOccurred(), "failed to patch pod to running")
 
+		runnerRunning := new(v1alpha1.EphemeralRunner)
+		err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, runnerRunning)
+		Expect(err).NotTo(HaveOccurred(), "failed to get ephemeral runner before listener-owned running patch")
+		runnerRunningOriginal := runnerRunning.DeepCopy()
+		runnerRunning.Status.Phase = v1alpha1.EphemeralRunnerPhaseRunning
+		err = k8sClient.Status().Patch(ctx, runnerRunning, client.MergeFrom(runnerRunningOriginal))
+		Expect(err).NotTo(HaveOccurred(), "failed to simulate listener running phase patch")
+
 		_, err = controller.Reconcile(ctx, request)
 		Expect(err).NotTo(HaveOccurred(), "failed to reconcile running pod")
 		expectEphemeralRunnerPhase(ctx, ephemeralRunner, v1alpha1.EphemeralRunnerPhaseRunning)

controllers/actions.github.com/resourcebuilder.go

@@ -1022,7 +1022,12 @@ func rulesForListenerRole(resourceNames []string) []rbacv1.PolicyRule {
 		},
 		{
 			APIGroups: []string{"actions.github.com"},
-			Resources: []string{"ephemeralrunners", "ephemeralrunners/status"},
+			Resources: []string{"ephemeralrunners"},
+			Verbs:     []string{"get", "patch"},
+		},
+		{
+			APIGroups: []string{"actions.github.com"},
+			Resources: []string{"ephemeralrunners/status"},
 			Verbs:     []string{"patch"},
 		},
 	}