Fix repeated pod label patches due to stale informer cache

The pod is read from the informer cache which lags behind the actual
patch, so the label guard check kept seeing the old unlabeled pod and
re-patching on every reconcile. Fix by annotating the EphemeralRunner
itself after a successful patch and using that as the idempotency guard
instead — the EphemeralRunner cache refreshes immediately since we just
patched it.

Generated with Claude Code
This commit is contained in:
Arjun Madan 2026-05-22 11:32:22 -04:00
parent 1aad1ea754
commit d8da2aa778
2 changed files with 21 additions and 1 deletions

View File

@ -842,7 +842,10 @@ func (r *EphemeralRunnerReconciler) labelPodWithJobInfo(ctx context.Context, eph
return nil
}
if pod.Labels[LabelKeyGitHubJobRepository] != "" {
// Use an annotation on the EphemeralRunner as the idempotency guard instead of
// the pod label. The pod is read from the informer cache which may lag behind the
// actual patch, causing repeated patches until the cache refreshes.
if ephemeralRunner.Annotations[AnnotationKeyGitHubJobRepository] != "" {
return nil
}
@ -869,6 +872,17 @@ func (r *EphemeralRunnerReconciler) labelPodWithJobInfo(ctx context.Context, eph
return fmt.Errorf("failed to patch pod with job info: %w", err)
}
// Annotate the EphemeralRunner so subsequent reconciles skip the patch even
// if the pod cache hasn't refreshed yet.
runnerPatch := client.MergeFrom(ephemeralRunner.DeepCopy())
if ephemeralRunner.Annotations == nil {
ephemeralRunner.Annotations = make(map[string]string)
}
ephemeralRunner.Annotations[AnnotationKeyGitHubJobRepository] = ephemeralRunner.Status.JobRepositoryName
if err := r.Patch(ctx, ephemeralRunner, runnerPatch); err != nil {
return fmt.Errorf("failed to patch ephemeral runner with job info annotation: %w", err)
}
log.Info("Patched pod with job info labels")
return nil
}

View File

@ -895,6 +895,12 @@ var _ = Describe("EphemeralRunner", func() {
Expect(updatedPod.Annotations[AnnotationKeyGitHubJobID]).To(Equal("12345"))
Expect(updatedPod.Annotations[AnnotationKeyWorkflowRunID]).To(Equal("67890"))
Expect(updatedPod.Annotations[AnnotationKeyGitHubJobRepository]).To(Equal("myorg/my-repo"))
// Verify the EphemeralRunner itself is annotated (used as idempotency guard)
updatedRunner := new(v1alpha1.EphemeralRunner)
err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updatedRunner)
Expect(err).To(BeNil())
Expect(updatedRunner.Annotations[AnnotationKeyGitHubJobRepository]).To(Equal("myorg/my-repo"))
})
It("It should not label the pod when no job is assigned", func() {