Wrap errors in controller helper methods and swap logic in cleanups (#3960)
This commit is contained in:
parent
7a5996f467
commit
2dab45c373
|
|
@ -284,15 +284,14 @@ func (r *AutoscalingListenerReconciler) cleanupResources(ctx context.Context, au
|
||||||
if listenerPod.ObjectMeta.DeletionTimestamp.IsZero() {
|
if listenerPod.ObjectMeta.DeletionTimestamp.IsZero() {
|
||||||
logger.Info("Deleting the listener pod")
|
logger.Info("Deleting the listener pod")
|
||||||
if err := r.Delete(ctx, listenerPod); err != nil {
|
if err := r.Delete(ctx, listenerPod); err != nil {
|
||||||
return false, fmt.Errorf("failed to delete listener pod: %v", err)
|
return false, fmt.Errorf("failed to delete listener pod: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, nil
|
return false, nil
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case kerrors.IsNotFound(err):
|
||||||
return false, fmt.Errorf("failed to get listener pods: %v", err)
|
|
||||||
|
|
||||||
default: // NOT FOUND
|
|
||||||
_ = r.publishRunningListener(autoscalingListener, false) // If error is returned, we never published metrics so it is safe to ignore
|
_ = r.publishRunningListener(autoscalingListener, false) // If error is returned, we never published metrics so it is safe to ignore
|
||||||
|
default:
|
||||||
|
return false, fmt.Errorf("failed to get listener pods: %w", err)
|
||||||
}
|
}
|
||||||
logger.Info("Listener pod is deleted")
|
logger.Info("Listener pod is deleted")
|
||||||
|
|
||||||
|
|
@ -303,12 +302,12 @@ func (r *AutoscalingListenerReconciler) cleanupResources(ctx context.Context, au
|
||||||
if secret.ObjectMeta.DeletionTimestamp.IsZero() {
|
if secret.ObjectMeta.DeletionTimestamp.IsZero() {
|
||||||
logger.Info("Deleting the listener config secret")
|
logger.Info("Deleting the listener config secret")
|
||||||
if err := r.Delete(ctx, &secret); err != nil {
|
if err := r.Delete(ctx, &secret); err != nil {
|
||||||
return false, fmt.Errorf("failed to delete listener config secret: %v", err)
|
return false, fmt.Errorf("failed to delete listener config secret: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, nil
|
return false, nil
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case !kerrors.IsNotFound(err):
|
||||||
return false, fmt.Errorf("failed to get listener config secret: %v", err)
|
return false, fmt.Errorf("failed to get listener config secret: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if autoscalingListener.Spec.Proxy != nil {
|
if autoscalingListener.Spec.Proxy != nil {
|
||||||
|
|
@ -320,12 +319,12 @@ func (r *AutoscalingListenerReconciler) cleanupResources(ctx context.Context, au
|
||||||
if proxySecret.ObjectMeta.DeletionTimestamp.IsZero() {
|
if proxySecret.ObjectMeta.DeletionTimestamp.IsZero() {
|
||||||
logger.Info("Deleting the listener proxy secret")
|
logger.Info("Deleting the listener proxy secret")
|
||||||
if err := r.Delete(ctx, proxySecret); err != nil {
|
if err := r.Delete(ctx, proxySecret); err != nil {
|
||||||
return false, fmt.Errorf("failed to delete listener proxy secret: %v", err)
|
return false, fmt.Errorf("failed to delete listener proxy secret: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, nil
|
return false, nil
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case !kerrors.IsNotFound(err):
|
||||||
return false, fmt.Errorf("failed to get listener proxy secret: %v", err)
|
return false, fmt.Errorf("failed to get listener proxy secret: %w", err)
|
||||||
}
|
}
|
||||||
logger.Info("Listener proxy secret is deleted")
|
logger.Info("Listener proxy secret is deleted")
|
||||||
}
|
}
|
||||||
|
|
@ -337,12 +336,12 @@ func (r *AutoscalingListenerReconciler) cleanupResources(ctx context.Context, au
|
||||||
if listenerRoleBinding.ObjectMeta.DeletionTimestamp.IsZero() {
|
if listenerRoleBinding.ObjectMeta.DeletionTimestamp.IsZero() {
|
||||||
logger.Info("Deleting the listener role binding")
|
logger.Info("Deleting the listener role binding")
|
||||||
if err := r.Delete(ctx, listenerRoleBinding); err != nil {
|
if err := r.Delete(ctx, listenerRoleBinding); err != nil {
|
||||||
return false, fmt.Errorf("failed to delete listener role binding: %v", err)
|
return false, fmt.Errorf("failed to delete listener role binding: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, nil
|
return false, nil
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case !kerrors.IsNotFound(err):
|
||||||
return false, fmt.Errorf("failed to get listener role binding: %v", err)
|
return false, fmt.Errorf("failed to get listener role binding: %w", err)
|
||||||
}
|
}
|
||||||
logger.Info("Listener role binding is deleted")
|
logger.Info("Listener role binding is deleted")
|
||||||
|
|
||||||
|
|
@ -353,12 +352,12 @@ func (r *AutoscalingListenerReconciler) cleanupResources(ctx context.Context, au
|
||||||
if listenerRole.ObjectMeta.DeletionTimestamp.IsZero() {
|
if listenerRole.ObjectMeta.DeletionTimestamp.IsZero() {
|
||||||
logger.Info("Deleting the listener role")
|
logger.Info("Deleting the listener role")
|
||||||
if err := r.Delete(ctx, listenerRole); err != nil {
|
if err := r.Delete(ctx, listenerRole); err != nil {
|
||||||
return false, fmt.Errorf("failed to delete listener role: %v", err)
|
return false, fmt.Errorf("failed to delete listener role: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, nil
|
return false, nil
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case !kerrors.IsNotFound(err):
|
||||||
return false, fmt.Errorf("failed to get listener role: %v", err)
|
return false, fmt.Errorf("failed to get listener role: %w", err)
|
||||||
}
|
}
|
||||||
logger.Info("Listener role is deleted")
|
logger.Info("Listener role is deleted")
|
||||||
|
|
||||||
|
|
@ -370,12 +369,12 @@ func (r *AutoscalingListenerReconciler) cleanupResources(ctx context.Context, au
|
||||||
if listenerSa.ObjectMeta.DeletionTimestamp.IsZero() {
|
if listenerSa.ObjectMeta.DeletionTimestamp.IsZero() {
|
||||||
logger.Info("Deleting the listener service account")
|
logger.Info("Deleting the listener service account")
|
||||||
if err := r.Delete(ctx, listenerSa); err != nil {
|
if err := r.Delete(ctx, listenerSa); err != nil {
|
||||||
return false, fmt.Errorf("failed to delete listener service account: %v", err)
|
return false, fmt.Errorf("failed to delete listener service account: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, nil
|
return false, nil
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case !kerrors.IsNotFound(err):
|
||||||
return false, fmt.Errorf("failed to get listener service account: %v", err)
|
return false, fmt.Errorf("failed to get listener service account: %w", err)
|
||||||
}
|
}
|
||||||
logger.Info("Listener service account is deleted")
|
logger.Info("Listener service account is deleted")
|
||||||
|
|
||||||
|
|
@ -447,7 +446,7 @@ func (r *AutoscalingListenerReconciler) createListenerPod(ctx context.Context, a
|
||||||
var err error
|
var err error
|
||||||
cert, err = r.certificate(ctx, autoscalingRunnerSet, autoscalingListener)
|
cert, err = r.certificate(ctx, autoscalingRunnerSet, autoscalingListener)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ctrl.Result{}, fmt.Errorf("failed to create certificate env var for listener: %v", err)
|
return ctrl.Result{}, fmt.Errorf("failed to create certificate env var for listener: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -335,12 +335,12 @@ func (r *AutoscalingRunnerSetReconciler) cleanupListener(ctx context.Context, au
|
||||||
if listener.ObjectMeta.DeletionTimestamp.IsZero() {
|
if listener.ObjectMeta.DeletionTimestamp.IsZero() {
|
||||||
logger.Info("Deleting the listener")
|
logger.Info("Deleting the listener")
|
||||||
if err := r.Delete(ctx, &listener); err != nil {
|
if err := r.Delete(ctx, &listener); err != nil {
|
||||||
return false, fmt.Errorf("failed to delete listener: %v", err)
|
return false, fmt.Errorf("failed to delete listener: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, nil
|
return false, nil
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case !kerrors.IsNotFound(err):
|
||||||
return false, fmt.Errorf("failed to get listener: %v", err)
|
return false, fmt.Errorf("failed to get listener: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Info("Listener is deleted")
|
logger.Info("Listener is deleted")
|
||||||
|
|
@ -351,7 +351,7 @@ func (r *AutoscalingRunnerSetReconciler) cleanupEphemeralRunnerSets(ctx context.
|
||||||
logger.Info("Cleaning up ephemeral runner sets")
|
logger.Info("Cleaning up ephemeral runner sets")
|
||||||
runnerSets, err := r.listEphemeralRunnerSets(ctx, autoscalingRunnerSet)
|
runnerSets, err := r.listEphemeralRunnerSets(ctx, autoscalingRunnerSet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("failed to list ephemeral runner sets: %v", err)
|
return false, fmt.Errorf("failed to list ephemeral runner sets: %w", err)
|
||||||
}
|
}
|
||||||
if runnerSets.empty() {
|
if runnerSets.empty() {
|
||||||
logger.Info("All ephemeral runner sets are deleted")
|
logger.Info("All ephemeral runner sets are deleted")
|
||||||
|
|
@ -360,7 +360,7 @@ func (r *AutoscalingRunnerSetReconciler) cleanupEphemeralRunnerSets(ctx context.
|
||||||
|
|
||||||
logger.Info("Deleting all ephemeral runner sets", "count", runnerSets.count())
|
logger.Info("Deleting all ephemeral runner sets", "count", runnerSets.count())
|
||||||
if err := r.deleteEphemeralRunnerSets(ctx, runnerSets.all(), logger); err != nil {
|
if err := r.deleteEphemeralRunnerSets(ctx, runnerSets.all(), logger); err != nil {
|
||||||
return false, fmt.Errorf("failed to delete ephemeral runner sets: %v", err)
|
return false, fmt.Errorf("failed to delete ephemeral runner sets: %w", err)
|
||||||
}
|
}
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
@ -375,7 +375,7 @@ func (r *AutoscalingRunnerSetReconciler) deleteEphemeralRunnerSets(ctx context.C
|
||||||
}
|
}
|
||||||
logger.Info("Deleting ephemeral runner set", "name", rs.Name)
|
logger.Info("Deleting ephemeral runner set", "name", rs.Name)
|
||||||
if err := r.Delete(ctx, rs); err != nil {
|
if err := r.Delete(ctx, rs); err != nil {
|
||||||
return fmt.Errorf("failed to delete EphemeralRunnerSet resource: %v", err)
|
return fmt.Errorf("failed to delete EphemeralRunnerSet resource: %w", err)
|
||||||
}
|
}
|
||||||
logger.Info("Deleted ephemeral runner set", "name", rs.Name)
|
logger.Info("Deleted ephemeral runner set", "name", rs.Name)
|
||||||
}
|
}
|
||||||
|
|
@ -670,7 +670,7 @@ func (r *AutoscalingRunnerSetReconciler) createAutoScalingListenerForRunnerSet(c
|
||||||
func (r *AutoscalingRunnerSetReconciler) listEphemeralRunnerSets(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) (*EphemeralRunnerSets, error) {
|
func (r *AutoscalingRunnerSetReconciler) listEphemeralRunnerSets(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) (*EphemeralRunnerSets, error) {
|
||||||
list := new(v1alpha1.EphemeralRunnerSetList)
|
list := new(v1alpha1.EphemeralRunnerSetList)
|
||||||
if err := r.List(ctx, list, client.InNamespace(autoscalingRunnerSet.Namespace), client.MatchingFields{resourceOwnerKey: autoscalingRunnerSet.Name}); err != nil {
|
if err := r.List(ctx, list, client.InNamespace(autoscalingRunnerSet.Namespace), client.MatchingFields{resourceOwnerKey: autoscalingRunnerSet.Name}); err != nil {
|
||||||
return nil, fmt.Errorf("failed to list ephemeral runner sets: %v", err)
|
return nil, fmt.Errorf("failed to list ephemeral runner sets: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &EphemeralRunnerSets{list: list}, nil
|
return &EphemeralRunnerSets{list: list}, nil
|
||||||
|
|
@ -814,7 +814,7 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRol
|
||||||
}
|
}
|
||||||
c.logger.Info("Removed finalizer from container mode kubernetes role binding", "name", roleBindingName)
|
c.logger.Info("Removed finalizer from container mode kubernetes role binding", "name", roleBindingName)
|
||||||
return
|
return
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case !kerrors.IsNotFound(err):
|
||||||
c.err = fmt.Errorf("failed to fetch kubernetes mode role binding: %w", err)
|
c.err = fmt.Errorf("failed to fetch kubernetes mode role binding: %w", err)
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
|
|
@ -856,11 +856,11 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRol
|
||||||
}
|
}
|
||||||
c.logger.Info("Removed finalizer from container mode kubernetes role")
|
c.logger.Info("Removed finalizer from container mode kubernetes role")
|
||||||
return
|
return
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case kerrors.IsNotFound(err):
|
||||||
c.err = fmt.Errorf("failed to fetch kubernetes mode role: %w", err)
|
c.logger.Info("Container mode kubernetes role has already been deleted", "name", roleName)
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
c.logger.Info("Container mode kubernetes role has already been deleted", "name", roleName)
|
c.err = fmt.Errorf("failed to fetch kubernetes mode role: %w", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -899,11 +899,11 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeSer
|
||||||
}
|
}
|
||||||
c.logger.Info("Removed finalizer from container mode kubernetes service account")
|
c.logger.Info("Removed finalizer from container mode kubernetes service account")
|
||||||
return
|
return
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case kerrors.IsNotFound(err):
|
||||||
c.err = fmt.Errorf("failed to fetch kubernetes mode service account: %w", err)
|
c.logger.Info("Container mode kubernetes service account has already been deleted", "name", serviceAccountName)
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
c.logger.Info("Container mode kubernetes service account has already been deleted", "name", serviceAccountName)
|
c.err = fmt.Errorf("failed to fetch kubernetes mode service account: %w", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -942,11 +942,11 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeNoPermissionServi
|
||||||
}
|
}
|
||||||
c.logger.Info("Removed finalizer from no permission service account", "name", serviceAccountName)
|
c.logger.Info("Removed finalizer from no permission service account", "name", serviceAccountName)
|
||||||
return
|
return
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case kerrors.IsNotFound(err):
|
||||||
c.err = fmt.Errorf("failed to fetch service account: %w", err)
|
c.logger.Info("No permission service account has already been deleted", "name", serviceAccountName)
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
c.logger.Info("No permission service account has already been deleted", "name", serviceAccountName)
|
c.err = fmt.Errorf("failed to fetch service account: %w", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -985,11 +985,11 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeGitHubSecretFinal
|
||||||
}
|
}
|
||||||
c.logger.Info("Removed finalizer from GitHub secret", "name", githubSecretName)
|
c.logger.Info("Removed finalizer from GitHub secret", "name", githubSecretName)
|
||||||
return
|
return
|
||||||
case err != nil && !kerrors.IsNotFound(err) && !kerrors.IsForbidden(err):
|
case kerrors.IsNotFound(err) || kerrors.IsForbidden(err):
|
||||||
c.err = fmt.Errorf("failed to fetch GitHub secret: %w", err)
|
c.logger.Info("GitHub secret has already been deleted", "name", githubSecretName)
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
c.logger.Info("GitHub secret has already been deleted", "name", githubSecretName)
|
c.err = fmt.Errorf("failed to fetch GitHub secret: %w", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1028,11 +1028,11 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleBindin
|
||||||
}
|
}
|
||||||
c.logger.Info("Removed finalizer from manager role binding", "name", managerRoleBindingName)
|
c.logger.Info("Removed finalizer from manager role binding", "name", managerRoleBindingName)
|
||||||
return
|
return
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case kerrors.IsNotFound(err):
|
||||||
c.err = fmt.Errorf("failed to fetch manager role binding: %w", err)
|
c.logger.Info("Manager role binding has already been deleted", "name", managerRoleBindingName)
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
c.logger.Info("Manager role binding has already been deleted", "name", managerRoleBindingName)
|
c.err = fmt.Errorf("failed to fetch manager role binding: %w", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1071,11 +1071,11 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleFinali
|
||||||
}
|
}
|
||||||
c.logger.Info("Removed finalizer from manager role", "name", managerRoleName)
|
c.logger.Info("Removed finalizer from manager role", "name", managerRoleName)
|
||||||
return
|
return
|
||||||
case err != nil && !kerrors.IsNotFound(err):
|
case kerrors.IsNotFound(err):
|
||||||
c.err = fmt.Errorf("failed to fetch manager role: %w", err)
|
c.logger.Info("Manager role has already been deleted", "name", managerRoleName)
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
c.logger.Info("Manager role has already been deleted", "name", managerRoleName)
|
c.err = fmt.Errorf("failed to fetch manager role: %w", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -344,7 +344,7 @@ func (r *EphemeralRunnerReconciler) cleanupResources(ctx context.Context, epheme
|
||||||
if pod.ObjectMeta.DeletionTimestamp.IsZero() {
|
if pod.ObjectMeta.DeletionTimestamp.IsZero() {
|
||||||
log.Info("Deleting the runner pod")
|
log.Info("Deleting the runner pod")
|
||||||
if err := r.Delete(ctx, pod); err != nil && !kerrors.IsNotFound(err) {
|
if err := r.Delete(ctx, pod); err != nil && !kerrors.IsNotFound(err) {
|
||||||
return false, fmt.Errorf("failed to delete pod: %v", err)
|
return false, fmt.Errorf("failed to delete pod: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, nil
|
return false, nil
|
||||||
|
|
@ -361,7 +361,7 @@ func (r *EphemeralRunnerReconciler) cleanupResources(ctx context.Context, epheme
|
||||||
if secret.ObjectMeta.DeletionTimestamp.IsZero() {
|
if secret.ObjectMeta.DeletionTimestamp.IsZero() {
|
||||||
log.Info("Deleting the jitconfig secret")
|
log.Info("Deleting the jitconfig secret")
|
||||||
if err := r.Delete(ctx, secret); err != nil && !kerrors.IsNotFound(err) {
|
if err := r.Delete(ctx, secret); err != nil && !kerrors.IsNotFound(err) {
|
||||||
return false, fmt.Errorf("failed to delete secret: %v", err)
|
return false, fmt.Errorf("failed to delete secret: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, nil
|
return false, nil
|
||||||
|
|
@ -377,7 +377,7 @@ func (r *EphemeralRunnerReconciler) cleanupContainerHooksResources(ctx context.C
|
||||||
log.Info("Cleaning up runner linked pods")
|
log.Info("Cleaning up runner linked pods")
|
||||||
done, err = r.cleanupRunnerLinkedPods(ctx, ephemeralRunner, log)
|
done, err = r.cleanupRunnerLinkedPods(ctx, ephemeralRunner, log)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("failed to clean up runner linked pods: %v", err)
|
return false, fmt.Errorf("failed to clean up runner linked pods: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !done {
|
if !done {
|
||||||
|
|
@ -402,7 +402,7 @@ func (r *EphemeralRunnerReconciler) cleanupRunnerLinkedPods(ctx context.Context,
|
||||||
var runnerLinkedPodList corev1.PodList
|
var runnerLinkedPodList corev1.PodList
|
||||||
err = r.List(ctx, &runnerLinkedPodList, client.InNamespace(ephemeralRunner.Namespace), runnerLinedLabels)
|
err = r.List(ctx, &runnerLinkedPodList, client.InNamespace(ephemeralRunner.Namespace), runnerLinedLabels)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("failed to list runner-linked pods: %v", err)
|
return false, fmt.Errorf("failed to list runner-linked pods: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(runnerLinkedPodList.Items) == 0 {
|
if len(runnerLinkedPodList.Items) == 0 {
|
||||||
|
|
@ -421,7 +421,7 @@ func (r *EphemeralRunnerReconciler) cleanupRunnerLinkedPods(ctx context.Context,
|
||||||
|
|
||||||
log.Info("Deleting container hooks runner-linked pod", "name", linkedPod.Name)
|
log.Info("Deleting container hooks runner-linked pod", "name", linkedPod.Name)
|
||||||
if err := r.Delete(ctx, linkedPod); err != nil && !kerrors.IsNotFound(err) {
|
if err := r.Delete(ctx, linkedPod); err != nil && !kerrors.IsNotFound(err) {
|
||||||
errs = append(errs, fmt.Errorf("failed to delete runner linked pod %q: %v", linkedPod.Name, err))
|
errs = append(errs, fmt.Errorf("failed to delete runner linked pod %q: %w", linkedPod.Name, err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -456,7 +456,7 @@ func (r *EphemeralRunnerReconciler) cleanupRunnerLinkedSecrets(ctx context.Conte
|
||||||
|
|
||||||
log.Info("Deleting container hooks runner-linked secret", "name", s.Name)
|
log.Info("Deleting container hooks runner-linked secret", "name", s.Name)
|
||||||
if err := r.Delete(ctx, s); err != nil && !kerrors.IsNotFound(err) {
|
if err := r.Delete(ctx, s); err != nil && !kerrors.IsNotFound(err) {
|
||||||
errs = append(errs, fmt.Errorf("failed to delete runner linked secret %q: %v", s.Name, err))
|
errs = append(errs, fmt.Errorf("failed to delete runner linked secret %q: %w", s.Name, err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -470,12 +470,12 @@ func (r *EphemeralRunnerReconciler) markAsFailed(ctx context.Context, ephemeralR
|
||||||
obj.Status.Reason = reason
|
obj.Status.Reason = reason
|
||||||
obj.Status.Message = errMessage
|
obj.Status.Message = errMessage
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return fmt.Errorf("failed to update ephemeral runner status Phase/Message: %v", err)
|
return fmt.Errorf("failed to update ephemeral runner status Phase/Message: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Removing the runner from the service")
|
log.Info("Removing the runner from the service")
|
||||||
if err := r.deleteRunnerFromService(ctx, ephemeralRunner, log); err != nil {
|
if err := r.deleteRunnerFromService(ctx, ephemeralRunner, log); err != nil {
|
||||||
return fmt.Errorf("failed to remove the runner from service: %v", err)
|
return fmt.Errorf("failed to remove the runner from service: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("EphemeralRunner is marked as Failed and deleted from the service")
|
log.Info("EphemeralRunner is marked as Failed and deleted from the service")
|
||||||
|
|
@ -487,7 +487,7 @@ func (r *EphemeralRunnerReconciler) markAsFinished(ctx context.Context, ephemera
|
||||||
if err := patchSubResource(ctx, r.Status(), ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) {
|
if err := patchSubResource(ctx, r.Status(), ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) {
|
||||||
obj.Status.Phase = corev1.PodSucceeded
|
obj.Status.Phase = corev1.PodSucceeded
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return fmt.Errorf("failed to update ephemeral runner with status finished: %v", err)
|
return fmt.Errorf("failed to update ephemeral runner with status finished: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("EphemeralRunner status is marked as Finished")
|
log.Info("EphemeralRunner status is marked as Finished")
|
||||||
|
|
@ -500,7 +500,7 @@ func (r *EphemeralRunnerReconciler) deletePodAsFailed(ctx context.Context, ephem
|
||||||
if pod.ObjectMeta.DeletionTimestamp.IsZero() {
|
if pod.ObjectMeta.DeletionTimestamp.IsZero() {
|
||||||
log.Info("Deleting the ephemeral runner pod", "podId", pod.UID)
|
log.Info("Deleting the ephemeral runner pod", "podId", pod.UID)
|
||||||
if err := r.Delete(ctx, pod); err != nil && !kerrors.IsNotFound(err) {
|
if err := r.Delete(ctx, pod); err != nil && !kerrors.IsNotFound(err) {
|
||||||
return fmt.Errorf("failed to delete pod with status failed: %v", err)
|
return fmt.Errorf("failed to delete pod with status failed: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -514,7 +514,7 @@ func (r *EphemeralRunnerReconciler) deletePodAsFailed(ctx context.Context, ephem
|
||||||
obj.Status.Reason = pod.Status.Reason
|
obj.Status.Reason = pod.Status.Reason
|
||||||
obj.Status.Message = pod.Status.Message
|
obj.Status.Message = pod.Status.Message
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return fmt.Errorf("failed to update ephemeral runner status: failed attempts: %v", err)
|
return fmt.Errorf("failed to update ephemeral runner status: failed attempts: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("EphemeralRunner pod is deleted and status is updated with failure count")
|
log.Info("EphemeralRunner pod is deleted and status is updated with failure count")
|
||||||
|
|
@ -528,7 +528,7 @@ func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Con
|
||||||
log.Info("Creating ephemeral runner JIT config")
|
log.Info("Creating ephemeral runner JIT config")
|
||||||
actionsClient, err := r.actionsClientFor(ctx, ephemeralRunner)
|
actionsClient, err := r.actionsClientFor(ctx, ephemeralRunner)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &ctrl.Result{}, fmt.Errorf("failed to get actions client for generating JIT config: %v", err)
|
return &ctrl.Result{}, fmt.Errorf("failed to get actions client for generating JIT config: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
jitSettings := &actions.RunnerScaleSetJitRunnerSetting{
|
jitSettings := &actions.RunnerScaleSetJitRunnerSetting{
|
||||||
|
|
@ -546,12 +546,12 @@ func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Con
|
||||||
if err != nil {
|
if err != nil {
|
||||||
actionsError := &actions.ActionsError{}
|
actionsError := &actions.ActionsError{}
|
||||||
if !errors.As(err, &actionsError) {
|
if !errors.As(err, &actionsError) {
|
||||||
return &ctrl.Result{}, fmt.Errorf("failed to generate JIT config with generic error: %v", err)
|
return &ctrl.Result{}, fmt.Errorf("failed to generate JIT config with generic error: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if actionsError.StatusCode != http.StatusConflict ||
|
if actionsError.StatusCode != http.StatusConflict ||
|
||||||
!actionsError.IsException("AgentExistsException") {
|
!actionsError.IsException("AgentExistsException") {
|
||||||
return &ctrl.Result{}, fmt.Errorf("failed to generate JIT config with Actions service error: %v", err)
|
return &ctrl.Result{}, fmt.Errorf("failed to generate JIT config with Actions service error: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// If the runner with the name we want already exists it means:
|
// If the runner with the name we want already exists it means:
|
||||||
|
|
@ -564,7 +564,7 @@ func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Con
|
||||||
log.Info("Getting runner jit config failed with conflict error, trying to get the runner by name", "runnerName", ephemeralRunner.Name)
|
log.Info("Getting runner jit config failed with conflict error, trying to get the runner by name", "runnerName", ephemeralRunner.Name)
|
||||||
existingRunner, err := actionsClient.GetRunnerByName(ctx, ephemeralRunner.Name)
|
existingRunner, err := actionsClient.GetRunnerByName(ctx, ephemeralRunner.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &ctrl.Result{}, fmt.Errorf("failed to get runner by name: %v", err)
|
return &ctrl.Result{}, fmt.Errorf("failed to get runner by name: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if existingRunner == nil {
|
if existingRunner == nil {
|
||||||
|
|
@ -577,7 +577,7 @@ func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Con
|
||||||
log.Info("Removing the runner with the same name")
|
log.Info("Removing the runner with the same name")
|
||||||
err := actionsClient.RemoveRunner(ctx, int64(existingRunner.Id))
|
err := actionsClient.RemoveRunner(ctx, int64(existingRunner.Id))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &ctrl.Result{}, fmt.Errorf("failed to remove runner from the service: %v", err)
|
return &ctrl.Result{}, fmt.Errorf("failed to remove runner from the service: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Removed the runner with the same name, re-queuing the reconciliation")
|
log.Info("Removed the runner with the same name, re-queuing the reconciliation")
|
||||||
|
|
@ -586,7 +586,7 @@ func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Con
|
||||||
|
|
||||||
// TODO: Do we want to mark the ephemeral runner as failed, and let EphemeralRunnerSet to clean it up, so we can recover from this situation?
|
// TODO: Do we want to mark the ephemeral runner as failed, and let EphemeralRunnerSet to clean it up, so we can recover from this situation?
|
||||||
// The situation is that the EphemeralRunner's name is already used by something else to register a runner, and we can't take the control back.
|
// The situation is that the EphemeralRunner's name is already used by something else to register a runner, and we can't take the control back.
|
||||||
return &ctrl.Result{}, fmt.Errorf("runner with the same name but doesn't belong to this RunnerScaleSet: %v", err)
|
return &ctrl.Result{}, fmt.Errorf("runner with the same name but doesn't belong to this RunnerScaleSet: %w", err)
|
||||||
}
|
}
|
||||||
log.Info("Created ephemeral runner JIT config", "runnerId", jitConfig.Runner.Id)
|
log.Info("Created ephemeral runner JIT config", "runnerId", jitConfig.Runner.Id)
|
||||||
|
|
||||||
|
|
@ -597,7 +597,7 @@ func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Con
|
||||||
obj.Status.RunnerJITConfig = jitConfig.EncodedJITConfig
|
obj.Status.RunnerJITConfig = jitConfig.EncodedJITConfig
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &ctrl.Result{}, fmt.Errorf("failed to update runner status for RunnerId/RunnerName/RunnerJITConfig: %v", err)
|
return &ctrl.Result{}, fmt.Errorf("failed to update runner status for RunnerId/RunnerName/RunnerJITConfig: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// We want to continue without a requeue for faster pod creation.
|
// We want to continue without a requeue for faster pod creation.
|
||||||
|
|
@ -691,12 +691,12 @@ func (r *EphemeralRunnerReconciler) createSecret(ctx context.Context, runner *v1
|
||||||
jitSecret := r.ResourceBuilder.newEphemeralRunnerJitSecret(runner)
|
jitSecret := r.ResourceBuilder.newEphemeralRunnerJitSecret(runner)
|
||||||
|
|
||||||
if err := ctrl.SetControllerReference(runner, jitSecret, r.Scheme); err != nil {
|
if err := ctrl.SetControllerReference(runner, jitSecret, r.Scheme); err != nil {
|
||||||
return &ctrl.Result{}, fmt.Errorf("failed to set controller reference: %v", err)
|
return &ctrl.Result{}, fmt.Errorf("failed to set controller reference: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Created new secret spec for ephemeral runner")
|
log.Info("Created new secret spec for ephemeral runner")
|
||||||
if err := r.Create(ctx, jitSecret); err != nil {
|
if err := r.Create(ctx, jitSecret); err != nil {
|
||||||
return &ctrl.Result{}, fmt.Errorf("failed to create jit secret: %v", err)
|
return &ctrl.Result{}, fmt.Errorf("failed to create jit secret: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Created ephemeral runner secret", "secretName", jitSecret.Name)
|
log.Info("Created ephemeral runner secret", "secretName", jitSecret.Name)
|
||||||
|
|
@ -743,7 +743,7 @@ func (r *EphemeralRunnerReconciler) updateRunStatusFromPod(ctx context.Context,
|
||||||
obj.Status.Message = pod.Status.Message
|
obj.Status.Message = pod.Status.Message
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to update runner status for Phase/Reason/Message/Ready: %v", err)
|
return fmt.Errorf("failed to update runner status for Phase/Reason/Message/Ready: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Updated ephemeral runner status")
|
log.Info("Updated ephemeral runner status")
|
||||||
|
|
@ -835,7 +835,7 @@ func (r EphemeralRunnerReconciler) runnerRegisteredWithService(ctx context.Conte
|
||||||
|
|
||||||
if actionsError.StatusCode != http.StatusNotFound ||
|
if actionsError.StatusCode != http.StatusNotFound ||
|
||||||
!actionsError.IsException("AgentNotFoundException") {
|
!actionsError.IsException("AgentNotFoundException") {
|
||||||
return false, fmt.Errorf("failed to check if runner exists in GitHub service: %v", err)
|
return false, fmt.Errorf("failed to check if runner exists in GitHub service: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Runner does not exist in GitHub service", "runnerId", runner.Status.RunnerId)
|
log.Info("Runner does not exist in GitHub service", "runnerId", runner.Status.RunnerId)
|
||||||
|
|
@ -849,7 +849,7 @@ func (r EphemeralRunnerReconciler) runnerRegisteredWithService(ctx context.Conte
|
||||||
func (r *EphemeralRunnerReconciler) deleteRunnerFromService(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner, log logr.Logger) error {
|
func (r *EphemeralRunnerReconciler) deleteRunnerFromService(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner, log logr.Logger) error {
|
||||||
client, err := r.actionsClientFor(ctx, ephemeralRunner)
|
client, err := r.actionsClientFor(ctx, ephemeralRunner)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to get actions client for runner: %v", err)
|
return fmt.Errorf("failed to get actions client for runner: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Removing runner from the service", "runnerId", ephemeralRunner.Status.RunnerId)
|
log.Info("Removing runner from the service", "runnerId", ephemeralRunner.Status.RunnerId)
|
||||||
|
|
|
||||||
|
|
@ -275,7 +275,7 @@ func (r *EphemeralRunnerSetReconciler) cleanUpProxySecret(ctx context.Context, e
|
||||||
proxySecret.Name = proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet)
|
proxySecret.Name = proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet)
|
||||||
|
|
||||||
if err := r.Delete(ctx, proxySecret); err != nil && !kerrors.IsNotFound(err) {
|
if err := r.Delete(ctx, proxySecret); err != nil && !kerrors.IsNotFound(err) {
|
||||||
return fmt.Errorf("failed to delete proxy secret: %v", err)
|
return fmt.Errorf("failed to delete proxy secret: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Deleted proxy secret")
|
log.Info("Deleted proxy secret")
|
||||||
|
|
@ -287,7 +287,7 @@ func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunners(ctx context.Conte
|
||||||
ephemeralRunnerList := new(v1alpha1.EphemeralRunnerList)
|
ephemeralRunnerList := new(v1alpha1.EphemeralRunnerList)
|
||||||
err := r.List(ctx, ephemeralRunnerList, client.InNamespace(ephemeralRunnerSet.Namespace), client.MatchingFields{resourceOwnerKey: ephemeralRunnerSet.Name})
|
err := r.List(ctx, ephemeralRunnerList, client.InNamespace(ephemeralRunnerSet.Namespace), client.MatchingFields{resourceOwnerKey: ephemeralRunnerSet.Name})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("failed to list child ephemeral runners: %v", err)
|
return false, fmt.Errorf("failed to list child ephemeral runners: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Actual Ephemeral runner counts", "count", len(ephemeralRunnerList.Items))
|
log.Info("Actual Ephemeral runner counts", "count", len(ephemeralRunnerList.Items))
|
||||||
|
|
@ -441,7 +441,7 @@ func (r *EphemeralRunnerSetReconciler) deleteIdleEphemeralRunners(ctx context.Co
|
||||||
}
|
}
|
||||||
actionsClient, err := r.actionsClientFor(ctx, ephemeralRunnerSet)
|
actionsClient, err := r.actionsClientFor(ctx, ephemeralRunnerSet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create actions client for ephemeral runner replica set: %v", err)
|
return fmt.Errorf("failed to create actions client for ephemeral runner replica set: %w", err)
|
||||||
}
|
}
|
||||||
var errs []error
|
var errs []error
|
||||||
deletedCount := 0
|
deletedCount := 0
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue