From a408112875439a5792cd46678b734a9c83d86e5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomasz=20S=C4=99k?= <tomasz.sek.88@gmail.com>
Date: Mon, 17 Jun 2019 22:46:48 +0200
Subject: [PATCH] Stop reconcile loop when error has been repeated more than 15
 times

---
 .../backuprestore/backuprestore.go            |  4 ---
 pkg/controller/jenkins/jenkins_controller.go  | 31 +++++++++++++++++++
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/pkg/controller/jenkins/configuration/backuprestore/backuprestore.go b/pkg/controller/jenkins/configuration/backuprestore/backuprestore.go
index de108e22..04ef752f 100644
--- a/pkg/controller/jenkins/configuration/backuprestore/backuprestore.go
+++ b/pkg/controller/jenkins/configuration/backuprestore/backuprestore.go
@@ -135,8 +135,6 @@ func (bar *BackupAndRestore) Restore(jenkinsClient jenkinsclient.Jenkins) error
 		return bar.k8sClient.Update(context.TODO(), jenkins)
 	}
 
-	//TODO after 3 fails stop
-
 	return err
 }
 
@@ -163,8 +161,6 @@ func (bar *BackupAndRestore) Backup() error {
 		return bar.k8sClient.Update(context.TODO(), jenkins)
 	}
 
-	//TODO after 3 fails stop
-
 	return err
 }
 
diff --git a/pkg/controller/jenkins/jenkins_controller.go b/pkg/controller/jenkins/jenkins_controller.go
index efc25ead..fe22723a 100644
--- a/pkg/controller/jenkins/jenkins_controller.go
+++ b/pkg/controller/jenkins/jenkins_controller.go
@@ -42,6 +42,13 @@ const (
 	reasonCRValidationFailure event.Reason = "CRValidationFailure"
 )
 
+type reconcileError struct {
+	err     error
+	counter uint64
+}
+
+var reconcileErrors = map[string]reconcileError{}
+
 // Add creates a new Jenkins Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
 func Add(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config) error {
@@ -120,6 +127,30 @@ func (r *ReconcileJenkins) Reconcile(request reconcile.Request) (reconcile.Resul
 	if err != nil && apierrors.IsConflict(err) {
 		return reconcile.Result{Requeue: true}, nil
 	} else if err != nil {
+		lastErrors, found := reconcileErrors[request.Name]
+		if found {
+			if err.Error() == lastErrors.err.Error() {
+				lastErrors.counter++
+			} else {
+				lastErrors.counter = 1
+				lastErrors.err = err
+			}
+		} else {
+			lastErrors = reconcileError{
+				err:     err,
+				counter: 1,
+			}
+		}
+		reconcileErrors[request.Name] = lastErrors
+		if lastErrors.counter >= 15 {
+			if log.Debug {
+				logger.V(log.VWarn).Info(fmt.Sprintf("Reconcile loop failed ten times with the same error, giving up: %+v", err))
+			} else {
+				logger.V(log.VWarn).Info(fmt.Sprintf("Reconcile loop failed ten times with the same error, giving up: %s", err))
+			}
+			return reconcile.Result{Requeue: false}, nil
+		}
+
 		if log.Debug {
 			logger.V(log.VWarn).Info(fmt.Sprintf("Reconcile loop failed: %+v", err))
 		} else {