Updated jobs and added more tests

pkg/controller/jenkins/jobs/jobs.go

@@ -30,8 +30,6 @@ const (
 	RunningStatus = "running"
 	// ExpiredStatus - this is custom build status for expired build, not present in jenkins build result
 	ExpiredStatus = "expired"
-	// MaxBuildRetires - determines max amount of retires for failed build
-	MaxBuildRetires = 3
 )
 
 var (
@@ -43,10 +41,12 @@ var (
 	ErrorBuildFailed = errors.New("build failed")
 	// ErrorAbortBuildFailed - this is custom error returned when jenkins build couldn't be aborted
 	ErrorAbortBuildFailed = errors.New("build abort failed")
-	// ErrorUncoverBuildFailed - this is custom error returned when jenkins build has failed and cannot be recovered
+	// ErrorUnrecoverableBuildFailed - this is custom error returned when jenkins build has failed and cannot be recovered
-	ErrorUncoverBuildFailed = errors.New("build failed and cannot be recovered")
+	ErrorUnrecoverableBuildFailed = errors.New("build failed and cannot be recovered")
 	// ErrorNotFound - this is error returned when jenkins build couldn't be found
 	ErrorNotFound = errors.New("404")
+	// BuildRetires - determines max amount of retires for failed build
+	BuildRetires = 3
 )
 
 // Jobs defines Jobs API tailored for operator sdk
@@ -168,17 +168,14 @@ func (jobs *Jobs) ensureRunningBuild(build virtuslabv1alpha1.Build, jenkins *vir
 
 	if build.Status == SuccessStatus {
 		jobs.logger.Info(fmt.Sprintf("Build finished successfully, name:'%s' hash:'%s' status:'%s'", build.Name, build.Hash, build.Status))
-		if !preserveStatus {
-			jobs.logger.Info(fmt.Sprintf("Removing build from status, name:'%s' hash:'%s'", build.Name, build.Hash))
-			err := jobs.removeBuildFromStatus(build, jenkins)
-			if err != nil {
-				jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't remove build from status, name:'%s' hash:'%s'", build.Name, build.Hash))
-				return false, err
-			}
-		}
 		return true, nil
 	}
 
+	if build.Status == FailureStatus || build.Status == UnstableStatus || build.Status == NotBuildStatus || build.Status == AbortedStatus {
+		jobs.logger.Info(fmt.Sprintf("Build failed, name:'%s' hash:'%s' status:'%s'", build.Name, build.Hash, build.Status))
+		return false, ErrorBuildFailed
+	}
+
 	return false, nil
 }
 
@@ -190,7 +187,7 @@ func (jobs *Jobs) ensureFailedBuild(build virtuslabv1alpha1.Build, jenkins *virt
 
 	jobs.logger.Info(fmt.Sprintf("Ensuring failed build, name:'%s' hash:'%s' status: '%s'", build.Name, build.Hash, build.Status))
 
-	if build.Retires < MaxBuildRetires {
+	if build.Retires < BuildRetires {
 		jobs.logger.Info(fmt.Sprintf("Retrying build, name:'%s' hash:'%s' retries: '%d'", build.Name, build.Hash, build.Retires))
 		build.Retires = build.Retires + 1
 		_, err := jobs.buildJob(build, parameters, jenkins)
@@ -198,7 +195,7 @@ func (jobs *Jobs) ensureFailedBuild(build virtuslabv1alpha1.Build, jenkins *virt
 			jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't retry build, name:'%s' hash:'%s'", build.Name, build.Hash))
 			return false, err
 		}
-		return false, ErrorBuildFailed
+		return false, nil
 	}
 
 	jobs.logger.Info(fmt.Sprintf("The retries limit was reached , name:'%s' hash:'%s' retries: '%d'", build.Name, build.Hash, build.Retires))
@@ -211,7 +208,7 @@ func (jobs *Jobs) ensureFailedBuild(build virtuslabv1alpha1.Build, jenkins *virt
 			return false, err
 		}
 	}
-	return false, ErrorUncoverBuildFailed
+	return false, ErrorUnrecoverableBuildFailed
 }
 
 func (jobs *Jobs) ensureExpiredBuild(build virtuslabv1alpha1.Build, jenkins *virtuslabv1alpha1.Jenkins, preserveStatus bool) (bool, error) {
@@ -285,18 +282,28 @@ func (jobs *Jobs) buildJob(build virtuslabv1alpha1.Build, parameters map[string]
 		return false, ErrorEmptyJenkinsCR
 	}
 
-	jobs.logger.Info(fmt.Sprintf("Running build, name:'%s' hash:'%s'", build.Name, build.Hash))
+	nextBuildNumber := int64(1)
-
+	job, err := jobs.jenkinsClient.GetJob(build.Name)
-	number, err := jobs.jenkinsClient.BuildJob(build.Name, parameters)
 	if err != nil {
-		jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't run build, name:'%s' hash:'%s' number:'%d'", build.Name, build.Hash, number))
+		jobs.logger.Info(fmt.Sprintf("Couldn't find jenkins job, name:'%s' hash:'%s'", build.Name, build.Hash))
+		return false, err
+	}
+
+	if job != nil {
+		nextBuildNumber = job.GetDetails().NextBuildNumber
+	}
+
+	jobs.logger.Info(fmt.Sprintf("Running build, name:'%s' hash:'%s' number:'%d'", build.Name, build.Hash, nextBuildNumber))
+	_, err = jobs.jenkinsClient.BuildJob(build.Name, parameters)
+	if err != nil {
+		jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't run build, name:'%s' hash:'%s' number:'%d'", build.Name, build.Hash, nextBuildNumber))
 		return false, err
 	}
 
 	build.Status = RunningStatus
-	build.Number = number
+	build.Number = nextBuildNumber
 
-	jobs.logger.Info(fmt.Sprintf("Updating build status, name:'%s' hash:'%s' status:'%s'", build.Name, build.Hash, build.Status))
+	jobs.logger.Info(fmt.Sprintf("Updating build status, name:'%s' hash:'%s' status:'%s' number:'%d'", build.Name, build.Hash, build.Status, build.Number))
 	err = jobs.updateBuildStatus(build, jenkins)
 	if err != nil {
 		jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't update build status, name:'%s' hash:'%s'", build.Name, build.Hash))

pkg/controller/jenkins/jobs/jobs_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/base64"
+	"fmt"
 	"os"
 	"testing"
 
@@ -29,53 +30,42 @@ func TestMain(m *testing.M) {
 
 func TestSuccessEnsureJob(t *testing.T) {
 	// given
+	ctx := context.TODO()
 	logger := logf.ZapLogger(false)
 	ctrl := gomock.NewController(t)
-	ctx := context.TODO()
 	defer ctrl.Finish()
 
 	buildName := "Test Job"
-	buildNumber := int64(1)
-	jenkinsClient := client.NewMockJenkins(ctrl)
-	fakeClient := fake.NewFakeClient()
-
 	hash := sha256.New()
 	hash.Write([]byte(buildName))
 	encodedHash := base64.URLEncoding.EncodeToString(hash.Sum(nil))
 
 	// when
 	jenkins := jenkinsCustomResource()
+	fakeClient := fake.NewFakeClient()
 	err := fakeClient.Create(ctx, jenkins)
 	assert.NoError(t, err)
 
+	for reconcileAttempt := 1; reconcileAttempt <= 2; reconcileAttempt++ {
+		logger.Info(fmt.Sprintf("Reconcile attempt #%d", reconcileAttempt))
+		buildNumber := int64(1)
+		jenkinsClient := client.NewMockJenkins(ctrl)
 		jobs := New(jenkinsClient, fakeClient, logger)
 
-	// first run - build should be scheduled and status updated
+		jenkinsClient.
+			EXPECT().
+			GetJob(buildName).
+			Return(&gojenkins.Job{
+				Raw: &gojenkins.JobResponse{
+					NextBuildNumber: buildNumber,
+				},
+			}, nil).AnyTimes()
+
 		jenkinsClient.
 			EXPECT().
 			BuildJob(buildName, gomock.Any()).
-		Return(buildNumber, nil)
+			Return(int64(0), nil).AnyTimes()
 
-	done, err := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
-	assert.NoError(t, err)
-	assert.False(t, done)
-
-	err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
-	assert.NoError(t, err)
-
-	assert.NotEmpty(t, jenkins.Status.Builds)
-	assert.Equal(t, len(jenkins.Status.Builds), 1)
-
-	build := jenkins.Status.Builds[0]
-	assert.Equal(t, build.Name, buildName)
-	assert.Equal(t, build.Hash, encodedHash)
-	assert.Equal(t, build.Number, buildNumber)
-	assert.Equal(t, build.Status, RunningStatus)
-	assert.Equal(t, build.Retires, 0)
-	assert.NotNil(t, build.CreateTime)
-	assert.NotNil(t, build.LastUpdateTime)
-
-	// second run - build should be success and status updated
 		jenkinsClient.
 			EXPECT().
 			GetBuild(buildName, buildNumber).
@@ -83,60 +73,10 @@ func TestSuccessEnsureJob(t *testing.T) {
 				Raw: &gojenkins.BuildResponse{
 					Result: SuccessStatus,
 				},
-		}, nil)
+			}, nil).AnyTimes()
-
-	done, err = jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
-	assert.NoError(t, err)
-	assert.True(t, done)
-
-	err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
-	assert.NoError(t, err)
-
-	assert.NotEmpty(t, jenkins.Status.Builds)
-	assert.Equal(t, len(jenkins.Status.Builds), 1)
-
-	build = jenkins.Status.Builds[0]
-	assert.Equal(t, build.Name, buildName)
-	assert.Equal(t, build.Hash, encodedHash)
-	assert.Equal(t, build.Number, buildNumber)
-	assert.Equal(t, build.Status, SuccessStatus)
-	assert.Equal(t, build.Retires, 0)
-	assert.NotNil(t, build.CreateTime)
-	assert.NotNil(t, build.LastUpdateTime)
-}
-
-func TestEnsureJobWithFailedBuild(t *testing.T) {
-	// given
-	logger := logf.ZapLogger(false)
-	ctrl := gomock.NewController(t)
-	ctx := context.TODO()
-	defer ctrl.Finish()
-
-	buildName := "Test Job"
-	buildNumber := int64(1)
-	jenkinsClient := client.NewMockJenkins(ctrl)
-	fakeClient := fake.NewFakeClient()
-
-	hash := sha256.New()
-	hash.Write([]byte(buildName))
-	encodedHash := base64.URLEncoding.EncodeToString(hash.Sum(nil))
-
-	// when
-	jenkins := jenkinsCustomResource()
-	err := fakeClient.Create(ctx, jenkins)
-	assert.NoError(t, err)
-
-	jobs := New(jenkinsClient, fakeClient, logger)
-
-	// first run - build should be scheduled and status updated
-	jenkinsClient.
-		EXPECT().
-		BuildJob(buildName, gomock.Any()).
-		Return(buildNumber, nil)
 
 		done, err := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
 		assert.NoError(t, err)
-	assert.False(t, done)
 
 		err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
 		assert.NoError(t, err)
@@ -148,78 +88,107 @@ func TestEnsureJobWithFailedBuild(t *testing.T) {
 		assert.Equal(t, build.Name, buildName)
 		assert.Equal(t, build.Hash, encodedHash)
 		assert.Equal(t, build.Number, buildNumber)
-	assert.Equal(t, build.Status, RunningStatus)
 		assert.Equal(t, build.Retires, 0)
 		assert.NotNil(t, build.CreateTime)
 		assert.NotNil(t, build.LastUpdateTime)
 
-	// second run - build should be failure and status updated
+		// first run - build should be scheduled and status updated
+		if reconcileAttempt == 1 {
+			assert.False(t, done)
+			assert.Equal(t, build.Status, RunningStatus)
+		}
+
+		// second run -job should be success and status updated
+		if reconcileAttempt == 2 {
+			assert.True(t, done)
+			assert.Equal(t, build.Status, SuccessStatus)
+		}
+	}
+}
+
+func TestEnsureJobWithFailedBuild(t *testing.T) {
+	// given
+	ctx := context.TODO()
+	logger := logf.ZapLogger(false)
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	buildName := "Test Job"
+	hash := sha256.New()
+	hash.Write([]byte(buildName))
+	encodedHash := base64.URLEncoding.EncodeToString(hash.Sum(nil))
+
+	// when
+	jenkins := jenkinsCustomResource()
+	fakeClient := fake.NewFakeClient()
+	err := fakeClient.Create(ctx, jenkins)
+	assert.NoError(t, err)
+
+	for reconcileAttempt := 1; reconcileAttempt <= 4; reconcileAttempt++ {
+		logger.Info(fmt.Sprintf("Reconcile attempt #%d", reconcileAttempt))
+		jenkinsClient := client.NewMockJenkins(ctrl)
+		jobs := New(jenkinsClient, fakeClient, logger)
+
+		// first run - build should be scheduled and status updated
+		if reconcileAttempt == 1 {
 			jenkinsClient.
 				EXPECT().
-		GetBuild(buildName, buildNumber).
+				GetJob(buildName).
+				Return(&gojenkins.Job{
+					Raw: &gojenkins.JobResponse{
+						NextBuildNumber: int64(1),
+					},
+				}, nil)
+
+			jenkinsClient.
+				EXPECT().
+				BuildJob(buildName, gomock.Any()).
+				Return(int64(0), nil)
+		}
+
+		// second run - build should be failure and status updated
+		if reconcileAttempt == 2 {
+			jenkinsClient.
+				EXPECT().
+				GetBuild(buildName, int64(1)).
 				Return(&gojenkins.Build{
 					Raw: &gojenkins.BuildResponse{
 						Result: FailureStatus,
 					},
 				}, nil)
-
+		}
-	done, err = jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
-	assert.NoError(t, err)
-	assert.False(t, done)
-
-	err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
-	assert.NoError(t, err)
-
-	assert.NotEmpty(t, jenkins.Status.Builds)
-	assert.Equal(t, len(jenkins.Status.Builds), 1)
-
-	build = jenkins.Status.Builds[0]
-	assert.Equal(t, build.Name, buildName)
-	assert.Equal(t, build.Hash, encodedHash)
-	assert.Equal(t, build.Number, buildNumber)
-	assert.Equal(t, build.Status, FailureStatus)
-	assert.Equal(t, build.Retires, 0)
-	assert.NotNil(t, build.CreateTime)
-	assert.NotNil(t, build.LastUpdateTime)
 
 		// third run - build should be rescheduled and status updated
+		if reconcileAttempt == 3 {
+			jenkinsClient.
+				EXPECT().
+				GetJob(buildName).
+				Return(&gojenkins.Job{
+					Raw: &gojenkins.JobResponse{
+						NextBuildNumber: int64(2),
+					},
+				}, nil)
+
 			jenkinsClient.
 				EXPECT().
 				BuildJob(buildName, gomock.Any()).
-		Return(buildNumber+1, nil)
+				Return(int64(0), nil)
-
+		}
-	done, err = jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
-	assert.EqualError(t, err, ErrorBuildFailed.Error())
-	assert.False(t, done)
-
-	err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
-	assert.NoError(t, err)
-
-	assert.NotEmpty(t, jenkins.Status.Builds)
-	assert.Equal(t, len(jenkins.Status.Builds), 1)
-
-	build = jenkins.Status.Builds[0]
-	assert.Equal(t, build.Name, buildName)
-	assert.Equal(t, build.Hash, encodedHash)
-	assert.Equal(t, build.Number, buildNumber+1)
-	assert.Equal(t, build.Status, RunningStatus)
-	assert.Equal(t, build.Retires, 1)
-	assert.NotNil(t, build.CreateTime)
-	assert.NotNil(t, build.LastUpdateTime)
 
 		// fourth run - build should be success and status updated
+		if reconcileAttempt == 4 {
 			jenkinsClient.
 				EXPECT().
-		GetBuild(buildName, buildNumber+1).
+				GetBuild(buildName, int64(2)).
 				Return(&gojenkins.Build{
 					Raw: &gojenkins.BuildResponse{
 						Result: SuccessStatus,
 					},
 				}, nil)
+		}
 
-	done, err = jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
+		done, errEnsureBuildJob := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
 		assert.NoError(t, err)
-	assert.True(t, done)
 
 		err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
 		assert.NoError(t, err)
@@ -227,14 +196,191 @@ func TestEnsureJobWithFailedBuild(t *testing.T) {
 		assert.NotEmpty(t, jenkins.Status.Builds)
 		assert.Equal(t, len(jenkins.Status.Builds), 1)
 
-	build = jenkins.Status.Builds[0]
+		build := jenkins.Status.Builds[0]
 		assert.Equal(t, build.Name, buildName)
 		assert.Equal(t, build.Hash, encodedHash)
-	assert.Equal(t, build.Number, buildNumber+1)
+
-	assert.Equal(t, build.Status, SuccessStatus)
-	assert.Equal(t, build.Retires, 1)
 		assert.NotNil(t, build.CreateTime)
 		assert.NotNil(t, build.LastUpdateTime)
+
+		// first run - build should be scheduled and status updated
+		if reconcileAttempt == 1 {
+			assert.NoError(t, errEnsureBuildJob)
+			assert.False(t, done)
+			assert.Equal(t, build.Number, int64(1))
+			assert.Equal(t, build.Status, RunningStatus)
+		}
+
+		// second run - build should be failure and status updated
+		if reconcileAttempt == 2 {
+			assert.Error(t, errEnsureBuildJob)
+			assert.False(t, done)
+			assert.Equal(t, build.Number, int64(1))
+			assert.Equal(t, build.Status, FailureStatus)
+		}
+
+		// third run - build should be rescheduled and status updated
+		if reconcileAttempt == 3 {
+			assert.NoError(t, errEnsureBuildJob)
+			assert.False(t, done)
+			assert.Equal(t, build.Number, int64(2))
+			assert.Equal(t, build.Status, RunningStatus)
+		}
+
+		// fourth run - build should be success and status updated
+		if reconcileAttempt == 4 {
+			assert.NoError(t, errEnsureBuildJob)
+			assert.True(t, done)
+			assert.Equal(t, build.Number, int64(2))
+			assert.Equal(t, build.Status, SuccessStatus)
+		}
+	}
+}
+
+func TestEnsureJobFailedWithMaxRetries(t *testing.T) {
+	// given
+	ctx := context.TODO()
+	logger := logf.ZapLogger(false)
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	buildName := "Test Job"
+	hash := sha256.New()
+	hash.Write([]byte(buildName))
+	encodedHash := base64.URLEncoding.EncodeToString(hash.Sum(nil))
+
+	// when
+	jenkins := jenkinsCustomResource()
+	fakeClient := fake.NewFakeClient()
+	err := fakeClient.Create(ctx, jenkins)
+	assert.NoError(t, err)
+
+	BuildRetires = 1 // override max build retries
+	for reconcileAttempt := 1; reconcileAttempt <= 5; reconcileAttempt++ {
+		logger.Info(fmt.Sprintf("Reconcile attempt #%d", reconcileAttempt))
+		jenkinsClient := client.NewMockJenkins(ctrl)
+		jobs := New(jenkinsClient, fakeClient, logger)
+
+		// first run - build should be scheduled and status updated
+		if reconcileAttempt == 1 {
+			jenkinsClient.
+				EXPECT().
+				GetJob(buildName).
+				Return(&gojenkins.Job{
+					Raw: &gojenkins.JobResponse{
+						NextBuildNumber: int64(1),
+					},
+				}, nil)
+
+			jenkinsClient.
+				EXPECT().
+				BuildJob(buildName, gomock.Any()).
+				Return(int64(0), nil)
+		}
+
+		// second run - build should be failure and status updated
+		if reconcileAttempt == 2 {
+			jenkinsClient.
+				EXPECT().
+				GetBuild(buildName, int64(1)).
+				Return(&gojenkins.Build{
+					Raw: &gojenkins.BuildResponse{
+						Result: FailureStatus,
+					},
+				}, nil)
+		}
+
+		// third run - build should be rescheduled and status updated
+		if reconcileAttempt == 3 {
+			jenkinsClient.
+				EXPECT().
+				GetJob(buildName).
+				Return(&gojenkins.Job{
+					Raw: &gojenkins.JobResponse{
+						NextBuildNumber: int64(2),
+					},
+				}, nil)
+
+			jenkinsClient.
+				EXPECT().
+				BuildJob(buildName, gomock.Any()).
+				Return(int64(0), nil)
+		}
+
+		// fourth run - build should be success and status updated
+		if reconcileAttempt == 4 {
+			jenkinsClient.
+				EXPECT().
+				GetBuild(buildName, int64(2)).
+				Return(&gojenkins.Build{
+					Raw: &gojenkins.BuildResponse{
+						Result: FailureStatus,
+					},
+				}, nil)
+		}
+
+		done, errEnsureBuildJob := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
+		assert.NoError(t, err)
+
+		err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
+		assert.NoError(t, err)
+
+		assert.NotEmpty(t, jenkins.Status.Builds)
+		assert.Equal(t, len(jenkins.Status.Builds), 1)
+
+		build := jenkins.Status.Builds[0]
+		assert.Equal(t, build.Name, buildName)
+		assert.Equal(t, build.Hash, encodedHash)
+
+		assert.NotNil(t, build.CreateTime)
+		assert.NotNil(t, build.LastUpdateTime)
+
+		// first run - build should be scheduled and status updated
+		if reconcileAttempt == 1 {
+			assert.NoError(t, errEnsureBuildJob)
+			assert.False(t, done)
+			assert.Equal(t, build.Number, int64(1))
+			assert.Equal(t, build.Retires, 0)
+			assert.Equal(t, build.Status, RunningStatus)
+		}
+
+		// second run - build should be failure and status updated
+		if reconcileAttempt == 2 {
+			assert.EqualError(t, errEnsureBuildJob, ErrorBuildFailed.Error())
+			assert.False(t, done)
+			assert.Equal(t, build.Number, int64(1))
+			assert.Equal(t, build.Retires, 0)
+			assert.Equal(t, build.Status, FailureStatus)
+		}
+
+		// third run - build should be rescheduled and status updated
+		if reconcileAttempt == 3 {
+			assert.NoError(t, errEnsureBuildJob)
+			assert.False(t, done)
+			//assert.Equal(t, build.Retires, 1)
+			assert.Equal(t, build.Number, int64(2))
+			assert.Equal(t, build.Retires, 1)
+			assert.Equal(t, build.Status, RunningStatus)
+		}
+
+		// fourth run - build should be failure and status updated
+		if reconcileAttempt == 4 {
+			assert.EqualError(t, errEnsureBuildJob, ErrorBuildFailed.Error())
+			assert.False(t, done)
+			assert.Equal(t, build.Number, int64(2))
+			assert.Equal(t, build.Retires, 1)
+			assert.Equal(t, build.Status, FailureStatus)
+		}
+
+		// fifth run - build should be unrecoverable failed and status updated
+		if reconcileAttempt == 5 {
+			assert.EqualError(t, errEnsureBuildJob, ErrorUnrecoverableBuildFailed.Error())
+			assert.False(t, done)
+			assert.Equal(t, build.Number, int64(2))
+			assert.Equal(t, build.Retires, 1)
+			assert.Equal(t, build.Status, FailureStatus)
+		}
+	}
 }
 
 func jenkinsCustomResource() *virtuslabv1alpha1.Jenkins {