diff --git a/pkg/controller/jenkins/jobs/jobs.go b/pkg/controller/jenkins/jobs/jobs.go index d5301853..9c39976f 100644 --- a/pkg/controller/jenkins/jobs/jobs.go +++ b/pkg/controller/jenkins/jobs/jobs.go @@ -30,8 +30,6 @@ const ( RunningStatus = "running" // ExpiredStatus - this is custom build status for expired build, not present in jenkins build result ExpiredStatus = "expired" - // MaxBuildRetires - determines max amount of retires for failed build - MaxBuildRetires = 3 ) var ( @@ -43,10 +41,12 @@ var ( ErrorBuildFailed = errors.New("build failed") // ErrorAbortBuildFailed - this is custom error returned when jenkins build couldn't be aborted ErrorAbortBuildFailed = errors.New("build abort failed") - // ErrorUncoverBuildFailed - this is custom error returned when jenkins build has failed and cannot be recovered - ErrorUncoverBuildFailed = errors.New("build failed and cannot be recovered") + // ErrorUnrecoverableBuildFailed - this is custom error returned when jenkins build has failed and cannot be recovered + ErrorUnrecoverableBuildFailed = errors.New("build failed and cannot be recovered") // ErrorNotFound - this is error returned when jenkins build couldn't be found ErrorNotFound = errors.New("404") + // BuildRetires - determines max amount of retires for failed build + BuildRetires = 3 ) // Jobs defines Jobs API tailored for operator sdk @@ -168,17 +168,14 @@ func (jobs *Jobs) ensureRunningBuild(build virtuslabv1alpha1.Build, jenkins *vir if build.Status == SuccessStatus { jobs.logger.Info(fmt.Sprintf("Build finished successfully, name:'%s' hash:'%s' status:'%s'", build.Name, build.Hash, build.Status)) - if !preserveStatus { - jobs.logger.Info(fmt.Sprintf("Removing build from status, name:'%s' hash:'%s'", build.Name, build.Hash)) - err := jobs.removeBuildFromStatus(build, jenkins) - if err != nil { - jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't remove build from status, name:'%s' hash:'%s'", build.Name, build.Hash)) - return false, err - } - } return true, nil } + if build.Status == FailureStatus || build.Status == UnstableStatus || build.Status == NotBuildStatus || build.Status == AbortedStatus { + jobs.logger.Info(fmt.Sprintf("Build failed, name:'%s' hash:'%s' status:'%s'", build.Name, build.Hash, build.Status)) + return false, ErrorBuildFailed + } + return false, nil } @@ -190,7 +187,7 @@ func (jobs *Jobs) ensureFailedBuild(build virtuslabv1alpha1.Build, jenkins *virt jobs.logger.Info(fmt.Sprintf("Ensuring failed build, name:'%s' hash:'%s' status: '%s'", build.Name, build.Hash, build.Status)) - if build.Retires < MaxBuildRetires { + if build.Retires < BuildRetires { jobs.logger.Info(fmt.Sprintf("Retrying build, name:'%s' hash:'%s' retries: '%d'", build.Name, build.Hash, build.Retires)) build.Retires = build.Retires + 1 _, err := jobs.buildJob(build, parameters, jenkins) @@ -198,7 +195,7 @@ func (jobs *Jobs) ensureFailedBuild(build virtuslabv1alpha1.Build, jenkins *virt jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't retry build, name:'%s' hash:'%s'", build.Name, build.Hash)) return false, err } - return false, ErrorBuildFailed + return false, nil } jobs.logger.Info(fmt.Sprintf("The retries limit was reached , name:'%s' hash:'%s' retries: '%d'", build.Name, build.Hash, build.Retires)) @@ -211,7 +208,7 @@ func (jobs *Jobs) ensureFailedBuild(build virtuslabv1alpha1.Build, jenkins *virt return false, err } } - return false, ErrorUncoverBuildFailed + return false, ErrorUnrecoverableBuildFailed } func (jobs *Jobs) ensureExpiredBuild(build virtuslabv1alpha1.Build, jenkins *virtuslabv1alpha1.Jenkins, preserveStatus bool) (bool, error) { @@ -285,18 +282,28 @@ func (jobs *Jobs) buildJob(build virtuslabv1alpha1.Build, parameters map[string] return false, ErrorEmptyJenkinsCR } - jobs.logger.Info(fmt.Sprintf("Running build, name:'%s' hash:'%s'", build.Name, build.Hash)) - - number, err := jobs.jenkinsClient.BuildJob(build.Name, parameters) + nextBuildNumber := int64(1) + job, err := jobs.jenkinsClient.GetJob(build.Name) if err != nil { - jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't run build, name:'%s' hash:'%s' number:'%d'", build.Name, build.Hash, number)) + jobs.logger.Info(fmt.Sprintf("Couldn't find jenkins job, name:'%s' hash:'%s'", build.Name, build.Hash)) + return false, err + } + + if job != nil { + nextBuildNumber = job.GetDetails().NextBuildNumber + } + + jobs.logger.Info(fmt.Sprintf("Running build, name:'%s' hash:'%s' number:'%d'", build.Name, build.Hash, nextBuildNumber)) + _, err = jobs.jenkinsClient.BuildJob(build.Name, parameters) + if err != nil { + jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't run build, name:'%s' hash:'%s' number:'%d'", build.Name, build.Hash, nextBuildNumber)) return false, err } build.Status = RunningStatus - build.Number = number + build.Number = nextBuildNumber - jobs.logger.Info(fmt.Sprintf("Updating build status, name:'%s' hash:'%s' status:'%s'", build.Name, build.Hash, build.Status)) + jobs.logger.Info(fmt.Sprintf("Updating build status, name:'%s' hash:'%s' status:'%s' number:'%d'", build.Name, build.Hash, build.Status, build.Number)) err = jobs.updateBuildStatus(build, jenkins) if err != nil { jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't update build status, name:'%s' hash:'%s'", build.Name, build.Hash)) diff --git a/pkg/controller/jenkins/jobs/jobs_test.go b/pkg/controller/jenkins/jobs/jobs_test.go index 65621132..352341db 100644 --- a/pkg/controller/jenkins/jobs/jobs_test.go +++ b/pkg/controller/jenkins/jobs/jobs_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/base64" + "fmt" "os" "testing" @@ -29,212 +30,357 @@ func TestMain(m *testing.M) { func TestSuccessEnsureJob(t *testing.T) { // given + ctx := context.TODO() logger := logf.ZapLogger(false) ctrl := gomock.NewController(t) - ctx := context.TODO() defer ctrl.Finish() buildName := "Test Job" - buildNumber := int64(1) - jenkinsClient := client.NewMockJenkins(ctrl) - fakeClient := fake.NewFakeClient() - hash := sha256.New() hash.Write([]byte(buildName)) encodedHash := base64.URLEncoding.EncodeToString(hash.Sum(nil)) // when jenkins := jenkinsCustomResource() + fakeClient := fake.NewFakeClient() err := fakeClient.Create(ctx, jenkins) assert.NoError(t, err) - jobs := New(jenkinsClient, fakeClient, logger) + for reconcileAttempt := 1; reconcileAttempt <= 2; reconcileAttempt++ { + logger.Info(fmt.Sprintf("Reconcile attempt #%d", reconcileAttempt)) + buildNumber := int64(1) + jenkinsClient := client.NewMockJenkins(ctrl) + jobs := New(jenkinsClient, fakeClient, logger) - // first run - build should be scheduled and status updated - jenkinsClient. - EXPECT(). - BuildJob(buildName, gomock.Any()). - Return(buildNumber, nil) + jenkinsClient. + EXPECT(). + GetJob(buildName). + Return(&gojenkins.Job{ + Raw: &gojenkins.JobResponse{ + NextBuildNumber: buildNumber, + }, + }, nil).AnyTimes() - done, err := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true) - assert.NoError(t, err) - assert.False(t, done) + jenkinsClient. + EXPECT(). + BuildJob(buildName, gomock.Any()). + Return(int64(0), nil).AnyTimes() - err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins) - assert.NoError(t, err) + jenkinsClient. + EXPECT(). + GetBuild(buildName, buildNumber). + Return(&gojenkins.Build{ + Raw: &gojenkins.BuildResponse{ + Result: SuccessStatus, + }, + }, nil).AnyTimes() - assert.NotEmpty(t, jenkins.Status.Builds) - assert.Equal(t, len(jenkins.Status.Builds), 1) + done, err := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true) + assert.NoError(t, err) - build := jenkins.Status.Builds[0] - assert.Equal(t, build.Name, buildName) - assert.Equal(t, build.Hash, encodedHash) - assert.Equal(t, build.Number, buildNumber) - assert.Equal(t, build.Status, RunningStatus) - assert.Equal(t, build.Retires, 0) - assert.NotNil(t, build.CreateTime) - assert.NotNil(t, build.LastUpdateTime) + err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins) + assert.NoError(t, err) - // second run - build should be success and status updated - jenkinsClient. - EXPECT(). - GetBuild(buildName, buildNumber). - Return(&gojenkins.Build{ - Raw: &gojenkins.BuildResponse{ - Result: SuccessStatus, - }, - }, nil) + assert.NotEmpty(t, jenkins.Status.Builds) + assert.Equal(t, len(jenkins.Status.Builds), 1) - done, err = jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true) - assert.NoError(t, err) - assert.True(t, done) + build := jenkins.Status.Builds[0] + assert.Equal(t, build.Name, buildName) + assert.Equal(t, build.Hash, encodedHash) + assert.Equal(t, build.Number, buildNumber) + assert.Equal(t, build.Retires, 0) + assert.NotNil(t, build.CreateTime) + assert.NotNil(t, build.LastUpdateTime) - err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins) - assert.NoError(t, err) + // first run - build should be scheduled and status updated + if reconcileAttempt == 1 { + assert.False(t, done) + assert.Equal(t, build.Status, RunningStatus) + } - assert.NotEmpty(t, jenkins.Status.Builds) - assert.Equal(t, len(jenkins.Status.Builds), 1) - - build = jenkins.Status.Builds[0] - assert.Equal(t, build.Name, buildName) - assert.Equal(t, build.Hash, encodedHash) - assert.Equal(t, build.Number, buildNumber) - assert.Equal(t, build.Status, SuccessStatus) - assert.Equal(t, build.Retires, 0) - assert.NotNil(t, build.CreateTime) - assert.NotNil(t, build.LastUpdateTime) + // second run -job should be success and status updated + if reconcileAttempt == 2 { + assert.True(t, done) + assert.Equal(t, build.Status, SuccessStatus) + } + } } func TestEnsureJobWithFailedBuild(t *testing.T) { // given + ctx := context.TODO() logger := logf.ZapLogger(false) ctrl := gomock.NewController(t) - ctx := context.TODO() defer ctrl.Finish() buildName := "Test Job" - buildNumber := int64(1) - jenkinsClient := client.NewMockJenkins(ctrl) - fakeClient := fake.NewFakeClient() - hash := sha256.New() hash.Write([]byte(buildName)) encodedHash := base64.URLEncoding.EncodeToString(hash.Sum(nil)) // when jenkins := jenkinsCustomResource() + fakeClient := fake.NewFakeClient() err := fakeClient.Create(ctx, jenkins) assert.NoError(t, err) - jobs := New(jenkinsClient, fakeClient, logger) + for reconcileAttempt := 1; reconcileAttempt <= 4; reconcileAttempt++ { + logger.Info(fmt.Sprintf("Reconcile attempt #%d", reconcileAttempt)) + jenkinsClient := client.NewMockJenkins(ctrl) + jobs := New(jenkinsClient, fakeClient, logger) - // first run - build should be scheduled and status updated - jenkinsClient. - EXPECT(). - BuildJob(buildName, gomock.Any()). - Return(buildNumber, nil) + // first run - build should be scheduled and status updated + if reconcileAttempt == 1 { + jenkinsClient. + EXPECT(). + GetJob(buildName). + Return(&gojenkins.Job{ + Raw: &gojenkins.JobResponse{ + NextBuildNumber: int64(1), + }, + }, nil) - done, err := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true) - assert.NoError(t, err) - assert.False(t, done) + jenkinsClient. + EXPECT(). + BuildJob(buildName, gomock.Any()). + Return(int64(0), nil) + } - err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins) + // second run - build should be failure and status updated + if reconcileAttempt == 2 { + jenkinsClient. + EXPECT(). + GetBuild(buildName, int64(1)). + Return(&gojenkins.Build{ + Raw: &gojenkins.BuildResponse{ + Result: FailureStatus, + }, + }, nil) + } + + // third run - build should be rescheduled and status updated + if reconcileAttempt == 3 { + jenkinsClient. + EXPECT(). + GetJob(buildName). + Return(&gojenkins.Job{ + Raw: &gojenkins.JobResponse{ + NextBuildNumber: int64(2), + }, + }, nil) + + jenkinsClient. + EXPECT(). + BuildJob(buildName, gomock.Any()). + Return(int64(0), nil) + } + + // fourth run - build should be success and status updated + if reconcileAttempt == 4 { + jenkinsClient. + EXPECT(). + GetBuild(buildName, int64(2)). + Return(&gojenkins.Build{ + Raw: &gojenkins.BuildResponse{ + Result: SuccessStatus, + }, + }, nil) + } + + done, errEnsureBuildJob := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true) + assert.NoError(t, err) + + err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins) + assert.NoError(t, err) + + assert.NotEmpty(t, jenkins.Status.Builds) + assert.Equal(t, len(jenkins.Status.Builds), 1) + + build := jenkins.Status.Builds[0] + assert.Equal(t, build.Name, buildName) + assert.Equal(t, build.Hash, encodedHash) + + assert.NotNil(t, build.CreateTime) + assert.NotNil(t, build.LastUpdateTime) + + // first run - build should be scheduled and status updated + if reconcileAttempt == 1 { + assert.NoError(t, errEnsureBuildJob) + assert.False(t, done) + assert.Equal(t, build.Number, int64(1)) + assert.Equal(t, build.Status, RunningStatus) + } + + // second run - build should be failure and status updated + if reconcileAttempt == 2 { + assert.Error(t, errEnsureBuildJob) + assert.False(t, done) + assert.Equal(t, build.Number, int64(1)) + assert.Equal(t, build.Status, FailureStatus) + } + + // third run - build should be rescheduled and status updated + if reconcileAttempt == 3 { + assert.NoError(t, errEnsureBuildJob) + assert.False(t, done) + assert.Equal(t, build.Number, int64(2)) + assert.Equal(t, build.Status, RunningStatus) + } + + // fourth run - build should be success and status updated + if reconcileAttempt == 4 { + assert.NoError(t, errEnsureBuildJob) + assert.True(t, done) + assert.Equal(t, build.Number, int64(2)) + assert.Equal(t, build.Status, SuccessStatus) + } + } +} + +func TestEnsureJobFailedWithMaxRetries(t *testing.T) { + // given + ctx := context.TODO() + logger := logf.ZapLogger(false) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + buildName := "Test Job" + hash := sha256.New() + hash.Write([]byte(buildName)) + encodedHash := base64.URLEncoding.EncodeToString(hash.Sum(nil)) + + // when + jenkins := jenkinsCustomResource() + fakeClient := fake.NewFakeClient() + err := fakeClient.Create(ctx, jenkins) assert.NoError(t, err) - assert.NotEmpty(t, jenkins.Status.Builds) - assert.Equal(t, len(jenkins.Status.Builds), 1) + BuildRetires = 1 // override max build retries + for reconcileAttempt := 1; reconcileAttempt <= 5; reconcileAttempt++ { + logger.Info(fmt.Sprintf("Reconcile attempt #%d", reconcileAttempt)) + jenkinsClient := client.NewMockJenkins(ctrl) + jobs := New(jenkinsClient, fakeClient, logger) - build := jenkins.Status.Builds[0] - assert.Equal(t, build.Name, buildName) - assert.Equal(t, build.Hash, encodedHash) - assert.Equal(t, build.Number, buildNumber) - assert.Equal(t, build.Status, RunningStatus) - assert.Equal(t, build.Retires, 0) - assert.NotNil(t, build.CreateTime) - assert.NotNil(t, build.LastUpdateTime) + // first run - build should be scheduled and status updated + if reconcileAttempt == 1 { + jenkinsClient. + EXPECT(). + GetJob(buildName). + Return(&gojenkins.Job{ + Raw: &gojenkins.JobResponse{ + NextBuildNumber: int64(1), + }, + }, nil) - // second run - build should be failure and status updated - jenkinsClient. - EXPECT(). - GetBuild(buildName, buildNumber). - Return(&gojenkins.Build{ - Raw: &gojenkins.BuildResponse{ - Result: FailureStatus, - }, - }, nil) + jenkinsClient. + EXPECT(). + BuildJob(buildName, gomock.Any()). + Return(int64(0), nil) + } - done, err = jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true) - assert.NoError(t, err) - assert.False(t, done) + // second run - build should be failure and status updated + if reconcileAttempt == 2 { + jenkinsClient. + EXPECT(). + GetBuild(buildName, int64(1)). + Return(&gojenkins.Build{ + Raw: &gojenkins.BuildResponse{ + Result: FailureStatus, + }, + }, nil) + } - err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins) - assert.NoError(t, err) + // third run - build should be rescheduled and status updated + if reconcileAttempt == 3 { + jenkinsClient. + EXPECT(). + GetJob(buildName). + Return(&gojenkins.Job{ + Raw: &gojenkins.JobResponse{ + NextBuildNumber: int64(2), + }, + }, nil) - assert.NotEmpty(t, jenkins.Status.Builds) - assert.Equal(t, len(jenkins.Status.Builds), 1) + jenkinsClient. + EXPECT(). + BuildJob(buildName, gomock.Any()). + Return(int64(0), nil) + } - build = jenkins.Status.Builds[0] - assert.Equal(t, build.Name, buildName) - assert.Equal(t, build.Hash, encodedHash) - assert.Equal(t, build.Number, buildNumber) - assert.Equal(t, build.Status, FailureStatus) - assert.Equal(t, build.Retires, 0) - assert.NotNil(t, build.CreateTime) - assert.NotNil(t, build.LastUpdateTime) + // fourth run - build should be success and status updated + if reconcileAttempt == 4 { + jenkinsClient. + EXPECT(). + GetBuild(buildName, int64(2)). + Return(&gojenkins.Build{ + Raw: &gojenkins.BuildResponse{ + Result: FailureStatus, + }, + }, nil) + } - // third run - build should be rescheduled and status updated - jenkinsClient. - EXPECT(). - BuildJob(buildName, gomock.Any()). - Return(buildNumber+1, nil) + done, errEnsureBuildJob := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true) + assert.NoError(t, err) - done, err = jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true) - assert.EqualError(t, err, ErrorBuildFailed.Error()) - assert.False(t, done) + err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins) + assert.NoError(t, err) - err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins) - assert.NoError(t, err) + assert.NotEmpty(t, jenkins.Status.Builds) + assert.Equal(t, len(jenkins.Status.Builds), 1) - assert.NotEmpty(t, jenkins.Status.Builds) - assert.Equal(t, len(jenkins.Status.Builds), 1) + build := jenkins.Status.Builds[0] + assert.Equal(t, build.Name, buildName) + assert.Equal(t, build.Hash, encodedHash) - build = jenkins.Status.Builds[0] - assert.Equal(t, build.Name, buildName) - assert.Equal(t, build.Hash, encodedHash) - assert.Equal(t, build.Number, buildNumber+1) - assert.Equal(t, build.Status, RunningStatus) - assert.Equal(t, build.Retires, 1) - assert.NotNil(t, build.CreateTime) - assert.NotNil(t, build.LastUpdateTime) + assert.NotNil(t, build.CreateTime) + assert.NotNil(t, build.LastUpdateTime) - // fourth run - build should be success and status updated - jenkinsClient. - EXPECT(). - GetBuild(buildName, buildNumber+1). - Return(&gojenkins.Build{ - Raw: &gojenkins.BuildResponse{ - Result: SuccessStatus, - }, - }, nil) + // first run - build should be scheduled and status updated + if reconcileAttempt == 1 { + assert.NoError(t, errEnsureBuildJob) + assert.False(t, done) + assert.Equal(t, build.Number, int64(1)) + assert.Equal(t, build.Retires, 0) + assert.Equal(t, build.Status, RunningStatus) + } - done, err = jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true) - assert.NoError(t, err) - assert.True(t, done) + // second run - build should be failure and status updated + if reconcileAttempt == 2 { + assert.EqualError(t, errEnsureBuildJob, ErrorBuildFailed.Error()) + assert.False(t, done) + assert.Equal(t, build.Number, int64(1)) + assert.Equal(t, build.Retires, 0) + assert.Equal(t, build.Status, FailureStatus) + } - err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins) - assert.NoError(t, err) + // third run - build should be rescheduled and status updated + if reconcileAttempt == 3 { + assert.NoError(t, errEnsureBuildJob) + assert.False(t, done) + //assert.Equal(t, build.Retires, 1) + assert.Equal(t, build.Number, int64(2)) + assert.Equal(t, build.Retires, 1) + assert.Equal(t, build.Status, RunningStatus) + } - assert.NotEmpty(t, jenkins.Status.Builds) - assert.Equal(t, len(jenkins.Status.Builds), 1) + // fourth run - build should be failure and status updated + if reconcileAttempt == 4 { + assert.EqualError(t, errEnsureBuildJob, ErrorBuildFailed.Error()) + assert.False(t, done) + assert.Equal(t, build.Number, int64(2)) + assert.Equal(t, build.Retires, 1) + assert.Equal(t, build.Status, FailureStatus) + } - build = jenkins.Status.Builds[0] - assert.Equal(t, build.Name, buildName) - assert.Equal(t, build.Hash, encodedHash) - assert.Equal(t, build.Number, buildNumber+1) - assert.Equal(t, build.Status, SuccessStatus) - assert.Equal(t, build.Retires, 1) - assert.NotNil(t, build.CreateTime) - assert.NotNil(t, build.LastUpdateTime) + // fifth run - build should be unrecoverable failed and status updated + if reconcileAttempt == 5 { + assert.EqualError(t, errEnsureBuildJob, ErrorUnrecoverableBuildFailed.Error()) + assert.False(t, done) + assert.Equal(t, build.Number, int64(2)) + assert.Equal(t, build.Retires, 1) + assert.Equal(t, build.Status, FailureStatus) + } + } } func jenkinsCustomResource() *virtuslabv1alpha1.Jenkins {