public_git
/
kubernetes-operator
mirror of https://github.com/jenkinsci/kubernetes-operator.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Updated jobs and added more tests

This commit is contained in:
antoniaklja 2019-01-03 16:07:14 +01:00
parent b5fca5950a
commit c404221154
2 changed files with 318 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
pkg/controller/jenkins/jobs/jobs.go
View File

@ -30,8 +30,6 @@ const (
RunningStatus = "running"
// ExpiredStatus - this is custom build status for expired build, not present in jenkins build result
ExpiredStatus = "expired"
// MaxBuildRetires - determines max amount of retires for failed build
MaxBuildRetires = 3
)
var (
@ -43,10 +41,12 @@ var (
ErrorBuildFailed = errors.New("build failed")
// ErrorAbortBuildFailed - this is custom error returned when jenkins build couldn't be aborted
ErrorAbortBuildFailed = errors.New("build abort failed")
// ErrorUncoverBuildFailed - this is custom error returned when jenkins build has failed and cannot be recovered
ErrorUncoverBuildFailed = errors.New("build failed and cannot be recovered")
// ErrorUnrecoverableBuildFailed - this is custom error returned when jenkins build has failed and cannot be recovered
ErrorUnrecoverableBuildFailed = errors.New("build failed and cannot be recovered")
// ErrorNotFound - this is error returned when jenkins build couldn't be found
ErrorNotFound = errors.New("404")
// BuildRetires - determines max amount of retires for failed build
BuildRetires = 3
)
// Jobs defines Jobs API tailored for operator sdk
@ -168,17 +168,14 @@ func (jobs *Jobs) ensureRunningBuild(build virtuslabv1alpha1.Build, jenkins *vir
if build.Status == SuccessStatus {
jobs.logger.Info(fmt.Sprintf("Build finished successfully, name:'%s' hash:'%s' status:'%s'", build.Name, build.Hash, build.Status))
if !preserveStatus {
jobs.logger.Info(fmt.Sprintf("Removing build from status, name:'%s' hash:'%s'", build.Name, build.Hash))
err := jobs.removeBuildFromStatus(build, jenkins)
if err != nil {
jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't remove build from status, name:'%s' hash:'%s'", build.Name, build.Hash))
return false, err
}
}
return true, nil
}
if build.Status == FailureStatus || build.Status == UnstableStatus || build.Status == NotBuildStatus || build.Status == AbortedStatus {
jobs.logger.Info(fmt.Sprintf("Build failed, name:'%s' hash:'%s' status:'%s'", build.Name, build.Hash, build.Status))
return false, ErrorBuildFailed
}
return false, nil
}
@ -190,7 +187,7 @@ func (jobs *Jobs) ensureFailedBuild(build virtuslabv1alpha1.Build, jenkins *virt
jobs.logger.Info(fmt.Sprintf("Ensuring failed build, name:'%s' hash:'%s' status: '%s'", build.Name, build.Hash, build.Status))
if build.Retires < MaxBuildRetires {
if build.Retires < BuildRetires {
jobs.logger.Info(fmt.Sprintf("Retrying build, name:'%s' hash:'%s' retries: '%d'", build.Name, build.Hash, build.Retires))
build.Retires = build.Retires + 1
_, err := jobs.buildJob(build, parameters, jenkins)
@ -198,7 +195,7 @@ func (jobs *Jobs) ensureFailedBuild(build virtuslabv1alpha1.Build, jenkins *virt
jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't retry build, name:'%s' hash:'%s'", build.Name, build.Hash))
return false, err
}
return false, ErrorBuildFailed
return false, nil
}
jobs.logger.Info(fmt.Sprintf("The retries limit was reached , name:'%s' hash:'%s' retries: '%d'", build.Name, build.Hash, build.Retires))
@ -211,7 +208,7 @@ func (jobs *Jobs) ensureFailedBuild(build virtuslabv1alpha1.Build, jenkins *virt
return false, err
}
}
return false, ErrorUncoverBuildFailed
return false, ErrorUnrecoverableBuildFailed
}
func (jobs *Jobs) ensureExpiredBuild(build virtuslabv1alpha1.Build, jenkins *virtuslabv1alpha1.Jenkins, preserveStatus bool) (bool, error) {
@ -285,18 +282,28 @@ func (jobs *Jobs) buildJob(build virtuslabv1alpha1.Build, parameters map[string]
return false, ErrorEmptyJenkinsCR
}
jobs.logger.Info(fmt.Sprintf("Running build, name:'%s' hash:'%s'", build.Name, build.Hash))
number, err := jobs.jenkinsClient.BuildJob(build.Name, parameters)
nextBuildNumber := int64(1)
job, err := jobs.jenkinsClient.GetJob(build.Name)
if err != nil {
jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't run build, name:'%s' hash:'%s' number:'%d'", build.Name, build.Hash, number))
jobs.logger.Info(fmt.Sprintf("Couldn't find jenkins job, name:'%s' hash:'%s'", build.Name, build.Hash))
return false, err
}
if job != nil {
nextBuildNumber = job.GetDetails().NextBuildNumber
}
jobs.logger.Info(fmt.Sprintf("Running build, name:'%s' hash:'%s' number:'%d'", build.Name, build.Hash, nextBuildNumber))
_, err = jobs.jenkinsClient.BuildJob(build.Name, parameters)
if err != nil {
jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't run build, name:'%s' hash:'%s' number:'%d'", build.Name, build.Hash, nextBuildNumber))
return false, err
}
build.Status = RunningStatus
build.Number = number
build.Number = nextBuildNumber
jobs.logger.Info(fmt.Sprintf("Updating build status, name:'%s' hash:'%s' status:'%s'", build.Name, build.Hash, build.Status))
jobs.logger.Info(fmt.Sprintf("Updating build status, name:'%s' hash:'%s' status:'%s' number:'%d'", build.Name, build.Hash, build.Status, build.Number))
err = jobs.updateBuildStatus(build, jenkins)
if err != nil {
jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't update build status, name:'%s' hash:'%s'", build.Name, build.Hash))

434
pkg/controller/jenkins/jobs/jobs_test.go
View File

@ -4,6 +4,7 @@ import (
"context"
"crypto/sha256"
"encoding/base64"
"fmt"
"os"
"testing"
@ -29,212 +30,357 @@ func TestMain(m *testing.M) {
func TestSuccessEnsureJob(t *testing.T) {
// given
ctx := context.TODO()
logger := logf.ZapLogger(false)
ctrl := gomock.NewController(t)
ctx := context.TODO()
defer ctrl.Finish()
buildName := "Test Job"
buildNumber := int64(1)
jenkinsClient := client.NewMockJenkins(ctrl)
fakeClient := fake.NewFakeClient()
hash := sha256.New()
hash.Write([]byte(buildName))
encodedHash := base64.URLEncoding.EncodeToString(hash.Sum(nil))
// when
jenkins := jenkinsCustomResource()
fakeClient := fake.NewFakeClient()
err := fakeClient.Create(ctx, jenkins)
assert.NoError(t, err)
jobs := New(jenkinsClient, fakeClient, logger)
for reconcileAttempt := 1; reconcileAttempt <= 2; reconcileAttempt++ {
logger.Info(fmt.Sprintf("Reconcile attempt #%d", reconcileAttempt))
buildNumber := int64(1)
jenkinsClient := client.NewMockJenkins(ctrl)
jobs := New(jenkinsClient, fakeClient, logger)
// first run - build should be scheduled and status updated
jenkinsClient.
EXPECT().
BuildJob(buildName, gomock.Any()).
Return(buildNumber, nil)
jenkinsClient.
EXPECT().
GetJob(buildName).
Return(&gojenkins.Job{
Raw: &gojenkins.JobResponse{
NextBuildNumber: buildNumber,
},
}, nil).AnyTimes()
done, err := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
assert.NoError(t, err)
assert.False(t, done)
jenkinsClient.
EXPECT().
BuildJob(buildName, gomock.Any()).
Return(int64(0), nil).AnyTimes()
err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
assert.NoError(t, err)
jenkinsClient.
EXPECT().
GetBuild(buildName, buildNumber).
Return(&gojenkins.Build{
Raw: &gojenkins.BuildResponse{
Result: SuccessStatus,
},
}, nil).AnyTimes()
assert.NotEmpty(t, jenkins.Status.Builds)
assert.Equal(t, len(jenkins.Status.Builds), 1)
done, err := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
assert.NoError(t, err)
build := jenkins.Status.Builds[0]
assert.Equal(t, build.Name, buildName)
assert.Equal(t, build.Hash, encodedHash)
assert.Equal(t, build.Number, buildNumber)
assert.Equal(t, build.Status, RunningStatus)
assert.Equal(t, build.Retires, 0)
assert.NotNil(t, build.CreateTime)
assert.NotNil(t, build.LastUpdateTime)
err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
assert.NoError(t, err)
// second run - build should be success and status updated
jenkinsClient.
EXPECT().
GetBuild(buildName, buildNumber).
Return(&gojenkins.Build{
Raw: &gojenkins.BuildResponse{
Result: SuccessStatus,
},
}, nil)
assert.NotEmpty(t, jenkins.Status.Builds)
assert.Equal(t, len(jenkins.Status.Builds), 1)
done, err = jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
assert.NoError(t, err)
assert.True(t, done)
build := jenkins.Status.Builds[0]
assert.Equal(t, build.Name, buildName)
assert.Equal(t, build.Hash, encodedHash)
assert.Equal(t, build.Number, buildNumber)
assert.Equal(t, build.Retires, 0)
assert.NotNil(t, build.CreateTime)
assert.NotNil(t, build.LastUpdateTime)
err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
assert.NoError(t, err)
// first run - build should be scheduled and status updated
if reconcileAttempt == 1 {
assert.False(t, done)
assert.Equal(t, build.Status, RunningStatus)
}
assert.NotEmpty(t, jenkins.Status.Builds)
assert.Equal(t, len(jenkins.Status.Builds), 1)
build = jenkins.Status.Builds[0]
assert.Equal(t, build.Name, buildName)
assert.Equal(t, build.Hash, encodedHash)
assert.Equal(t, build.Number, buildNumber)
assert.Equal(t, build.Status, SuccessStatus)
assert.Equal(t, build.Retires, 0)
assert.NotNil(t, build.CreateTime)
assert.NotNil(t, build.LastUpdateTime)
// second run -job should be success and status updated
if reconcileAttempt == 2 {
assert.True(t, done)
assert.Equal(t, build.Status, SuccessStatus)
}
}
}
func TestEnsureJobWithFailedBuild(t *testing.T) {
// given
ctx := context.TODO()
logger := logf.ZapLogger(false)
ctrl := gomock.NewController(t)
ctx := context.TODO()
defer ctrl.Finish()
buildName := "Test Job"
buildNumber := int64(1)
jenkinsClient := client.NewMockJenkins(ctrl)
fakeClient := fake.NewFakeClient()
hash := sha256.New()
hash.Write([]byte(buildName))
encodedHash := base64.URLEncoding.EncodeToString(hash.Sum(nil))
// when
jenkins := jenkinsCustomResource()
fakeClient := fake.NewFakeClient()
err := fakeClient.Create(ctx, jenkins)
assert.NoError(t, err)
jobs := New(jenkinsClient, fakeClient, logger)
for reconcileAttempt := 1; reconcileAttempt <= 4; reconcileAttempt++ {
logger.Info(fmt.Sprintf("Reconcile attempt #%d", reconcileAttempt))
jenkinsClient := client.NewMockJenkins(ctrl)
jobs := New(jenkinsClient, fakeClient, logger)
// first run - build should be scheduled and status updated
jenkinsClient.
EXPECT().
BuildJob(buildName, gomock.Any()).
Return(buildNumber, nil)
// first run - build should be scheduled and status updated
if reconcileAttempt == 1 {
jenkinsClient.
EXPECT().
GetJob(buildName).
Return(&gojenkins.Job{
Raw: &gojenkins.JobResponse{
NextBuildNumber: int64(1),
},
}, nil)
done, err := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
assert.NoError(t, err)
assert.False(t, done)
jenkinsClient.
EXPECT().
BuildJob(buildName, gomock.Any()).
Return(int64(0), nil)
}
err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
// second run - build should be failure and status updated
if reconcileAttempt == 2 {
jenkinsClient.
EXPECT().
GetBuild(buildName, int64(1)).
Return(&gojenkins.Build{
Raw: &gojenkins.BuildResponse{
Result: FailureStatus,
},
}, nil)
}
// third run - build should be rescheduled and status updated
if reconcileAttempt == 3 {
jenkinsClient.
EXPECT().
GetJob(buildName).
Return(&gojenkins.Job{
Raw: &gojenkins.JobResponse{
NextBuildNumber: int64(2),
},
}, nil)
jenkinsClient.
EXPECT().
BuildJob(buildName, gomock.Any()).
Return(int64(0), nil)
}
// fourth run - build should be success and status updated
if reconcileAttempt == 4 {
jenkinsClient.
EXPECT().
GetBuild(buildName, int64(2)).
Return(&gojenkins.Build{
Raw: &gojenkins.BuildResponse{
Result: SuccessStatus,
},
}, nil)
}
done, errEnsureBuildJob := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
assert.NoError(t, err)
err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
assert.NoError(t, err)
assert.NotEmpty(t, jenkins.Status.Builds)
assert.Equal(t, len(jenkins.Status.Builds), 1)
build := jenkins.Status.Builds[0]
assert.Equal(t, build.Name, buildName)
assert.Equal(t, build.Hash, encodedHash)
assert.NotNil(t, build.CreateTime)
assert.NotNil(t, build.LastUpdateTime)
// first run - build should be scheduled and status updated
if reconcileAttempt == 1 {
assert.NoError(t, errEnsureBuildJob)
assert.False(t, done)
assert.Equal(t, build.Number, int64(1))
assert.Equal(t, build.Status, RunningStatus)
}
// second run - build should be failure and status updated
if reconcileAttempt == 2 {
assert.Error(t, errEnsureBuildJob)
assert.False(t, done)
assert.Equal(t, build.Number, int64(1))
assert.Equal(t, build.Status, FailureStatus)
}
// third run - build should be rescheduled and status updated
if reconcileAttempt == 3 {
assert.NoError(t, errEnsureBuildJob)
assert.False(t, done)
assert.Equal(t, build.Number, int64(2))
assert.Equal(t, build.Status, RunningStatus)
}
// fourth run - build should be success and status updated
if reconcileAttempt == 4 {
assert.NoError(t, errEnsureBuildJob)
assert.True(t, done)
assert.Equal(t, build.Number, int64(2))
assert.Equal(t, build.Status, SuccessStatus)
}
}
}
func TestEnsureJobFailedWithMaxRetries(t *testing.T) {
// given
ctx := context.TODO()
logger := logf.ZapLogger(false)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
buildName := "Test Job"
hash := sha256.New()
hash.Write([]byte(buildName))
encodedHash := base64.URLEncoding.EncodeToString(hash.Sum(nil))
// when
jenkins := jenkinsCustomResource()
fakeClient := fake.NewFakeClient()
err := fakeClient.Create(ctx, jenkins)
assert.NoError(t, err)
assert.NotEmpty(t, jenkins.Status.Builds)
assert.Equal(t, len(jenkins.Status.Builds), 1)
BuildRetires = 1 // override max build retries
for reconcileAttempt := 1; reconcileAttempt <= 5; reconcileAttempt++ {
logger.Info(fmt.Sprintf("Reconcile attempt #%d", reconcileAttempt))
jenkinsClient := client.NewMockJenkins(ctrl)
jobs := New(jenkinsClient, fakeClient, logger)
build := jenkins.Status.Builds[0]
assert.Equal(t, build.Name, buildName)
assert.Equal(t, build.Hash, encodedHash)
assert.Equal(t, build.Number, buildNumber)
assert.Equal(t, build.Status, RunningStatus)
assert.Equal(t, build.Retires, 0)
assert.NotNil(t, build.CreateTime)
assert.NotNil(t, build.LastUpdateTime)
// first run - build should be scheduled and status updated
if reconcileAttempt == 1 {
jenkinsClient.
EXPECT().
GetJob(buildName).
Return(&gojenkins.Job{
Raw: &gojenkins.JobResponse{
NextBuildNumber: int64(1),
},
}, nil)
// second run - build should be failure and status updated
jenkinsClient.
EXPECT().
GetBuild(buildName, buildNumber).
Return(&gojenkins.Build{
Raw: &gojenkins.BuildResponse{
Result: FailureStatus,
},
}, nil)
jenkinsClient.
EXPECT().
BuildJob(buildName, gomock.Any()).
Return(int64(0), nil)
}
done, err = jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
assert.NoError(t, err)
assert.False(t, done)
// second run - build should be failure and status updated
if reconcileAttempt == 2 {
jenkinsClient.
EXPECT().
GetBuild(buildName, int64(1)).
Return(&gojenkins.Build{
Raw: &gojenkins.BuildResponse{
Result: FailureStatus,
},
}, nil)
}
err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
assert.NoError(t, err)
// third run - build should be rescheduled and status updated
if reconcileAttempt == 3 {
jenkinsClient.
EXPECT().
GetJob(buildName).
Return(&gojenkins.Job{
Raw: &gojenkins.JobResponse{
NextBuildNumber: int64(2),
},
}, nil)
assert.NotEmpty(t, jenkins.Status.Builds)
assert.Equal(t, len(jenkins.Status.Builds), 1)
jenkinsClient.
EXPECT().
BuildJob(buildName, gomock.Any()).
Return(int64(0), nil)
}
build = jenkins.Status.Builds[0]
assert.Equal(t, build.Name, buildName)
assert.Equal(t, build.Hash, encodedHash)
assert.Equal(t, build.Number, buildNumber)
assert.Equal(t, build.Status, FailureStatus)
assert.Equal(t, build.Retires, 0)
assert.NotNil(t, build.CreateTime)
assert.NotNil(t, build.LastUpdateTime)
// fourth run - build should be success and status updated
if reconcileAttempt == 4 {
jenkinsClient.
EXPECT().
GetBuild(buildName, int64(2)).
Return(&gojenkins.Build{
Raw: &gojenkins.BuildResponse{
Result: FailureStatus,
},
}, nil)
}
// third run - build should be rescheduled and status updated
jenkinsClient.
EXPECT().
BuildJob(buildName, gomock.Any()).
Return(buildNumber+1, nil)
done, errEnsureBuildJob := jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
assert.NoError(t, err)
done, err = jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
assert.EqualError(t, err, ErrorBuildFailed.Error())
assert.False(t, done)
err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
assert.NoError(t, err)
err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
assert.NoError(t, err)
assert.NotEmpty(t, jenkins.Status.Builds)
assert.Equal(t, len(jenkins.Status.Builds), 1)
assert.NotEmpty(t, jenkins.Status.Builds)
assert.Equal(t, len(jenkins.Status.Builds), 1)
build := jenkins.Status.Builds[0]
assert.Equal(t, build.Name, buildName)
assert.Equal(t, build.Hash, encodedHash)
build = jenkins.Status.Builds[0]
assert.Equal(t, build.Name, buildName)
assert.Equal(t, build.Hash, encodedHash)
assert.Equal(t, build.Number, buildNumber+1)
assert.Equal(t, build.Status, RunningStatus)
assert.Equal(t, build.Retires, 1)
assert.NotNil(t, build.CreateTime)
assert.NotNil(t, build.LastUpdateTime)
assert.NotNil(t, build.CreateTime)
assert.NotNil(t, build.LastUpdateTime)
// fourth run - build should be success and status updated
jenkinsClient.
EXPECT().
GetBuild(buildName, buildNumber+1).
Return(&gojenkins.Build{
Raw: &gojenkins.BuildResponse{
Result: SuccessStatus,
},
}, nil)
// first run - build should be scheduled and status updated
if reconcileAttempt == 1 {
assert.NoError(t, errEnsureBuildJob)
assert.False(t, done)
assert.Equal(t, build.Number, int64(1))
assert.Equal(t, build.Retires, 0)
assert.Equal(t, build.Status, RunningStatus)
}
done, err = jobs.EnsureBuildJob(buildName, encodedHash, nil, jenkins, true)
assert.NoError(t, err)
assert.True(t, done)
// second run - build should be failure and status updated
if reconcileAttempt == 2 {
assert.EqualError(t, errEnsureBuildJob, ErrorBuildFailed.Error())
assert.False(t, done)
assert.Equal(t, build.Number, int64(1))
assert.Equal(t, build.Retires, 0)
assert.Equal(t, build.Status, FailureStatus)
}
err = fakeClient.Get(ctx, types.NamespacedName{Name: jenkins.Name, Namespace: jenkins.Namespace}, jenkins)
assert.NoError(t, err)
// third run - build should be rescheduled and status updated
if reconcileAttempt == 3 {
assert.NoError(t, errEnsureBuildJob)
assert.False(t, done)
//assert.Equal(t, build.Retires, 1)
assert.Equal(t, build.Number, int64(2))
assert.Equal(t, build.Retires, 1)
assert.Equal(t, build.Status, RunningStatus)
}
assert.NotEmpty(t, jenkins.Status.Builds)
assert.Equal(t, len(jenkins.Status.Builds), 1)
// fourth run - build should be failure and status updated
if reconcileAttempt == 4 {
assert.EqualError(t, errEnsureBuildJob, ErrorBuildFailed.Error())
assert.False(t, done)
assert.Equal(t, build.Number, int64(2))
assert.Equal(t, build.Retires, 1)
assert.Equal(t, build.Status, FailureStatus)
}
build = jenkins.Status.Builds[0]
assert.Equal(t, build.Name, buildName)
assert.Equal(t, build.Hash, encodedHash)
assert.Equal(t, build.Number, buildNumber+1)
assert.Equal(t, build.Status, SuccessStatus)
assert.Equal(t, build.Retires, 1)
assert.NotNil(t, build.CreateTime)
assert.NotNil(t, build.LastUpdateTime)
// fifth run - build should be unrecoverable failed and status updated
if reconcileAttempt == 5 {
assert.EqualError(t, errEnsureBuildJob, ErrorUnrecoverableBuildFailed.Error())
assert.False(t, done)
assert.Equal(t, build.Number, int64(2))
assert.Equal(t, build.Retires, 1)
assert.Equal(t, build.Status, FailureStatus)
}
}
}
func jenkinsCustomResource() *virtuslabv1alpha1.Jenkins {