Initial version of jobs API

pkg/controller/jenkins/configuration/user/reconcile.go

@@ -1,10 +1,12 @@
 package user
 
 import (
+	"time"
+
 	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
 	jenkinsclient "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/client"
 	"github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/user/seedjobs"
-
+	"github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/jobs"
 	"github.com/go-logr/logr"
 	k8s "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -31,10 +33,25 @@ func New(k8sClient k8s.Client, jenkinsClient jenkinsclient.Jenkins, logger logr.
 
 // Reconcile it's a main reconciliation loop for user supplied configuration
 func (r *ReconcileUserConfiguration) Reconcile() (*reconcile.Result, error) {
-	err := seedjobs.EnsureSeedJobs(r.jenkinsClient, r.k8sClient, r.jenkins)
+	return r.reconcileSeedJobs()
+}
+
+func (r *ReconcileUserConfiguration) reconcileSeedJobs() (*reconcile.Result, error) {
+	seedJobs := seedjobs.New(r.jenkinsClient, r.k8sClient, r.logger)
+	done, err := seedJobs.EnsureSeedJobs(r.jenkins)
 	if err != nil {
+		if err == jobs.ErrorBuildFailed {
+			return &reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10}, nil
+		}
+
+		if err == jobs.ErrorEmptyJenkinsCR || err == jobs.ErrorUncoverBuildFailed {
+			return nil, nil
+		}
+
 		return &reconcile.Result{}, err
 	}
-
+	if !done {
+		return &reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10}, nil
+	}
 	return nil, nil
 }

pkg/controller/jenkins/configuration/user/seedjobs/seedjobs.go

@@ -2,11 +2,15 @@ package seedjobs
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/base64"
 	"fmt"
 
 	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
 	jenkinsclient "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/client"
 
+	"github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/jobs"
+	"github.com/go-logr/logr"
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	k8s "sigs.k8s.io/controller-runtime/pkg/client"
@@ -24,70 +28,88 @@ const (
 	displayNameParameterName      = "SEED_JOB_DISPLAY_NAME"
 )
 
+// SeedJobs defines API for configuring and ensuring Jenkins Seed Jobs and Deploy Keys
+type SeedJobs struct {
+	jenkinsClient jenkinsclient.Jenkins
+	k8sClient     k8s.Client
+	logger        logr.Logger
+}
+
+// New creates seedJobs client
+func New(jenkinsClient jenkinsclient.Jenkins, k8sClient k8s.Client, logger logr.Logger) *SeedJobs {
+	return &SeedJobs{
+		jenkinsClient: jenkinsClient,
+		k8sClient:     k8sClient,
+		logger:        logger,
+	}
+}
+
 // EnsureSeedJobs configures seed job and runs it for every entry from Jenkins.Spec.SeedJobs
-func EnsureSeedJobs(jenkinsClient jenkinsclient.Jenkins, k8sClient k8s.Client, jenkins *virtuslabv1alpha1.Jenkins) error {
-	err := configureSeedJob(jenkinsClient)
+func (s *SeedJobs) EnsureSeedJobs(jenkins *virtuslabv1alpha1.Jenkins) (done bool, err error) {
+	err = s.configureSeedJob()
 	if err != nil {
-		return err
+		return false, err
 	}
-	err = buildAndVerifySeedJobs(jenkinsClient, k8sClient, jenkins)
+	done, err = s.buildSeedJobs(jenkins)
+	if err != nil {
+		return false, err
+	}
+	return done, nil
+}
+
+func (s *SeedJobs) configureSeedJob() error {
+	_, err := s.jenkinsClient.CreateOrUpdateJob(seedJobConfigXML, ConfigureSeedJobsName)
 	if err != nil {
 		return err
 	}
 	return nil
 }
 
-func configureSeedJob(jenkinsClient jenkinsclient.Jenkins) error {
-	_, err := jenkinsClient.CreateOrUpdateJob(seedJobConfigXML, ConfigureSeedJobsName)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func buildAndVerifySeedJobs(jenkinsClient jenkinsclient.Jenkins, k8sClient k8s.Client, jenkins *virtuslabv1alpha1.Jenkins) error {
+func (s *SeedJobs) buildSeedJobs(jenkins *virtuslabv1alpha1.Jenkins) (done bool, err error) {
+	allDone := true
 	seedJobs := jenkins.Spec.SeedJobs
 	for _, seedJob := range seedJobs {
-		privateKey, err := privateKeyFromSecret(k8sClient, jenkins.Namespace, seedJob)
+		privateKey, err := s.privateKeyFromSecret(jenkins.Namespace, seedJob)
 		if err != nil {
-			return err
+			return false, err
 		}
-		err = buildAndVerifySeedJob(
-			jenkinsClient,
-			seedJob.ID,
-			privateKey,
-			seedJob.RepositoryURL,
-			seedJob.RepositoryBranch, seedJob.Targets, fmt.Sprintf("Seed Job from %s", seedJob.ID))
+		parameters := map[string]string{
+			deployKeyIDParameterName:      seedJob.ID,
+			privateKeyParameterName:       privateKey,
+			repositoryURLParameterName:    seedJob.RepositoryURL,
+			repositoryBranchParameterName: seedJob.RepositoryBranch,
+			targetsParameterName:          seedJob.Targets,
+			displayNameParameterName:      fmt.Sprintf("Seed Job from %s", seedJob.ID),
+		}
+
+		hash := sha256.New()
+		hash.Write([]byte(parameters[deployKeyIDParameterName]))
+		hash.Write([]byte(parameters[privateKeyParameterName]))
+		hash.Write([]byte(parameters[repositoryURLParameterName]))
+		hash.Write([]byte(parameters[repositoryBranchParameterName]))
+		hash.Write([]byte(parameters[targetsParameterName]))
+		hash.Write([]byte(parameters[displayNameParameterName]))
+		encodedHash := base64.URLEncoding.EncodeToString(hash.Sum(nil))
+
+		jobsClient := jobs.New(s.jenkinsClient, s.k8sClient, s.logger)
+		done, err := jobsClient.EnsureBuildJob(ConfigureSeedJobsName, encodedHash, parameters, jenkins, true)
 		if err != nil {
-			return err
+			return false, err
+		}
+
+		if !done {
+			allDone = false
 		}
 	}
-	return nil
+
+	return allDone, nil
 }
 
-func buildAndVerifySeedJob(jenkinsClient jenkinsclient.Jenkins, deployKeyID, privateKey, repositoryURL, repositoryBranch, targets, displayName string) error {
-	// FIXME this function should build job and verify job status when finished (state in cr status)
-	// requeue when job is running and check job status next time
-	options := map[string]string{
-		deployKeyIDParameterName:      deployKeyID,
-		privateKeyParameterName:       privateKey,
-		repositoryURLParameterName:    repositoryURL,
-		repositoryBranchParameterName: repositoryBranch,
-		targetsParameterName:          targets,
-		displayNameParameterName:      displayName,
-	}
-	_, err := jenkinsClient.BuildJob(ConfigureSeedJobsName, options)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func privateKeyFromSecret(k8sClient k8s.Client, namespace string, seedJob virtuslabv1alpha1.SeedJob) (string, error) {
+func (s *SeedJobs) privateKeyFromSecret(namespace string, seedJob virtuslabv1alpha1.SeedJob) (string, error) {
 	if seedJob.PrivateKey.SecretKeyRef != nil {
 		deployKeySecret := &v1.Secret{}
 		namespaceName := types.NamespacedName{Namespace: namespace, Name: seedJob.PrivateKey.SecretKeyRef.Name}
-		err := k8sClient.Get(context.TODO(), namespaceName, deployKeySecret)
+		err := s.k8sClient.Get(context.TODO(), namespaceName, deployKeySecret)
 		if err != nil {
 			return "", err
 		}

pkg/controller/jenkins/configuration/user/seedjobs/seedjobs_test.go

@@ -0,0 +1 @@
+package seedjobs

pkg/controller/jenkins/jobs/doc.go

@@ -0,0 +1,2 @@
+// Package jobs implements common jenkins jobs operations
+package jobs

pkg/controller/jenkins/jobs/jobs.go

@@ -0,0 +1,339 @@
+package jobs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
+	"github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/client"
+	"github.com/VirtusLab/jenkins-operator/pkg/log"
+	"github.com/go-logr/logr"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8s "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// SuccessStatus - the build had no errors
+	SuccessStatus = "success"
+	// UnstableStatus - the build had some errors but they were not fatal. For example, some tests failed
+	UnstableStatus = "unstable"
+	// NotBuildStatus - this status code is used in a multi-stage build (like maven2) where a problem in earlier stage prevented later stages from building
+	NotBuildStatus = "not_build"
+	// FailureStatus - the build had a fatal error
+	FailureStatus = "failure"
+	// AbortedStatus - the build was manually aborted
+	AbortedStatus = "aborted"
+	// RunningStatus - this is custom build status for running build, not present in jenkins build result
+	RunningStatus = "running"
+	// ExpiredStatus - this is custom build status for expired build, not present in jenkins build result
+	ExpiredStatus = "expired"
+	// MaxBuildRetires - determines max amount of retires for failed build
+	MaxBuildRetires = 3
+)
+
+var (
+	// ErrorEmptyJenkinsCR - this is custom error returned when jenkins custom resource is empty
+	ErrorEmptyJenkinsCR = errors.New("empty jenkins cr")
+	// ErrorUnexpectedBuildStatus - this is custom error returned when jenkins build has unexpected status
+	ErrorUnexpectedBuildStatus = errors.New("unexpected build status")
+	// ErrorBuildFailed - this is custom error returned when jenkins build has failed
+	ErrorBuildFailed = errors.New("build failed")
+	// ErrorAbortBuildFailed - this is custom error returned when jenkins build couldn't be aborted
+	ErrorAbortBuildFailed = errors.New("build failed")
+	// ErrorUncoverBuildFailed - this is custom error returned when jenkins build has failed and cannot be recovered
+	ErrorUncoverBuildFailed = errors.New("build failed and cannot be recovered")
+	// ErrorNotFound - this is error returned when jenkins build couldn't be found
+	ErrorNotFound = errors.New("404")
+)
+
+// Jobs defines Jobs API tailored for operator sdk
+type Jobs struct {
+	jenkinsClient client.Jenkins
+	logger        logr.Logger
+	k8sClient     k8s.Client
+}
+
+// New creates jobs client
+func New(jenkinsClient client.Jenkins, k8sClient k8s.Client, logger logr.Logger) *Jobs {
+	return &Jobs{
+		jenkinsClient: jenkinsClient,
+		k8sClient:     k8sClient,
+		logger:        logger,
+	}
+}
+
+// EnsureBuildJob function takes care of jenkins build lifecycle according to the lifecycle of reconciliation loop
+// implementation guarantees that jenkins build can be properly handled even after operator pod restart
+// entire state is saved in Jenkins.Status.Builds section
+// function return 'true' when build finished successfully or false when reconciliation loop should requeue this function
+// preserveStatus determines that build won't be removed from Jenkins.Status.Builds section
+func (jobs *Jobs) EnsureBuildJob(name, hash string, parameters map[string]string, jenkins *virtuslabv1alpha1.Jenkins, preserveStatus bool) (done bool, err error) {
+	jobs.logger.Info(fmt.Sprintf("Ensuring build, name:'%s' hash:'%s'", name, hash))
+
+	build, err := jobs.getBuildFromStatus(name, hash, jenkins)
+	if err != nil {
+		return false, err
+	}
+
+	if build != nil {
+		jobs.logger.Info(fmt.Sprintf("Build exists in status, name:'%s' hash:'%s' status: '%s'", name, hash, build.Status))
+		switch strings.ToLower(build.Status) {
+		case SuccessStatus:
+			return jobs.ensureSuccessBuild(*build, jenkins, preserveStatus)
+		case RunningStatus:
+			return jobs.ensureRunningBuild(*build, jenkins, preserveStatus)
+		case UnstableStatus, NotBuildStatus, FailureStatus, AbortedStatus:
+			return jobs.ensureFailedBuild(*build, jenkins, parameters, preserveStatus)
+		case ExpiredStatus:
+			return jobs.ensureExpiredBuild(*build, jenkins, preserveStatus)
+		default:
+			jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Unexpected build status, name:'%s' hash:'%s' status:'%s'", name, hash, build.Status))
+			return false, ErrorUnexpectedBuildStatus
+		}
+	}
+
+	// build is run first time - build job and update status
+	jobs.logger.Info(fmt.Sprintf("Build doesn't exist, running and updating status, name:'%s' hash:'%s'", name, hash))
+	return jobs.buildJob(name, hash, parameters, jenkins)
+}
+
+func (jobs *Jobs) getBuildFromStatus(name string, hash string, jenkins *virtuslabv1alpha1.Jenkins) (*virtuslabv1alpha1.Build, error) {
+	if jenkins != nil {
+		builds := jenkins.Status.Builds
+		for _, build := range builds {
+			if build.Name == name && build.Hash == hash {
+				return &build, nil
+			}
+		}
+	}
+	return nil, nil
+}
+
+func (jobs *Jobs) ensureSuccessBuild(build virtuslabv1alpha1.Build, jenkins *virtuslabv1alpha1.Jenkins, preserveStatus bool) (bool, error) {
+	if jenkins == nil {
+		jobs.logger.V(log.VWarn).Info("Jenkins CR is empty")
+		return false, ErrorEmptyJenkinsCR
+	}
+
+	jobs.logger.Info(fmt.Sprintf("Ensuring success build, name:'%s' hash:'%s'", build.Name, build.Hash))
+
+	if !preserveStatus {
+		err := jobs.removeBuildFromStatus(build, jenkins)
+		jobs.logger.Info(fmt.Sprintf("Removing build from status, name:'%s' hash:'%s'", build.Name, build.Hash))
+		if err != nil {
+			jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't remove build from status, name:'%s' hash:'%s'", build.Name, build.Hash))
+			return false, err
+		}
+	}
+	return true, nil
+}
+
+func (jobs *Jobs) ensureRunningBuild(build virtuslabv1alpha1.Build, jenkins *virtuslabv1alpha1.Jenkins, preserveStatus bool) (bool, error) {
+	if jenkins == nil {
+		jobs.logger.V(log.VWarn).Info("Jenkins CR is empty")
+		return false, ErrorEmptyJenkinsCR
+	}
+
+	jobs.logger.Info(fmt.Sprintf("Ensuring running build, name:'%s' hash:'%s'", build.Name, build.Hash))
+	// FIXME (antoniaklja) implement build expiration
+
+	jenkinsBuild, err := jobs.jenkinsClient.GetBuild(build.Name, build.Number)
+	if isNotFoundError(err) {
+		jobs.logger.Info(fmt.Sprintf("Build still running , name:'%s' hash:'%s'", build.Name, build.Hash))
+		return false, nil
+	} else if err != nil {
+		jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't get jenkins build, name:'%s' number:'%d'", build.Name, build.Number))
+		return false, err
+	}
+
+	if jenkinsBuild.GetResult() != "" {
+		build.Status = strings.ToLower(jenkinsBuild.GetResult())
+	}
+
+	jobs.logger.Info(fmt.Sprintf("Updating build status, name:'%s' hash:'%s' status:'%s'", build.Name, build.Hash, build.Status))
+	err = jobs.updateBuildStatus(build, jenkins)
+	if err != nil {
+		jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't update build status, name:'%s' hash:'%s'", build.Name, build.Hash))
+		return false, err
+	}
+
+	if build.Status == SuccessStatus {
+		jobs.logger.Info(fmt.Sprintf("Build finished successfully, name:'%s' hash:'%s' status:'%s'", build.Name, build.Hash, build.Status))
+		if !preserveStatus {
+			jobs.logger.Info(fmt.Sprintf("Removing build from status, name:'%s' hash:'%s'", build.Name, build.Hash))
+			err := jobs.removeBuildFromStatus(build, jenkins)
+			if err != nil {
+				jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't remove build from status, name:'%s' hash:'%s'", build.Name, build.Hash))
+				return false, err
+			}
+		}
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func (jobs *Jobs) ensureFailedBuild(build virtuslabv1alpha1.Build, jenkins *virtuslabv1alpha1.Jenkins, parameters map[string]string, preserveStatus bool) (bool, error) {
+	if jenkins == nil {
+		jobs.logger.V(log.VWarn).Info("Jenkins CR is empty")
+		return false, ErrorEmptyJenkinsCR
+	}
+
+	jobs.logger.Info(fmt.Sprintf("Ensuring failed build, name:'%s' hash:'%s' status: '%s'", build.Name, build.Hash, build.Status))
+
+	if build.Retires < MaxBuildRetires {
+		jobs.logger.Info(fmt.Sprintf("Retrying build, name:'%s' hash:'%s' retries: '%d'", build.Name, build.Hash, build.Retires))
+		_, err := jobs.buildJob(build.Name, build.Hash, parameters, jenkins)
+		if err != nil {
+			jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't retry build, name:'%s' hash:'%s'", build.Name, build.Hash))
+			return false, err
+		}
+		return false, ErrorBuildFailed
+	}
+
+	jobs.logger.Info(fmt.Sprintf("The retries limit was reached , name:'%s' hash:'%s' retries: '%d'", build.Name, build.Hash, build.Retires))
+
+	if !preserveStatus {
+		jobs.logger.Info(fmt.Sprintf("Removing build from status, name:'%s' hash:'%s'", build.Name, build.Hash))
+		err := jobs.removeBuildFromStatus(build, jenkins)
+		if err != nil {
+			jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't remove build from status, name:'%s' hash:'%s'", build.Name, build.Hash))
+			return false, err
+		}
+	}
+	return false, ErrorUncoverBuildFailed
+}
+
+func (jobs *Jobs) ensureExpiredBuild(build virtuslabv1alpha1.Build, jenkins *virtuslabv1alpha1.Jenkins, preserveStatus bool) (bool, error) {
+	if jenkins == nil {
+		jobs.logger.V(log.VWarn).Info("Jenkins CR is empty")
+		return false, ErrorEmptyJenkinsCR
+	}
+
+	jobs.logger.Info(fmt.Sprintf("Ensuring expired build, name:'%s' hash:'%s' status: '%s'", build.Name, build.Hash, build.Status))
+
+	jenkinsBuild, err := jobs.jenkinsClient.GetBuild(build.Name, build.Number)
+	if err != nil {
+		return false, err
+	}
+
+	_, err = jenkinsBuild.Stop()
+	if err != nil {
+		return false, err
+	}
+
+	jenkinsBuild, err = jobs.jenkinsClient.GetBuild(build.Name, build.Number)
+	if err != nil {
+		return false, err
+	}
+
+	if jenkinsBuild.GetResult() != AbortedStatus {
+		return false, ErrorAbortBuildFailed
+	}
+
+	err = jobs.updateBuildStatus(build, jenkins)
+	if err != nil {
+		return false, err
+	}
+
+	// TODO(antoniaklja) clean up k8s resources
+
+	if !preserveStatus {
+		jobs.logger.Info(fmt.Sprintf("Removing build from status, name:'%s' hash:'%s'", build.Name, build.Hash))
+		err = jobs.removeBuildFromStatus(build, jenkins)
+		if err != nil {
+			jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't remove build from status, name:'%s' hash:'%s'", build.Name, build.Hash))
+			return false, err
+		}
+	}
+
+	return true, nil
+}
+
+func (jobs *Jobs) removeBuildFromStatus(build virtuslabv1alpha1.Build, jenkins *virtuslabv1alpha1.Jenkins) error {
+	if jenkins == nil {
+		return ErrorEmptyJenkinsCR
+	}
+
+	builds := make([]virtuslabv1alpha1.Build, len(jenkins.Status.Builds), len(jenkins.Status.Builds))
+	for _, existingBuild := range jenkins.Status.Builds {
+		if existingBuild.Name != build.Name && existingBuild.Hash != build.Hash {
+			builds = append(builds, existingBuild)
+		}
+	}
+	jenkins.Status.Builds = builds
+	err := jobs.k8sClient.Update(context.TODO(), jenkins)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (jobs *Jobs) buildJob(name string, hash string, parameters map[string]string, jenkins *virtuslabv1alpha1.Jenkins) (bool, error) {
+	if jenkins == nil {
+		return false, ErrorEmptyJenkinsCR
+	}
+
+	jobs.logger.Info(fmt.Sprintf("Running build, name:'%s' hash:'%s'", name, hash))
+
+	number, err := jobs.jenkinsClient.BuildJob(name, parameters)
+	if err != nil {
+		jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't run build, name:'%s' hash:'%s' number:'%d'", name, hash, number))
+		return false, err
+	}
+
+	build := virtuslabv1alpha1.Build{
+		Name:   name,
+		Number: number,
+		Hash:   hash,
+		Status: RunningStatus,
+	}
+
+	jobs.logger.Info(fmt.Sprintf("Updating build status, name:'%s' hash:'%s' status:'%s'", name, hash, build.Status))
+	err = jobs.updateBuildStatus(build, jenkins)
+	if err != nil {
+		jobs.logger.V(log.VWarn).Info(fmt.Sprintf("Couldn't update build status, name:'%s' hash:'%s'", build.Name, build.Hash))
+		return false, err
+	}
+	return false, nil
+}
+
+func (jobs *Jobs) updateBuildStatus(build virtuslabv1alpha1.Build, jenkins *virtuslabv1alpha1.Jenkins) error {
+	if jenkins == nil {
+		return ErrorEmptyJenkinsCR
+	}
+
+	// get index of existing build from status if exists
+	buildIndex := -1
+	for index, existingBuild := range jenkins.Status.Builds {
+		if build.Name == existingBuild.Name && build.Hash == existingBuild.Hash {
+			buildIndex = index
+		}
+	}
+
+	// update build status
+	now := metav1.Now()
+	build.LastUpdateTime = &now
+	if buildIndex >= 0 {
+		jenkins.Status.Builds[buildIndex] = build
+	} else {
+		build.CreateTime = &now
+		jenkins.Status.Builds = append(jenkins.Status.Builds, build)
+	}
+	err := jobs.k8sClient.Update(context.TODO(), jenkins)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func isNotFoundError(err error) bool {
+	if err != nil {
+		return err.Error() == ErrorNotFound.Error()
+	}
+	return false
+}

pkg/controller/jenkins/jobs/jobs_test.go

@@ -0,0 +1 @@
+package jobs