#29 Restart Jenkins pod when any seed job has been deleted

pkg/apis/jenkins/v1alpha2/jenkins_types.go

@@ -334,6 +334,10 @@ type JenkinsStatus struct {
 	// UserAndPasswordHash is a SHA256 hash made from user and password
 	// +optional
 	UserAndPasswordHash string `json:"userAndPasswordHash,omitempty"`
+
+	// CreatedSeedJobs contains list of seed job id already created in Jenkins
+	// +optional
+	CreatedSeedJobs []string `json:"createdSeedJobs,omitempty"`
 }
 
 // BuildStatus defines type of Jenkins build job status

pkg/controller/jenkins/configuration/base/reconcile.go

@@ -121,7 +121,7 @@ func (r *ReconcileJenkinsBaseConfiguration) Reconcile() (reconcile.Result, jenki
 	}
 	if !ok {
 		r.logger.Info("Some plugins have changed, restarting Jenkins")
-		return reconcile.Result{Requeue: true}, nil, r.restartJenkinsMasterPod(metaObject)
+		return reconcile.Result{Requeue: true}, nil, r.restartJenkinsMasterPod()
 	}
 
 	result, err = r.ensureBaseConfiguration(jenkinsClient)
@@ -368,7 +368,7 @@ func (r *ReconcileJenkinsBaseConfiguration) createService(meta metav1.ObjectMeta
 	return stackerr.WithStack(r.updateResource(&service))
 }
 
-func (r *ReconcileJenkinsBaseConfiguration) getJenkinsMasterPod(meta metav1.ObjectMeta) (*corev1.Pod, error) {
+func (r *ReconcileJenkinsBaseConfiguration) getJenkinsMasterPod() (*corev1.Pod, error) {
 	jenkinsMasterPodName := resources.GetJenkinsMasterPodName(*r.jenkins)
 	currentJenkinsMasterPod := &corev1.Pod{}
 	err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: jenkinsMasterPodName, Namespace: r.jenkins.Namespace}, currentJenkinsMasterPod)
@@ -385,7 +385,7 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureJenkinsMasterPod(meta metav1.O
 	}
 
 	// Check if this Pod already exists
-	currentJenkinsMasterPod, err := r.getJenkinsMasterPod(meta)
+	currentJenkinsMasterPod, err := r.getJenkinsMasterPod()
 	if err != nil && errors.IsNotFound(err) {
 		jenkinsMasterPod := resources.NewJenkinsMasterPod(meta, r.jenkins)
 		if !reflect.DeepEqual(jenkinsMasterPod.Spec.Containers[0].Command, resources.GetJenkinsMasterContainerBaseCommand()) {
@@ -433,7 +433,7 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureJenkinsMasterPod(meta metav1.O
 		return reconcile.Result{Requeue: true}, nil
 	}
 	if currentJenkinsMasterPod != nil && r.isRecreatePodNeeded(*currentJenkinsMasterPod, userAndPasswordHash) {
-		return reconcile.Result{Requeue: true}, r.restartJenkinsMasterPod(meta)
+		return reconcile.Result{Requeue: true}, r.restartJenkinsMasterPod()
 	}
 
 	return reconcile.Result{}, nil
@@ -638,8 +638,8 @@ func (r *ReconcileJenkinsBaseConfiguration) compareVolumes(actualPod corev1.Pod)
 	)
 }
 
-func (r *ReconcileJenkinsBaseConfiguration) restartJenkinsMasterPod(meta metav1.ObjectMeta) error {
+func (r *ReconcileJenkinsBaseConfiguration) restartJenkinsMasterPod() error {
-	currentJenkinsMasterPod, err := r.getJenkinsMasterPod(meta)
+	currentJenkinsMasterPod, err := r.getJenkinsMasterPod()
 	if err != nil {
 		return err
 	}
@@ -648,7 +648,7 @@ func (r *ReconcileJenkinsBaseConfiguration) restartJenkinsMasterPod(meta metav1.
 }
 
 func (r *ReconcileJenkinsBaseConfiguration) detectJenkinsMasterPodStartingIssues(meta metav1.ObjectMeta) (stopReconcileLoop bool, err error) {
-	jenkinsMasterPod, err := r.getJenkinsMasterPod(meta)
+	jenkinsMasterPod, err := r.getJenkinsMasterPod()
 	if err != nil {
 		return false, err
 	}
@@ -696,7 +696,7 @@ func (r *ReconcileJenkinsBaseConfiguration) filterEvents(source corev1.EventList
 }
 
 func (r *ReconcileJenkinsBaseConfiguration) waitForJenkins(meta metav1.ObjectMeta) (reconcile.Result, error) {
-	jenkinsMasterPod, err := r.getJenkinsMasterPod(meta)
+	jenkinsMasterPod, err := r.getJenkinsMasterPod()
 	if err != nil {
 		return reconcile.Result{}, err
 	}
@@ -715,7 +715,7 @@ func (r *ReconcileJenkinsBaseConfiguration) waitForJenkins(meta metav1.ObjectMet
 	for _, containerStatus := range jenkinsMasterPod.Status.ContainerStatuses {
 		if containerStatus.State.Terminated != nil {
 			r.logger.Info(fmt.Sprintf("Container '%s' is terminated, status '%+v', recreating pod", containerStatus.Name, containerStatus))
-			return reconcile.Result{Requeue: true}, r.restartJenkinsMasterPod(meta)
+			return reconcile.Result{Requeue: true}, r.restartJenkinsMasterPod()
 		}
 		if !containerStatus.Ready {
 			r.logger.V(log.VDebug).Info(fmt.Sprintf("Container '%s' not ready, readiness probe failed", containerStatus.Name))
@@ -743,7 +743,7 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureJenkinsClient(meta metav1.Obje
 	if err != nil {
 		return nil, stackerr.WithStack(err)
 	}
-	currentJenkinsMasterPod, err := r.getJenkinsMasterPod(meta)
+	currentJenkinsMasterPod, err := r.getJenkinsMasterPod()
 	if err != nil {
 		return nil, err
 	}

pkg/controller/jenkins/configuration/user/reconcile.go

@@ -98,7 +98,7 @@ func (r *ReconcileUserConfiguration) ensureSeedJobs() (reconcile.Result, error)
 	}
 	// build not finished yet - requeue reconciliation loop with timeout
 	if !done {
-		return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10}, nil
+		return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil
 	}
 	return reconcile.Result{}, nil
 }

pkg/controller/jenkins/configuration/user/seedjobs/seedjobs.go

@@ -5,6 +5,7 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
+	"reflect"
 
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
 	jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
@@ -61,6 +62,11 @@ func New(jenkinsClient jenkinsclient.Jenkins, k8sClient k8s.Client, logger logr.
 
 // EnsureSeedJobs configures seed job and runs it for every entry from Jenkins.Spec.SeedJobs
 func (s *SeedJobs) EnsureSeedJobs(jenkins *v1alpha2.Jenkins) (done bool, err error) {
+	if s.isRecreatePodNeeded(*jenkins) {
+		s.logger.Info("Some seed job has been deleted, recreating pod")
+		return false, s.restartJenkinsMasterPod(*jenkins)
+	}
+
 	if err = s.createJob(); err != nil {
 		s.logger.V(log.VWarn).Info("Couldn't create jenkins seed job")
 		return false, err
@@ -75,6 +81,13 @@ func (s *SeedJobs) EnsureSeedJobs(jenkins *v1alpha2.Jenkins) (done bool, err err
 		s.logger.V(log.VWarn).Info("Couldn't build jenkins seed job")
 		return false, err
 	}
+
+	seedJobIDs := s.getAllSeedJobIDs(*jenkins)
+	if done && !reflect.DeepEqual(seedJobIDs, jenkins.Status.CreatedSeedJobs) {
+		jenkins.Status.CreatedSeedJobs = seedJobIDs
+		return false, stackerr.WithStack(s.k8sClient.Update(context.TODO(), jenkins))
+	}
+
 	return done, nil
 }
 
@@ -175,6 +188,51 @@ func (s *SeedJobs) credentialValue(namespace string, seedJob v1alpha2.SeedJob) (
 	return "", nil
 }
 
+func (s *SeedJobs) getAllSeedJobIDs(jenkins v1alpha2.Jenkins) []string {
+	var ids []string
+	for _, seedJob := range jenkins.Spec.SeedJobs {
+		ids = append(ids, seedJob.ID)
+	}
+	return ids
+}
+
+//TODO move to k8sClient
+func (s *SeedJobs) getJenkinsMasterPod(jenkins v1alpha2.Jenkins) (*corev1.Pod, error) {
+	jenkinsMasterPodName := resources.GetJenkinsMasterPodName(jenkins)
+	currentJenkinsMasterPod := &corev1.Pod{}
+	err := s.k8sClient.Get(context.TODO(), types.NamespacedName{Name: jenkinsMasterPodName, Namespace: jenkins.Namespace}, currentJenkinsMasterPod)
+	if err != nil {
+		return nil, err // don't wrap error
+	}
+	return currentJenkinsMasterPod, nil
+}
+
+//TODO move to k8sClient
+func (s *SeedJobs) restartJenkinsMasterPod(jenkins v1alpha2.Jenkins) error {
+	currentJenkinsMasterPod, err := s.getJenkinsMasterPod(jenkins)
+	if err != nil {
+		return err
+	}
+	s.logger.Info(fmt.Sprintf("Terminating Jenkins Master Pod %s/%s", currentJenkinsMasterPod.Namespace, currentJenkinsMasterPod.Name))
+	return stackerr.WithStack(s.k8sClient.Delete(context.TODO(), currentJenkinsMasterPod))
+}
+
+func (s *SeedJobs) isRecreatePodNeeded(jenkins v1alpha2.Jenkins) bool {
+	for _, createdSeedJob := range jenkins.Status.CreatedSeedJobs {
+		found := false
+		for _, seedJob := range jenkins.Spec.SeedJobs {
+			if createdSeedJob == seedJob.ID {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return true
+		}
+	}
+	return false
+}
+
 // seedJobConfigXML this is the XML representation of seed job
 var seedJobConfigXML = `
 <flow-definition plugin="workflow-job@2.30">

pkg/controller/jenkins/configuration/user/seedjobs/seedjobs_test.go

@@ -105,7 +105,7 @@ func TestEnsureSeedJobs(t *testing.T) {
 
 		// second run - should update and finish job
 		if reconcileAttempt == 2 {
-			assert.True(t, done)
+			assert.False(t, done)
 			assert.Equal(t, string(v1alpha2.BuildSuccessStatus), string(build.Status))
 		}
 
@@ -151,3 +151,71 @@ func jenkinsCustomResource() *v1alpha2.Jenkins {
 		},
 	}
 }
+
+func TestSeedJobs_isRecreatePodNeeded(t *testing.T) {
+	seedJobsClient := New(nil, nil, nil)
+	t.Run("empty", func(t *testing.T) {
+		jenkins := v1alpha2.Jenkins{}
+
+		got := seedJobsClient.isRecreatePodNeeded(jenkins)
+
+		assert.False(t, got)
+	})
+	t.Run("same", func(t *testing.T) {
+		jenkins := v1alpha2.Jenkins{
+			Spec: v1alpha2.JenkinsSpec{
+				SeedJobs: []v1alpha2.SeedJob{
+					{
+						ID: "name",
+					},
+				},
+			},
+			Status: v1alpha2.JenkinsStatus{
+				CreatedSeedJobs: []string{"name"},
+			},
+		}
+
+		got := seedJobsClient.isRecreatePodNeeded(jenkins)
+
+		assert.False(t, got)
+	})
+	t.Run("removed one", func(t *testing.T) {
+		jenkins := v1alpha2.Jenkins{
+			Spec: v1alpha2.JenkinsSpec{
+				SeedJobs: []v1alpha2.SeedJob{
+					{
+						ID: "name1",
+					},
+				},
+			},
+			Status: v1alpha2.JenkinsStatus{
+				CreatedSeedJobs: []string{"name1", "name2"},
+			},
+		}
+
+		got := seedJobsClient.isRecreatePodNeeded(jenkins)
+
+		assert.True(t, got)
+	})
+	t.Run("renamed one", func(t *testing.T) {
+		jenkins := v1alpha2.Jenkins{
+			Spec: v1alpha2.JenkinsSpec{
+				SeedJobs: []v1alpha2.SeedJob{
+					{
+						ID: "name1",
+					},
+					{
+						ID: "name3",
+					},
+				},
+			},
+			Status: v1alpha2.JenkinsStatus{
+				CreatedSeedJobs: []string{"name1", "name2"},
+			},
+		}
+
+		got := seedJobsClient.isRecreatePodNeeded(jenkins)
+
+		assert.True(t, got)
+	})
+}