From dda0a0075ea7a8ad35200401d54e46c4fd8c5032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20S=C4=99k?= Date: Sat, 29 Jun 2019 12:19:35 +0200 Subject: [PATCH] #29 Restart Jenkins pod when any seed job has been deleted --- pkg/apis/jenkins/v1alpha2/jenkins_types.go | 4 ++ .../jenkins/configuration/base/reconcile.go | 20 +++--- .../jenkins/configuration/user/reconcile.go | 2 +- .../configuration/user/seedjobs/seedjobs.go | 58 +++++++++++++++ .../user/seedjobs/seedjobs_test.go | 70 ++++++++++++++++++- 5 files changed, 142 insertions(+), 12 deletions(-) diff --git a/pkg/apis/jenkins/v1alpha2/jenkins_types.go b/pkg/apis/jenkins/v1alpha2/jenkins_types.go index 16463fe5..8cf14afd 100644 --- a/pkg/apis/jenkins/v1alpha2/jenkins_types.go +++ b/pkg/apis/jenkins/v1alpha2/jenkins_types.go @@ -334,6 +334,10 @@ type JenkinsStatus struct { // UserAndPasswordHash is a SHA256 hash made from user and password // +optional UserAndPasswordHash string `json:"userAndPasswordHash,omitempty"` + + // CreatedSeedJobs contains list of seed job id already created in Jenkins + // +optional + CreatedSeedJobs []string `json:"createdSeedJobs,omitempty"` } // BuildStatus defines type of Jenkins build job status diff --git a/pkg/controller/jenkins/configuration/base/reconcile.go b/pkg/controller/jenkins/configuration/base/reconcile.go index 3eac48b5..5e0ec682 100644 --- a/pkg/controller/jenkins/configuration/base/reconcile.go +++ b/pkg/controller/jenkins/configuration/base/reconcile.go @@ -121,7 +121,7 @@ func (r *ReconcileJenkinsBaseConfiguration) Reconcile() (reconcile.Result, jenki } if !ok { r.logger.Info("Some plugins have changed, restarting Jenkins") - return reconcile.Result{Requeue: true}, nil, r.restartJenkinsMasterPod(metaObject) + return reconcile.Result{Requeue: true}, nil, r.restartJenkinsMasterPod() } result, err = r.ensureBaseConfiguration(jenkinsClient) @@ -368,7 +368,7 @@ func (r *ReconcileJenkinsBaseConfiguration) createService(meta metav1.ObjectMeta return stackerr.WithStack(r.updateResource(&service)) } -func (r *ReconcileJenkinsBaseConfiguration) getJenkinsMasterPod(meta metav1.ObjectMeta) (*corev1.Pod, error) { +func (r *ReconcileJenkinsBaseConfiguration) getJenkinsMasterPod() (*corev1.Pod, error) { jenkinsMasterPodName := resources.GetJenkinsMasterPodName(*r.jenkins) currentJenkinsMasterPod := &corev1.Pod{} err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: jenkinsMasterPodName, Namespace: r.jenkins.Namespace}, currentJenkinsMasterPod) @@ -385,7 +385,7 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureJenkinsMasterPod(meta metav1.O } // Check if this Pod already exists - currentJenkinsMasterPod, err := r.getJenkinsMasterPod(meta) + currentJenkinsMasterPod, err := r.getJenkinsMasterPod() if err != nil && errors.IsNotFound(err) { jenkinsMasterPod := resources.NewJenkinsMasterPod(meta, r.jenkins) if !reflect.DeepEqual(jenkinsMasterPod.Spec.Containers[0].Command, resources.GetJenkinsMasterContainerBaseCommand()) { @@ -433,7 +433,7 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureJenkinsMasterPod(meta metav1.O return reconcile.Result{Requeue: true}, nil } if currentJenkinsMasterPod != nil && r.isRecreatePodNeeded(*currentJenkinsMasterPod, userAndPasswordHash) { - return reconcile.Result{Requeue: true}, r.restartJenkinsMasterPod(meta) + return reconcile.Result{Requeue: true}, r.restartJenkinsMasterPod() } return reconcile.Result{}, nil @@ -638,8 +638,8 @@ func (r *ReconcileJenkinsBaseConfiguration) compareVolumes(actualPod corev1.Pod) ) } -func (r *ReconcileJenkinsBaseConfiguration) restartJenkinsMasterPod(meta metav1.ObjectMeta) error { - currentJenkinsMasterPod, err := r.getJenkinsMasterPod(meta) +func (r *ReconcileJenkinsBaseConfiguration) restartJenkinsMasterPod() error { + currentJenkinsMasterPod, err := r.getJenkinsMasterPod() if err != nil { return err } @@ -648,7 +648,7 @@ func (r *ReconcileJenkinsBaseConfiguration) restartJenkinsMasterPod(meta metav1. } func (r *ReconcileJenkinsBaseConfiguration) detectJenkinsMasterPodStartingIssues(meta metav1.ObjectMeta) (stopReconcileLoop bool, err error) { - jenkinsMasterPod, err := r.getJenkinsMasterPod(meta) + jenkinsMasterPod, err := r.getJenkinsMasterPod() if err != nil { return false, err } @@ -696,7 +696,7 @@ func (r *ReconcileJenkinsBaseConfiguration) filterEvents(source corev1.EventList } func (r *ReconcileJenkinsBaseConfiguration) waitForJenkins(meta metav1.ObjectMeta) (reconcile.Result, error) { - jenkinsMasterPod, err := r.getJenkinsMasterPod(meta) + jenkinsMasterPod, err := r.getJenkinsMasterPod() if err != nil { return reconcile.Result{}, err } @@ -715,7 +715,7 @@ func (r *ReconcileJenkinsBaseConfiguration) waitForJenkins(meta metav1.ObjectMet for _, containerStatus := range jenkinsMasterPod.Status.ContainerStatuses { if containerStatus.State.Terminated != nil { r.logger.Info(fmt.Sprintf("Container '%s' is terminated, status '%+v', recreating pod", containerStatus.Name, containerStatus)) - return reconcile.Result{Requeue: true}, r.restartJenkinsMasterPod(meta) + return reconcile.Result{Requeue: true}, r.restartJenkinsMasterPod() } if !containerStatus.Ready { r.logger.V(log.VDebug).Info(fmt.Sprintf("Container '%s' not ready, readiness probe failed", containerStatus.Name)) @@ -743,7 +743,7 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureJenkinsClient(meta metav1.Obje if err != nil { return nil, stackerr.WithStack(err) } - currentJenkinsMasterPod, err := r.getJenkinsMasterPod(meta) + currentJenkinsMasterPod, err := r.getJenkinsMasterPod() if err != nil { return nil, err } diff --git a/pkg/controller/jenkins/configuration/user/reconcile.go b/pkg/controller/jenkins/configuration/user/reconcile.go index 4afcddaf..c172dfc2 100644 --- a/pkg/controller/jenkins/configuration/user/reconcile.go +++ b/pkg/controller/jenkins/configuration/user/reconcile.go @@ -98,7 +98,7 @@ func (r *ReconcileUserConfiguration) ensureSeedJobs() (reconcile.Result, error) } // build not finished yet - requeue reconciliation loop with timeout if !done { - return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10}, nil + return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } return reconcile.Result{}, nil } diff --git a/pkg/controller/jenkins/configuration/user/seedjobs/seedjobs.go b/pkg/controller/jenkins/configuration/user/seedjobs/seedjobs.go index f7f025e4..26eccdc8 100644 --- a/pkg/controller/jenkins/configuration/user/seedjobs/seedjobs.go +++ b/pkg/controller/jenkins/configuration/user/seedjobs/seedjobs.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/base64" "fmt" + "reflect" "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2" jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client" @@ -61,6 +62,11 @@ func New(jenkinsClient jenkinsclient.Jenkins, k8sClient k8s.Client, logger logr. // EnsureSeedJobs configures seed job and runs it for every entry from Jenkins.Spec.SeedJobs func (s *SeedJobs) EnsureSeedJobs(jenkins *v1alpha2.Jenkins) (done bool, err error) { + if s.isRecreatePodNeeded(*jenkins) { + s.logger.Info("Some seed job has been deleted, recreating pod") + return false, s.restartJenkinsMasterPod(*jenkins) + } + if err = s.createJob(); err != nil { s.logger.V(log.VWarn).Info("Couldn't create jenkins seed job") return false, err @@ -75,6 +81,13 @@ func (s *SeedJobs) EnsureSeedJobs(jenkins *v1alpha2.Jenkins) (done bool, err err s.logger.V(log.VWarn).Info("Couldn't build jenkins seed job") return false, err } + + seedJobIDs := s.getAllSeedJobIDs(*jenkins) + if done && !reflect.DeepEqual(seedJobIDs, jenkins.Status.CreatedSeedJobs) { + jenkins.Status.CreatedSeedJobs = seedJobIDs + return false, stackerr.WithStack(s.k8sClient.Update(context.TODO(), jenkins)) + } + return done, nil } @@ -175,6 +188,51 @@ func (s *SeedJobs) credentialValue(namespace string, seedJob v1alpha2.SeedJob) ( return "", nil } +func (s *SeedJobs) getAllSeedJobIDs(jenkins v1alpha2.Jenkins) []string { + var ids []string + for _, seedJob := range jenkins.Spec.SeedJobs { + ids = append(ids, seedJob.ID) + } + return ids +} + +//TODO move to k8sClient +func (s *SeedJobs) getJenkinsMasterPod(jenkins v1alpha2.Jenkins) (*corev1.Pod, error) { + jenkinsMasterPodName := resources.GetJenkinsMasterPodName(jenkins) + currentJenkinsMasterPod := &corev1.Pod{} + err := s.k8sClient.Get(context.TODO(), types.NamespacedName{Name: jenkinsMasterPodName, Namespace: jenkins.Namespace}, currentJenkinsMasterPod) + if err != nil { + return nil, err // don't wrap error + } + return currentJenkinsMasterPod, nil +} + +//TODO move to k8sClient +func (s *SeedJobs) restartJenkinsMasterPod(jenkins v1alpha2.Jenkins) error { + currentJenkinsMasterPod, err := s.getJenkinsMasterPod(jenkins) + if err != nil { + return err + } + s.logger.Info(fmt.Sprintf("Terminating Jenkins Master Pod %s/%s", currentJenkinsMasterPod.Namespace, currentJenkinsMasterPod.Name)) + return stackerr.WithStack(s.k8sClient.Delete(context.TODO(), currentJenkinsMasterPod)) +} + +func (s *SeedJobs) isRecreatePodNeeded(jenkins v1alpha2.Jenkins) bool { + for _, createdSeedJob := range jenkins.Status.CreatedSeedJobs { + found := false + for _, seedJob := range jenkins.Spec.SeedJobs { + if createdSeedJob == seedJob.ID { + found = true + break + } + } + if !found { + return true + } + } + return false +} + // seedJobConfigXML this is the XML representation of seed job var seedJobConfigXML = ` diff --git a/pkg/controller/jenkins/configuration/user/seedjobs/seedjobs_test.go b/pkg/controller/jenkins/configuration/user/seedjobs/seedjobs_test.go index e2ae3824..3dda0eb3 100644 --- a/pkg/controller/jenkins/configuration/user/seedjobs/seedjobs_test.go +++ b/pkg/controller/jenkins/configuration/user/seedjobs/seedjobs_test.go @@ -105,7 +105,7 @@ func TestEnsureSeedJobs(t *testing.T) { // second run - should update and finish job if reconcileAttempt == 2 { - assert.True(t, done) + assert.False(t, done) assert.Equal(t, string(v1alpha2.BuildSuccessStatus), string(build.Status)) } @@ -151,3 +151,71 @@ func jenkinsCustomResource() *v1alpha2.Jenkins { }, } } + +func TestSeedJobs_isRecreatePodNeeded(t *testing.T) { + seedJobsClient := New(nil, nil, nil) + t.Run("empty", func(t *testing.T) { + jenkins := v1alpha2.Jenkins{} + + got := seedJobsClient.isRecreatePodNeeded(jenkins) + + assert.False(t, got) + }) + t.Run("same", func(t *testing.T) { + jenkins := v1alpha2.Jenkins{ + Spec: v1alpha2.JenkinsSpec{ + SeedJobs: []v1alpha2.SeedJob{ + { + ID: "name", + }, + }, + }, + Status: v1alpha2.JenkinsStatus{ + CreatedSeedJobs: []string{"name"}, + }, + } + + got := seedJobsClient.isRecreatePodNeeded(jenkins) + + assert.False(t, got) + }) + t.Run("removed one", func(t *testing.T) { + jenkins := v1alpha2.Jenkins{ + Spec: v1alpha2.JenkinsSpec{ + SeedJobs: []v1alpha2.SeedJob{ + { + ID: "name1", + }, + }, + }, + Status: v1alpha2.JenkinsStatus{ + CreatedSeedJobs: []string{"name1", "name2"}, + }, + } + + got := seedJobsClient.isRecreatePodNeeded(jenkins) + + assert.True(t, got) + }) + t.Run("renamed one", func(t *testing.T) { + jenkins := v1alpha2.Jenkins{ + Spec: v1alpha2.JenkinsSpec{ + SeedJobs: []v1alpha2.SeedJob{ + { + ID: "name1", + }, + { + ID: "name3", + }, + }, + }, + Status: v1alpha2.JenkinsStatus{ + CreatedSeedJobs: []string{"name1", "name2"}, + }, + } + + got := seedJobsClient.isRecreatePodNeeded(jenkins) + + assert.True(t, got) + }) +}