Merge pull request #84 from jakalkhalili/master

#69 Add support for seed job configuration

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.8.1
+	github.com/robfig/cron v1.2.0
 	github.com/sergi/go-diff v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.3
 	github.com/stretchr/testify v1.3.0

go.sum

@@ -185,6 +185,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
+github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=

internal/render/render.go

@@ -1,4 +1,4 @@
-package resources
+package render
 
 import (
 	"bytes"
@@ -7,8 +7,8 @@ import (
 	"github.com/pkg/errors"
 )
 
-// render executes a parsed template (go-template) with configuration from data
+// Render executes a parsed template (go-template) with configuration from data
-func render(template *template.Template, data interface{}) (string, error) {
+func Render(template *template.Template, data interface{}) (string, error) {
 	var buffer bytes.Buffer
 	if err := template.Execute(&buffer, data); err != nil {
 		return "", errors.WithStack(err)

pkg/apis/jenkins/v1alpha2/jenkins_types.go

@@ -528,6 +528,34 @@ type SeedJob struct {
 	// JenkinsCredentialType is the https://jenkinsci.github.io/kubernetes-credentials-provider-plugin/ credential type
 	// +optional
 	JenkinsCredentialType JenkinsCredentialType `json:"credentialType,omitempty"`
+
+	// GitHubPushTrigger is used for GitHub web hooks
+	// +optional
+	GitHubPushTrigger bool `json:"githubPushTrigger,omitempty"`
+
+	// BuildPeriodically is setting for scheduled trigger
+	// +optional
+	BuildPeriodically string `json:"buildPeriodically,omitempty"`
+
+	// PollSCM is setting for polling changes in SCM
+	// +optional
+	PollSCM string `json:"pollSCM,omitempty"`
+
+	// IgnoreMissingFiles is setting for Job DSL API plugin to ignore files that miss
+	// +optional
+	IgnoreMissingFiles bool `json:"ignoreMissingFiles,omitempty"`
+
+	// AdditionalClasspath is setting for Job DSL API plugin to set Additional Classpath
+	// +optional
+	AdditionalClasspath string `json:"additionalClasspath,omitempty"`
+
+	// FailOnMissingPlugin is setting for Job DSL API plugin that fails job if required plugin is missing
+	// +optional
+	FailOnMissingPlugin bool `json:"failOnMissingPlugin,omitempty"`
+
+	// UnstableOnDeprecation is setting for Job DSL API plugin that sets build status as unstable if build using deprecated features
+	// +optional
+	UnstableOnDeprecation bool `json:"unstableOnDeprecation,omitempty"`
 }
 
 // Handler defines a specific action that should be taken

pkg/controller/jenkins/configuration/base/resources/init_configuration_configmap.go

@@ -2,10 +2,11 @@ package resources
 
 import (
 	"fmt"
-	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
 	"text/template"
 
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
+	"github.com/jenkinsci/kubernetes-operator/internal/render"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -48,7 +49,7 @@ func buildCreateJenkinsOperatorUserGroovyScript() (*string, error) {
 		OperatorUserCreatedFilePath: jenkinsHomePath + "/operatorUserCreated",
 	}
 
-	output, err := render(createOperatorUserGroovyFmtTemplate, data)
+	output, err := render.Render(createOperatorUserGroovyFmtTemplate, data)
 	if err != nil {
 		return nil, err
 	}

pkg/controller/jenkins/configuration/base/resources/scripts_configmap.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
+	"github.com/jenkinsci/kubernetes-operator/internal/render"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -287,7 +288,7 @@ func buildInitBashScript(jenkins *v1alpha2.Jenkins) (*string, error) {
 		JenkinsScriptsVolumePath: JenkinsScriptsVolumePath,
 	}
 
-	output, err := render(initBashTemplate, data)
+	output, err := render.Render(initBashTemplate, data)
 	if err != nil {
 		return nil, err
 	}

pkg/controller/jenkins/configuration/user/seedjobs/seedjobs.go

@@ -6,7 +6,9 @@ import (
 	"encoding/base64"
 	"fmt"
 	"reflect"
+	"text/template"
 
+	"github.com/jenkinsci/kubernetes-operator/internal/render"
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
 	jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
@@ -39,8 +41,94 @@ const (
 
 	// AgentName is the name of seed job agent
 	AgentName = "jnlp"
+
+	creatingGroovyScriptName = "seed-job-groovy-script.groovy"
 )
 
+var seedJobGroovyScriptTemplate = template.Must(template.New(creatingGroovyScriptName).Parse(`
+import hudson.model.FreeStyleProject;
+import hudson.plugins.git.GitSCM;
+import hudson.plugins.git.BranchSpec;
+import hudson.triggers.SCMTrigger;
+import hudson.triggers.TimerTrigger;
+import hudson.util.Secret;
+import javaposse.jobdsl.plugin.*;
+import jenkins.model.Jenkins;
+import jenkins.model.JenkinsLocationConfiguration;
+import com.cloudbees.plugins.credentials.CredentialsScope;
+import com.cloudbees.plugins.credentials.domains.Domain;
+import com.cloudbees.plugins.credentials.SystemCredentialsProvider;
+import jenkins.model.JenkinsLocationConfiguration;
+import org.jenkinsci.plugins.workflow.job.WorkflowJob;
+import org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition;
+{{ if .GitHubPushTrigger }}
+import com.cloudbees.jenkins.GitHubPushTrigger;
+{{ end }}
+import hudson.model.FreeStyleProject;
+import hudson.model.labels.LabelAtom;
+import hudson.plugins.git.BranchSpec;
+import hudson.plugins.git.GitSCM;
+import hudson.plugins.git.SubmoduleConfig;
+import hudson.plugins.git.extensions.impl.CloneOption;
+import javaposse.jobdsl.plugin.ExecuteDslScripts;
+import javaposse.jobdsl.plugin.LookupStrategy;
+import javaposse.jobdsl.plugin.RemovedJobAction;
+import javaposse.jobdsl.plugin.RemovedViewAction;
+
+import static com.google.common.collect.Lists.newArrayList;
+
+Jenkins jenkins = Jenkins.instance
+
+def jobDslSeedName = "{{ .ID }} - {{ .SeedJobSuffix }}";
+def jobRef = jenkins.getItem(jobDslSeedName)
+
+def repoList = GitSCM.createRepoList("{{ .RepositoryURL }}", "{{ .CredentialID }}")
+def gitExtensions = [new CloneOption(true, true, ";", 10)]
+def scm = new GitSCM(
+        repoList,
+        newArrayList(new BranchSpec("{{ .RepositoryBranch }}")),
+        false,
+        Collections.<SubmoduleConfig>emptyList(),
+        null,
+        null,
+        gitExtensions
+)
+
+def executeDslScripts = new ExecuteDslScripts()
+executeDslScripts.setTargets("{{ .Targets }}")
+executeDslScripts.setSandbox(false)
+executeDslScripts.setRemovedJobAction(RemovedJobAction.DELETE)
+executeDslScripts.setRemovedViewAction(RemovedViewAction.DELETE)
+executeDslScripts.setLookupStrategy(LookupStrategy.SEED_JOB)
+executeDslScripts.setAdditionalClasspath("{{ .AdditionalClasspath }}")
+executeDslScripts.setFailOnMissingPlugin({{ .FailOnMissingPlugin }})
+executeDslScripts.setUnstableOnDeprecation({{ .UnstableOnDeprecation }})
+executeDslScripts.setIgnoreMissingFiles({{ .IgnoreMissingFiles }})
+
+if (jobRef == null) {
+        jobRef = jenkins.createProject(FreeStyleProject, jobDslSeedName)
+}
+
+jobRef.getBuildersList().clear()
+jobRef.getBuildersList().add(executeDslScripts)
+jobRef.setDisplayName("Seed Job from {{ .ID }}")
+jobRef.setScm(scm)
+
+{{ if .PollSCM }}
+jobRef.addTrigger(new SCMTrigger("{{ .PollSCM }}"))
+{{ end }}
+
+{{ if .GitHubPushTrigger }}
+jobRef.addTrigger(new GitHubPushTrigger())
+{{ end }}
+
+{{ if .BuildPeriodically }}
+jobRef.addTrigger(new TimerTrigger("{{ .BuildPeriodically }}"))
+{{ end}}
+jobRef.setAssignedLabel(new LabelAtom("{{ .AgentName }}"))
+jenkins.getQueue().schedule(jobRef)
+`))
+
 // SeedJobs defines API for configuring and ensuring Jenkins Seed Jobs and Deploy Keys
 type SeedJobs struct {
 	jenkinsClient jenkinsclient.Jenkins
@@ -112,7 +200,10 @@ func (s *SeedJobs) createJobs(jenkins *v1alpha2.Jenkins) (requeue bool, err erro
 			return true, err
 		}
 
-		groovyScript := seedJobCreatingGroovyScript(seedJob)
+		groovyScript, err := seedJobCreatingGroovyScript(seedJob)
+		if err != nil {
+			return true, err
+		}
 
 		hash := sha256.New()
 		hash.Write([]byte(groovyScript))
@@ -316,72 +407,43 @@ func agentDeployment(jenkinsManifest *v1alpha2.Jenkins, namespace string, agentN
 	}
 }
 
-func seedJobCreatingGroovyScript(s v1alpha2.SeedJob) string {
+func seedJobCreatingGroovyScript(seedJob v1alpha2.SeedJob) (string, error) {
-	return `
+	data := struct {
-import hudson.model.FreeStyleProject;
+		ID                    string
-import hudson.plugins.git.GitSCM;
+		CredentialID          string
-import hudson.plugins.git.BranchSpec;
+		Targets               string
-import hudson.triggers.SCMTrigger;
+		RepositoryBranch      string
-import hudson.util.Secret;
+		RepositoryURL         string
-import javaposse.jobdsl.plugin.*;
+		GitHubPushTrigger     bool
-import jenkins.model.Jenkins;
+		BuildPeriodically     string
-import jenkins.model.JenkinsLocationConfiguration;
+		PollSCM               string
-import com.cloudbees.plugins.credentials.CredentialsScope;
+		IgnoreMissingFiles    bool
-import com.cloudbees.plugins.credentials.domains.Domain;
+		AdditionalClasspath   string
-import com.cloudbees.plugins.credentials.SystemCredentialsProvider;
+		FailOnMissingPlugin   bool
-import jenkins.model.JenkinsLocationConfiguration;
+		UnstableOnDeprecation bool
-import org.jenkinsci.plugins.workflow.job.WorkflowJob;
+		SeedJobSuffix         string
-import org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition;
+		AgentName             string
+	}{
+		ID:                    seedJob.ID,
+		CredentialID:          seedJob.CredentialID,
+		Targets:               seedJob.Targets,
+		RepositoryBranch:      seedJob.RepositoryBranch,
+		RepositoryURL:         seedJob.RepositoryURL,
+		GitHubPushTrigger:     seedJob.GitHubPushTrigger,
+		BuildPeriodically:     seedJob.BuildPeriodically,
+		PollSCM:               seedJob.PollSCM,
+		IgnoreMissingFiles:    seedJob.IgnoreMissingFiles,
+		AdditionalClasspath:   seedJob.AdditionalClasspath,
+		FailOnMissingPlugin:   seedJob.FailOnMissingPlugin,
+		UnstableOnDeprecation: seedJob.UnstableOnDeprecation,
+		SeedJobSuffix:         constants.SeedJobSuffix,
+		AgentName:             AgentName,
+	}
 
-import hudson.model.FreeStyleProject
+	output, err := render.Render(seedJobGroovyScriptTemplate, data)
-import hudson.model.labels.LabelAtom
+	if err != nil {
-import hudson.plugins.git.BranchSpec
+		return "", err
-import hudson.plugins.git.GitSCM
+	}
-import hudson.plugins.git.SubmoduleConfig
-import hudson.plugins.git.extensions.impl.CloneOption
-import javaposse.jobdsl.plugin.ExecuteDslScripts
-import javaposse.jobdsl.plugin.LookupStrategy
-import javaposse.jobdsl.plugin.RemovedJobAction
-import javaposse.jobdsl.plugin.RemovedViewAction
 
-import static com.google.common.collect.Lists.newArrayList
+	return output, nil
-
-Jenkins jenkins = Jenkins.instance
-
-def jobDslSeedName = "` + s.ID + `-` + constants.SeedJobSuffix + `";
-def jobRef = jenkins.getItem(jobDslSeedName)
-
-def repoList = GitSCM.createRepoList("` + s.RepositoryURL + `", "` + s.CredentialID + `")
-def gitExtensions = [new CloneOption(true, true, ";", 10)]
-def scm = new GitSCM(
-        repoList,
-        newArrayList(new BranchSpec("` + s.RepositoryBranch + `")),
-        false,
-        Collections.<SubmoduleConfig>emptyList(),
-        null,
-        null,
-        gitExtensions
-)
-
-def executeDslScripts = new ExecuteDslScripts()
-executeDslScripts.setTargets("` + s.Targets + `")
-executeDslScripts.setSandbox(false)
-executeDslScripts.setRemovedJobAction(RemovedJobAction.DELETE)
-executeDslScripts.setRemovedViewAction(RemovedViewAction.DELETE)
-executeDslScripts.setLookupStrategy(LookupStrategy.SEED_JOB)
-
-if (jobRef == null) {
-        jobRef = jenkins.createProject(FreeStyleProject, jobDslSeedName)
-}
-
-jobRef.getBuildersList().clear()
-jobRef.getBuildersList().add(executeDslScripts)
-jobRef.setDisplayName("` + fmt.Sprintf("Seed Job from %s", s.ID) + `")
-jobRef.setScm(scm)
-
-jobRef.setAssignedLabel(new LabelAtom("` + AgentName + `"))
-
-jenkins.getQueue().schedule(jobRef)
-
-`
 }

pkg/controller/jenkins/configuration/user/seedjobs/seedjobs_test.go

@@ -87,10 +87,13 @@ func TestEnsureSeedJobs(t *testing.T) {
 			},
 		}
 
+		seedJobCreatingScript, err := seedJobCreatingGroovyScript(jenkins.Spec.SeedJobs[0])
+		assert.NoError(t, err)
+
 		jenkinsClient.EXPECT().GetNode(agentName).Return(nil, nil).AnyTimes()
 		jenkinsClient.EXPECT().CreateNode(agentName, 1, "The jenkins-operator generated agent", "/home/jenkins", agentName).Return(testNode, nil).AnyTimes()
 		jenkinsClient.EXPECT().GetNodeSecret(agentName).Return(agentSecret, nil).AnyTimes()
-		jenkinsClient.EXPECT().ExecuteScript(seedJobCreatingGroovyScript(jenkins.Spec.SeedJobs[0])).AnyTimes()
+		jenkinsClient.EXPECT().ExecuteScript(seedJobCreatingScript).AnyTimes()
 
 		seedJobClient := New(jenkinsClient, fakeClient, logger)
 

pkg/controller/jenkins/configuration/user/seedjobs/validate.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/go-logr/logr"
 	stackerr "github.com/pkg/errors"
+	"github.com/robfig/cron"
 	"k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
@@ -87,10 +88,60 @@ func (r *SeedJobs) ValidateSeedJobs(jenkins v1alpha2.Jenkins) (bool, error) {
 				}
 			}
 		}
+
+		if len(seedJob.BuildPeriodically) > 0 {
+			if !r.validateSchedule(seedJob, seedJob.BuildPeriodically, "buildPeriodically") {
+				valid = false
+			}
+		}
+
+		if len(seedJob.PollSCM) > 0 {
+			if !r.validateSchedule(seedJob, seedJob.PollSCM, "pollSCM") {
+				valid = false
+			}
+		}
+
+		if seedJob.GitHubPushTrigger {
+			if !r.validateGitHubPushTrigger(jenkins) {
+				valid = false
+			}
+		}
 	}
+
 	return valid, nil
 }
 
+func (r *SeedJobs) validateSchedule(job v1alpha2.SeedJob, str string, key string) bool {
+	_, err := cron.Parse(str)
+	if err != nil {
+		r.logger.V(log.VWarn).Info(fmt.Sprintf("`%s` schedule '%s' is invalid cron spec in `%s`", key, str, job.ID))
+		return false
+	}
+	return true
+}
+
+func (r *SeedJobs) validateGitHubPushTrigger(jenkins v1alpha2.Jenkins) bool {
+	exists := false
+	for _, plugin := range jenkins.Spec.Master.BasePlugins {
+		if plugin.Name == "github" {
+			exists = true
+		}
+	}
+
+	userExists := false
+	for _, plugin := range jenkins.Spec.Master.Plugins {
+		if plugin.Name == "github" {
+			userExists = true
+		}
+	}
+
+	if !exists && !userExists {
+		r.logger.V(log.VWarn).Info("githubPushTrigger is set. This function requires `github` plugin installed in .Spec.Master.Plugins because seed jobs Push Trigger function needs it")
+		return false
+	}
+	return true
+}
+
 func (r *SeedJobs) validateIfIDIsUnique(seedJobs []v1alpha2.SeedJob) bool {
 	ids := map[string]bool{}
 	for _, seedJob := range seedJobs {

pkg/controller/jenkins/configuration/user/seedjobs/validate_test.go

@@ -442,6 +442,104 @@ func TestValidateSeedJobs(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Equal(t, false, result)
 	})
+	t.Run("Invalid with wrong cron spec", func(t *testing.T) {
+		jenkins := v1alpha2.Jenkins{
+			Spec: v1alpha2.JenkinsSpec{
+				SeedJobs: []v1alpha2.SeedJob{
+					{
+						ID:                    "example",
+						CredentialID:          "jenkins-operator-e2e",
+						JenkinsCredentialType: v1alpha2.NoJenkinsCredentialCredentialType,
+						Targets:               "cicd/jobs/*.jenkins",
+						RepositoryBranch:      "master",
+						RepositoryURL:         "https://github.com/jenkinsci/kubernetes-operator.git",
+						BuildPeriodically:     "invalid-cron-spec",
+					},
+				},
+			},
+		}
+
+		seedJobs := New(nil, fake.NewFakeClient(), logf.ZapLogger(false))
+		result, err := seedJobs.ValidateSeedJobs(jenkins)
+
+		assert.NoError(t, err)
+		assert.False(t, result)
+	})
+	t.Run("Valid with good cron spec", func(t *testing.T) {
+		jenkins := v1alpha2.Jenkins{
+			Spec: v1alpha2.JenkinsSpec{
+				SeedJobs: []v1alpha2.SeedJob{
+					{
+						ID:                    "example",
+						CredentialID:          "jenkins-operator-e2e",
+						JenkinsCredentialType: v1alpha2.NoJenkinsCredentialCredentialType,
+						Targets:               "cicd/jobs/*.jenkins",
+						RepositoryBranch:      "master",
+						RepositoryURL:         "https://github.com/jenkinsci/kubernetes-operator.git",
+						BuildPeriodically:     "1 2 3 4 5",
+						PollSCM:               "1 2 3 4 5",
+					},
+				},
+			},
+		}
+
+		seedJobs := New(nil, fake.NewFakeClient(), logf.ZapLogger(false))
+		result, err := seedJobs.ValidateSeedJobs(jenkins)
+
+		assert.NoError(t, err)
+		assert.True(t, result)
+	})
+	t.Run("Invalid with set githubPushTrigger and not installed github plugin", func(t *testing.T) {
+		jenkins := v1alpha2.Jenkins{
+			Spec: v1alpha2.JenkinsSpec{
+				SeedJobs: []v1alpha2.SeedJob{
+					{
+						ID:                    "example",
+						CredentialID:          "jenkins-operator-e2e",
+						JenkinsCredentialType: v1alpha2.NoJenkinsCredentialCredentialType,
+						Targets:               "cicd/jobs/*.jenkins",
+						RepositoryBranch:      "master",
+						RepositoryURL:         "https://github.com/jenkinsci/kubernetes-operator.git",
+						GitHubPushTrigger:     true,
+					},
+				},
+			},
+		}
+
+		seedJobs := New(nil, fake.NewFakeClient(), logf.ZapLogger(false))
+		result, err := seedJobs.ValidateSeedJobs(jenkins)
+
+		assert.NoError(t, err)
+		assert.False(t, result)
+	})
+	t.Run("Invalid with set githubPushTrigger and not installed github plugin", func(t *testing.T) {
+		jenkins := v1alpha2.Jenkins{
+			Spec: v1alpha2.JenkinsSpec{
+				SeedJobs: []v1alpha2.SeedJob{
+					{
+						ID:                    "example",
+						CredentialID:          "jenkins-operator-e2e",
+						JenkinsCredentialType: v1alpha2.NoJenkinsCredentialCredentialType,
+						Targets:               "cicd/jobs/*.jenkins",
+						RepositoryBranch:      "master",
+						RepositoryURL:         "https://github.com/jenkinsci/kubernetes-operator.git",
+						GitHubPushTrigger:     true,
+					},
+				},
+				Master: v1alpha2.JenkinsMaster{
+					Plugins: []v1alpha2.Plugin{
+						{Name: "github", Version: "latest"},
+					},
+				},
+			},
+		}
+
+		seedJobs := New(nil, fake.NewFakeClient(), logf.ZapLogger(false))
+		result, err := seedJobs.ValidateSeedJobs(jenkins)
+
+		assert.NoError(t, err)
+		assert.True(t, result)
+	})
 }
 
 func TestValidateIfIDIsUnique(t *testing.T) {