public_git
/
kubernetes-operator
mirror of https://github.com/jenkinsci/kubernetes-operator.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #84 from jakalkhalili/master 

#69 Add support for seed job configuration
This commit is contained in:
Tomasz Sęk 2019-09-02 15:06:40 +02:00 committed by GitHub
parent 878aac1725 833a95f2e8
commit 36d936d1fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 321 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -31,6 +31,7 @@ require (
github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/errors v0.8.1
github.com/robfig/cron v1.2.0
github.com/sergi/go-diff v1.0.0 // indirect
github.com/spf13/pflag v1.0.3
github.com/stretchr/testify v1.3.0

2
go.sum
View File

@ -185,6 +185,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=

6
pkg/controller/jenkins/configuration/base/resources/render.go → internal/render/render.go
View File

@ -1,4 +1,4 @@
package resources
package render
import (
"bytes"
@ -7,8 +7,8 @@ import (
"github.com/pkg/errors"
)
// render executes a parsed template (go-template) with configuration from data
func render(template *template.Template, data interface{}) (string, error) {
// Render executes a parsed template (go-template) with configuration from data
func Render(template *template.Template, data interface{}) (string, error) {
var buffer bytes.Buffer
if err := template.Execute(&buffer, data); err != nil {
return "", errors.WithStack(err)

28
pkg/apis/jenkins/v1alpha2/jenkins_types.go
View File

@ -528,6 +528,34 @@ type SeedJob struct {
// JenkinsCredentialType is the https://jenkinsci.github.io/kubernetes-credentials-provider-plugin/ credential type
// +optional
JenkinsCredentialType JenkinsCredentialType `json:"credentialType,omitempty"`
// GitHubPushTrigger is used for GitHub web hooks
// +optional
GitHubPushTrigger bool `json:"githubPushTrigger,omitempty"`
// BuildPeriodically is setting for scheduled trigger
// +optional
BuildPeriodically string `json:"buildPeriodically,omitempty"`
// PollSCM is setting for polling changes in SCM
// +optional
PollSCM string `json:"pollSCM,omitempty"`
// IgnoreMissingFiles is setting for Job DSL API plugin to ignore files that miss
// +optional
IgnoreMissingFiles bool `json:"ignoreMissingFiles,omitempty"`
// AdditionalClasspath is setting for Job DSL API plugin to set Additional Classpath
// +optional
AdditionalClasspath string `json:"additionalClasspath,omitempty"`
// FailOnMissingPlugin is setting for Job DSL API plugin that fails job if required plugin is missing
// +optional
FailOnMissingPlugin bool `json:"failOnMissingPlugin,omitempty"`
// UnstableOnDeprecation is setting for Job DSL API plugin that sets build status as unstable if build using deprecated features
// +optional
UnstableOnDeprecation bool `json:"unstableOnDeprecation,omitempty"`
}
// Handler defines a specific action that should be taken

5
pkg/controller/jenkins/configuration/base/resources/init_configuration_configmap.go
View File

@ -2,10 +2,11 @@ package resources
import (
"fmt"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
"text/template"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
"github.com/jenkinsci/kubernetes-operator/internal/render"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -48,7 +49,7 @@ func buildCreateJenkinsOperatorUserGroovyScript() (*string, error) {
OperatorUserCreatedFilePath: jenkinsHomePath + "/operatorUserCreated",
}
output, err := render(createOperatorUserGroovyFmtTemplate, data)
output, err := render.Render(createOperatorUserGroovyFmtTemplate, data)
if err != nil {
return nil, err
}

3
pkg/controller/jenkins/configuration/base/resources/scripts_configmap.go
View File

@ -6,6 +6,7 @@ import (
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
"github.com/jenkinsci/kubernetes-operator/internal/render"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -287,7 +288,7 @@ func buildInitBashScript(jenkins *v1alpha2.Jenkins) (*string, error) {
JenkinsScriptsVolumePath: JenkinsScriptsVolumePath,
}
output, err := render(initBashTemplate, data)
output, err := render.Render(initBashTemplate, data)
if err != nil {
return nil, err
}

196
pkg/controller/jenkins/configuration/user/seedjobs/seedjobs.go
View File

@ -6,7 +6,9 @@ import (
"encoding/base64"
"fmt"
"reflect"
"text/template"
"github.com/jenkinsci/kubernetes-operator/internal/render"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
@ -39,8 +41,94 @@ const (
// AgentName is the name of seed job agent
AgentName = "jnlp"
creatingGroovyScriptName = "seed-job-groovy-script.groovy"
)
var seedJobGroovyScriptTemplate = template.Must(template.New(creatingGroovyScriptName).Parse(`
import hudson.model.FreeStyleProject;
import hudson.plugins.git.GitSCM;
import hudson.plugins.git.BranchSpec;
import hudson.triggers.SCMTrigger;
import hudson.triggers.TimerTrigger;
import hudson.util.Secret;
import javaposse.jobdsl.plugin.*;
import jenkins.model.Jenkins;
import jenkins.model.JenkinsLocationConfiguration;
import com.cloudbees.plugins.credentials.CredentialsScope;
import com.cloudbees.plugins.credentials.domains.Domain;
import com.cloudbees.plugins.credentials.SystemCredentialsProvider;
import jenkins.model.JenkinsLocationConfiguration;
import org.jenkinsci.plugins.workflow.job.WorkflowJob;
import org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition;
{{ if .GitHubPushTrigger }}
import com.cloudbees.jenkins.GitHubPushTrigger;
{{ end }}
import hudson.model.FreeStyleProject;
import hudson.model.labels.LabelAtom;
import hudson.plugins.git.BranchSpec;
import hudson.plugins.git.GitSCM;
import hudson.plugins.git.SubmoduleConfig;
import hudson.plugins.git.extensions.impl.CloneOption;
import javaposse.jobdsl.plugin.ExecuteDslScripts;
import javaposse.jobdsl.plugin.LookupStrategy;
import javaposse.jobdsl.plugin.RemovedJobAction;
import javaposse.jobdsl.plugin.RemovedViewAction;
import static com.google.common.collect.Lists.newArrayList;
Jenkins jenkins = Jenkins.instance
def jobDslSeedName = "{{ .ID }} - {{ .SeedJobSuffix }}";
def jobRef = jenkins.getItem(jobDslSeedName)
def repoList = GitSCM.createRepoList("{{ .RepositoryURL }}", "{{ .CredentialID }}")
def gitExtensions = [new CloneOption(true, true, ";", 10)]
def scm = new GitSCM(
repoList,
newArrayList(new BranchSpec("{{ .RepositoryBranch }}")),
false,
Collections.<SubmoduleConfig>emptyList(),
null,
null,
gitExtensions
)
def executeDslScripts = new ExecuteDslScripts()
executeDslScripts.setTargets("{{ .Targets }}")
executeDslScripts.setSandbox(false)
executeDslScripts.setRemovedJobAction(RemovedJobAction.DELETE)
executeDslScripts.setRemovedViewAction(RemovedViewAction.DELETE)
executeDslScripts.setLookupStrategy(LookupStrategy.SEED_JOB)
executeDslScripts.setAdditionalClasspath("{{ .AdditionalClasspath }}")
executeDslScripts.setFailOnMissingPlugin({{ .FailOnMissingPlugin }})
executeDslScripts.setUnstableOnDeprecation({{ .UnstableOnDeprecation }})
executeDslScripts.setIgnoreMissingFiles({{ .IgnoreMissingFiles }})
if (jobRef == null) {
jobRef = jenkins.createProject(FreeStyleProject, jobDslSeedName)
}
jobRef.getBuildersList().clear()
jobRef.getBuildersList().add(executeDslScripts)
jobRef.setDisplayName("Seed Job from {{ .ID }}")
jobRef.setScm(scm)
{{ if .PollSCM }}
jobRef.addTrigger(new SCMTrigger("{{ .PollSCM }}"))
{{ end }}
{{ if .GitHubPushTrigger }}
jobRef.addTrigger(new GitHubPushTrigger())
{{ end }}
{{ if .BuildPeriodically }}
jobRef.addTrigger(new TimerTrigger("{{ .BuildPeriodically }}"))
{{ end}}
jobRef.setAssignedLabel(new LabelAtom("{{ .AgentName }}"))
jenkins.getQueue().schedule(jobRef)
`))
// SeedJobs defines API for configuring and ensuring Jenkins Seed Jobs and Deploy Keys
type SeedJobs struct {
jenkinsClient jenkinsclient.Jenkins
@ -112,7 +200,10 @@ func (s *SeedJobs) createJobs(jenkins *v1alpha2.Jenkins) (requeue bool, err erro
return true, err
}
groovyScript := seedJobCreatingGroovyScript(seedJob)
groovyScript, err := seedJobCreatingGroovyScript(seedJob)
if err != nil {
return true, err
}
hash := sha256.New()
hash.Write([]byte(groovyScript))
@ -316,72 +407,43 @@ func agentDeployment(jenkinsManifest *v1alpha2.Jenkins, namespace string, agentN
}
}
func seedJobCreatingGroovyScript(s v1alpha2.SeedJob) string {
return `
import hudson.model.FreeStyleProject;
import hudson.plugins.git.GitSCM;
import hudson.plugins.git.BranchSpec;
import hudson.triggers.SCMTrigger;
import hudson.util.Secret;
import javaposse.jobdsl.plugin.*;
import jenkins.model.Jenkins;
import jenkins.model.JenkinsLocationConfiguration;
import com.cloudbees.plugins.credentials.CredentialsScope;
import com.cloudbees.plugins.credentials.domains.Domain;
import com.cloudbees.plugins.credentials.SystemCredentialsProvider;
import jenkins.model.JenkinsLocationConfiguration;
import org.jenkinsci.plugins.workflow.job.WorkflowJob;
import org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition;
func seedJobCreatingGroovyScript(seedJob v1alpha2.SeedJob) (string, error) {
data := struct {
ID string
CredentialID string
Targets string
RepositoryBranch string
RepositoryURL string
GitHubPushTrigger bool
BuildPeriodically string
PollSCM string
IgnoreMissingFiles bool
AdditionalClasspath string
FailOnMissingPlugin bool
UnstableOnDeprecation bool
SeedJobSuffix string
AgentName string
}{
ID: seedJob.ID,
CredentialID: seedJob.CredentialID,
Targets: seedJob.Targets,
RepositoryBranch: seedJob.RepositoryBranch,
RepositoryURL: seedJob.RepositoryURL,
GitHubPushTrigger: seedJob.GitHubPushTrigger,
BuildPeriodically: seedJob.BuildPeriodically,
PollSCM: seedJob.PollSCM,
IgnoreMissingFiles: seedJob.IgnoreMissingFiles,
AdditionalClasspath: seedJob.AdditionalClasspath,
FailOnMissingPlugin: seedJob.FailOnMissingPlugin,
UnstableOnDeprecation: seedJob.UnstableOnDeprecation,
SeedJobSuffix: constants.SeedJobSuffix,
AgentName: AgentName,
}
import hudson.model.FreeStyleProject
import hudson.model.labels.LabelAtom
import hudson.plugins.git.BranchSpec
import hudson.plugins.git.GitSCM
import hudson.plugins.git.SubmoduleConfig
import hudson.plugins.git.extensions.impl.CloneOption
import javaposse.jobdsl.plugin.ExecuteDslScripts
import javaposse.jobdsl.plugin.LookupStrategy
import javaposse.jobdsl.plugin.RemovedJobAction
import javaposse.jobdsl.plugin.RemovedViewAction
output, err := render.Render(seedJobGroovyScriptTemplate, data)
if err != nil {
return "", err
}
import static com.google.common.collect.Lists.newArrayList
Jenkins jenkins = Jenkins.instance
def jobDslSeedName = "` + s.ID + `-` + constants.SeedJobSuffix + `";
def jobRef = jenkins.getItem(jobDslSeedName)
def repoList = GitSCM.createRepoList("` + s.RepositoryURL + `", "` + s.CredentialID + `")
def gitExtensions = [new CloneOption(true, true, ";", 10)]
def scm = new GitSCM(
repoList,
newArrayList(new BranchSpec("` + s.RepositoryBranch + `")),
false,
Collections.<SubmoduleConfig>emptyList(),
null,
null,
gitExtensions
)
def executeDslScripts = new ExecuteDslScripts()
executeDslScripts.setTargets("` + s.Targets + `")
executeDslScripts.setSandbox(false)
executeDslScripts.setRemovedJobAction(RemovedJobAction.DELETE)
executeDslScripts.setRemovedViewAction(RemovedViewAction.DELETE)
executeDslScripts.setLookupStrategy(LookupStrategy.SEED_JOB)
if (jobRef == null) {
jobRef = jenkins.createProject(FreeStyleProject, jobDslSeedName)
}
jobRef.getBuildersList().clear()
jobRef.getBuildersList().add(executeDslScripts)
jobRef.setDisplayName("` + fmt.Sprintf("Seed Job from %s", s.ID) + `")
jobRef.setScm(scm)
jobRef.setAssignedLabel(new LabelAtom("` + AgentName + `"))
jenkins.getQueue().schedule(jobRef)
`
return output, nil
}

5
pkg/controller/jenkins/configuration/user/seedjobs/seedjobs_test.go
View File

@ -87,10 +87,13 @@ func TestEnsureSeedJobs(t *testing.T) {
},
}
seedJobCreatingScript, err := seedJobCreatingGroovyScript(jenkins.Spec.SeedJobs[0])
assert.NoError(t, err)
jenkinsClient.EXPECT().GetNode(agentName).Return(nil, nil).AnyTimes()
jenkinsClient.EXPECT().CreateNode(agentName, 1, "The jenkins-operator generated agent", "/home/jenkins", agentName).Return(testNode, nil).AnyTimes()
jenkinsClient.EXPECT().GetNodeSecret(agentName).Return(agentSecret, nil).AnyTimes()
jenkinsClient.EXPECT().ExecuteScript(seedJobCreatingGroovyScript(jenkins.Spec.SeedJobs[0])).AnyTimes()
jenkinsClient.EXPECT().ExecuteScript(seedJobCreatingScript).AnyTimes()
seedJobClient := New(jenkinsClient, fakeClient, logger)

51
pkg/controller/jenkins/configuration/user/seedjobs/validate.go
View File

@ -12,6 +12,7 @@ import (
"github.com/go-logr/logr"
stackerr "github.com/pkg/errors"
"github.com/robfig/cron"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
@ -87,10 +88,60 @@ func (r *SeedJobs) ValidateSeedJobs(jenkins v1alpha2.Jenkins) (bool, error) {
}
}
}
if len(seedJob.BuildPeriodically) > 0 {
if !r.validateSchedule(seedJob, seedJob.BuildPeriodically, "buildPeriodically") {
valid = false
}
}
if len(seedJob.PollSCM) > 0 {
if !r.validateSchedule(seedJob, seedJob.PollSCM, "pollSCM") {
valid = false
}
}
if seedJob.GitHubPushTrigger {
if !r.validateGitHubPushTrigger(jenkins) {
valid = false
}
}
}
return valid, nil
}
func (r *SeedJobs) validateSchedule(job v1alpha2.SeedJob, str string, key string) bool {
_, err := cron.Parse(str)
if err != nil {
r.logger.V(log.VWarn).Info(fmt.Sprintf("`%s` schedule '%s' is invalid cron spec in `%s`", key, str, job.ID))
return false
}
return true
}
func (r *SeedJobs) validateGitHubPushTrigger(jenkins v1alpha2.Jenkins) bool {
exists := false
for _, plugin := range jenkins.Spec.Master.BasePlugins {
if plugin.Name == "github" {
exists = true
}
}
userExists := false
for _, plugin := range jenkins.Spec.Master.Plugins {
if plugin.Name == "github" {
userExists = true
}
}
if !exists && !userExists {
r.logger.V(log.VWarn).Info("githubPushTrigger is set. This function requires `github` plugin installed in .Spec.Master.Plugins because seed jobs Push Trigger function needs it")
return false
}
return true
}
func (r *SeedJobs) validateIfIDIsUnique(seedJobs []v1alpha2.SeedJob) bool {
ids := map[string]bool{}
for _, seedJob := range seedJobs {

98
pkg/controller/jenkins/configuration/user/seedjobs/validate_test.go
View File

@ -442,6 +442,104 @@ func TestValidateSeedJobs(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, false, result)
})
t.Run("Invalid with wrong cron spec", func(t *testing.T) {
jenkins := v1alpha2.Jenkins{
Spec: v1alpha2.JenkinsSpec{
SeedJobs: []v1alpha2.SeedJob{
{
ID: "example",
CredentialID: "jenkins-operator-e2e",
JenkinsCredentialType: v1alpha2.NoJenkinsCredentialCredentialType,
Targets: "cicd/jobs/*.jenkins",
RepositoryBranch: "master",
RepositoryURL: "https://github.com/jenkinsci/kubernetes-operator.git",
BuildPeriodically: "invalid-cron-spec",
},
},
},
}
seedJobs := New(nil, fake.NewFakeClient(), logf.ZapLogger(false))
result, err := seedJobs.ValidateSeedJobs(jenkins)
assert.NoError(t, err)
assert.False(t, result)
})
t.Run("Valid with good cron spec", func(t *testing.T) {
jenkins := v1alpha2.Jenkins{
Spec: v1alpha2.JenkinsSpec{
SeedJobs: []v1alpha2.SeedJob{
{
ID: "example",
CredentialID: "jenkins-operator-e2e",
JenkinsCredentialType: v1alpha2.NoJenkinsCredentialCredentialType,
Targets: "cicd/jobs/*.jenkins",
RepositoryBranch: "master",
RepositoryURL: "https://github.com/jenkinsci/kubernetes-operator.git",
BuildPeriodically: "1 2 3 4 5",
PollSCM: "1 2 3 4 5",
},
},
},
}
seedJobs := New(nil, fake.NewFakeClient(), logf.ZapLogger(false))
result, err := seedJobs.ValidateSeedJobs(jenkins)
assert.NoError(t, err)
assert.True(t, result)
})
t.Run("Invalid with set githubPushTrigger and not installed github plugin", func(t *testing.T) {
jenkins := v1alpha2.Jenkins{
Spec: v1alpha2.JenkinsSpec{
SeedJobs: []v1alpha2.SeedJob{
{
ID: "example",
CredentialID: "jenkins-operator-e2e",
JenkinsCredentialType: v1alpha2.NoJenkinsCredentialCredentialType,
Targets: "cicd/jobs/*.jenkins",
RepositoryBranch: "master",
RepositoryURL: "https://github.com/jenkinsci/kubernetes-operator.git",
GitHubPushTrigger: true,
},
},
},
}
seedJobs := New(nil, fake.NewFakeClient(), logf.ZapLogger(false))
result, err := seedJobs.ValidateSeedJobs(jenkins)
assert.NoError(t, err)
assert.False(t, result)
})
t.Run("Invalid with set githubPushTrigger and not installed github plugin", func(t *testing.T) {
jenkins := v1alpha2.Jenkins{
Spec: v1alpha2.JenkinsSpec{
SeedJobs: []v1alpha2.SeedJob{
{
ID: "example",
CredentialID: "jenkins-operator-e2e",
JenkinsCredentialType: v1alpha2.NoJenkinsCredentialCredentialType,
Targets: "cicd/jobs/*.jenkins",
RepositoryBranch: "master",
RepositoryURL: "https://github.com/jenkinsci/kubernetes-operator.git",
GitHubPushTrigger: true,
},
},
Master: v1alpha2.JenkinsMaster{
Plugins: []v1alpha2.Plugin{
{Name: "github", Version: "latest"},
},
},
},
}
seedJobs := New(nil, fake.NewFakeClient(), logf.ZapLogger(false))
result, err := seedJobs.ValidateSeedJobs(jenkins)
assert.NoError(t, err)
assert.True(t, result)
})
}
func TestValidateIfIDIsUnique(t *testing.T) {