Initial version of user reconciliation loop and seed jobs

Makefile

@@ -160,7 +160,7 @@ else
 	sed -i 's|REPLACE_ARGS||g' deploy/namespace-init.yaml
 endif
 
-	@RUNNING_TESTS=1 go test -parallel=2 "./test/e2e/" -tags "$(BUILDTAGS) cgo" -v \
+	@RUNNING_TESTS=1 go test -parallel=1 "./test/e2e/" -tags "$(BUILDTAGS) cgo" -v \
 		-root=$(CURRENT_DIRECTORY) -kubeconfig=$(HOME)/.kube/config -globalMan deploy/crds/virtuslab_v1alpha1_jenkins_crd.yaml -namespacedMan deploy/namespace-init.yaml
 
 .PHONY: vet

README.md

@@ -6,11 +6,79 @@ Kubernetes native Jenkins operator.
 
 Can be found [here][developer_guide].
 
+## Configuration
+
+This section describes Jenkins configuration.
+
+### Seed Jobs
+
+Jenkins operator uses [job-dsl][job-dsl] and [ssh-credentials][ssh-credentials] plugins for configuring seed jobs
+and deploy keys.
+
+
+It can be configured using `Jenkins.spec.seedJobs` section from custom resource manifest:
+
+```
+apiVersion: virtuslab.com/v1alpha1
+kind: Jenkins
+metadata:
+  name: example
+spec:
+  master:
+   image: jenkins/jenkins
+  seedJobs:
+  - id: jenkins-operator
+    targets: "cicd/jobs/*.jenkins"
+    description: "Jenkins Operator e2e tests repository"
+    repositoryBranch: master
+    repositoryUrl: git@github.com:VirtusLab/jenkins-operator-e2e.git
+    privateKey:
+      secretKeyRef:
+        name: deploy-keys
+        key: jenkins-operator-e2e
+```
+
+And corresponding Kubernetes Secret (in the same namespace) with private key:
+
+```
+apiVersion: v1
+kind: Secret
+metadata:
+  name: deploy-keys
+data:
+  jenkins-operator-e2e: |
+    -----BEGIN RSA PRIVATE KEY-----
+    MIIJKAIBAAKCAgEAxxDpleJjMCN5nusfW/AtBAZhx8UVVlhhhIKXvQ+dFODQIdzO
+    oDXybs1zVHWOj31zqbbJnsfsVZ9Uf3p9k6xpJ3WFY9b85WasqTDN1xmSd6swD4N8
+    ...
+```
+
+If your GitHub repository is public, you don't have to configure `privateKey` and create Kubernetes Secret:
+
+```
+apiVersion: virtuslab.com/v1alpha1
+kind: Jenkins
+metadata:
+  name: example
+spec:
+  master:
+   image: jenkins/jenkins
+  seedJobs:
+  - id: jenkins-operator-e2e
+    targets: "cicd/jobs/*.jenkins"
+    description: "Jenkins Operator e2e tests repository"
+    repositoryBranch: master
+    repositoryUrl: https://github.com/VirtusLab/jenkins-operator-e2e.git
+```
+
+Jenkins operator will automatically configure and trigger Seed Job Pipeline for all entries from `Jenkins.spec.seedJobs`.
+
 ## TODO
 
 Common:
 - simple library for sending Kubernetes events
 - implement Jenkins.Status in custom resource
+- implement ensure for Jenkins jobs - state in Jenkins.Status 
 
 Base configuration:
 - install configuration as a code Jenkins plugin
@@ -21,9 +89,11 @@ Base configuration:
 User configuration:
 - user reconciliation loop (work in progress)
 - configure seed jobs and deploy keys (work in progress)
-- e2e tests for seed jobs
+- e2e tests for seed jobs (work in progress)
 - backup and restore for Jenkins jobs running as standalone job
 - trigger backup job before pod deletion using preStop k8s hooks
 - verify Jenkins configuration events
 
 [developer_guide]:doc/developer-guide.md
+[job-dsl]:https://github.com/jenkinsci/job-dsl-plugin
+[ssh-credentials]:https://github.com/jenkinsci/ssh-credentials-plugin

deploy/crds/virtuslab_v1alpha1_jenkins_cr.yaml

@@ -5,3 +5,15 @@ metadata:
 spec:
   master:
    image: jenkins/jenkins
+  seedJobs:
+  - id: jenkins-operator-e2e
+    targets: "cicd/jobs/*.jenkins"
+    description: "Jenkins Operator e2e tests repository"
+    repositoryBranch: master
+    repositoryUrl: https://github.com/VirtusLab/jenkins-operator-e2e.git
+#    Use configuration below if your GitHub repository is private
+#    repositoryUrl: git@github.com:VirtusLab/jenkins-operator-e2e.git
+#    privateKey:
+#      secretKeyRef:
+#        name: deploy-keys
+#        key: jenkins-operator-e2e

deploy/operator.yaml

@@ -17,7 +17,7 @@ spec:
       containers:
         - name: jenkins-operator
           # Replace this with the built image name
-          image: REPLACE_IMAGE
+          image: jenkins-operator
           ports:
           - containerPort: 60000
             name: metrics

deploy/seed_jobs_secret.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: deploy-keys
+data:
+  jenkins-operator-e2e: |
+    REDACTED

pkg/apis/virtuslab/v1alpha1/jenkins_types.go

@@ -13,6 +13,7 @@ type JenkinsSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file
 	Master   JenkinsMaster `json:"master,omitempty"`
+	SeedJobs []SeedJob     `json:"seedJobs,omitempty"`
 }
 
 // JenkinsMaster defines the Jenkins master pod attributes
@@ -27,6 +28,7 @@ type JenkinsStatus struct {
 	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
 	// Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file
 	BaseConfigurationCompletedTime *metav1.Time `json:"baseConfigurationCompletedTime,omitempty"`
+	UserConfigurationCompletedTime *metav1.Time `json:"userConfigurationCompletedTime,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -50,6 +52,21 @@ type JenkinsList struct {
 	Items           []Jenkins `json:"items"`
 }
 
+// SeedJob defined configuration for seed jobs and deploy keys
+type SeedJob struct {
+	ID               string     `json:"id"`
+	Description      string     `json:"description,omitempty"`
+	Targets          string     `json:"targets,omitempty"`
+	RepositoryBranch string     `json:"repositoryBranch,omitempty"`
+	RepositoryURL    string     `json:"repositoryUrl"`
+	PrivateKey       PrivateKey `json:"privateKey,omitempty"`
+}
+
+// PrivateKey contains a private key
+type PrivateKey struct {
+	SecretKeyRef *corev1.SecretKeySelector `json:"secretKeyRef"`
+}
+
 func init() {
 	SchemeBuilder.Register(&Jenkins{}, &JenkinsList{})
 }

pkg/apis/virtuslab/v1alpha1/zz_generated.deepcopy.go

@@ -21,6 +21,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	v1 "k8s.io/api/core/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -113,6 +114,13 @@ func (in *JenkinsMaster) DeepCopy() *JenkinsMaster {
 func (in *JenkinsSpec) DeepCopyInto(out *JenkinsSpec) {
 	*out = *in
 	in.Master.DeepCopyInto(&out.Master)
+	if in.SeedJobs != nil {
+		in, out := &in.SeedJobs, &out.SeedJobs
+		*out = make([]SeedJob, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }
 
@@ -133,6 +141,10 @@ func (in *JenkinsStatus) DeepCopyInto(out *JenkinsStatus) {
 		in, out := &in.BaseConfigurationCompletedTime, &out.BaseConfigurationCompletedTime
 		*out = (*in).DeepCopy()
 	}
+	if in.UserConfigurationCompletedTime != nil {
+		in, out := &in.UserConfigurationCompletedTime, &out.UserConfigurationCompletedTime
+		*out = (*in).DeepCopy()
+	}
 	return
 }
 
@@ -145,3 +157,41 @@ func (in *JenkinsStatus) DeepCopy() *JenkinsStatus {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PrivateKey) DeepCopyInto(out *PrivateKey) {
+	*out = *in
+	if in.SecretKeyRef != nil {
+		in, out := &in.SecretKeyRef, &out.SecretKeyRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrivateKey.
+func (in *PrivateKey) DeepCopy() *PrivateKey {
+	if in == nil {
+		return nil
+	}
+	out := new(PrivateKey)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SeedJob) DeepCopyInto(out *SeedJob) {
+	*out = *in
+	in.PrivateKey.DeepCopyInto(&out.PrivateKey)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SeedJob.
+func (in *SeedJob) DeepCopy() *SeedJob {
+	if in == nil {
+		return nil
+	}
+	out := new(SeedJob)
+	in.DeepCopyInto(out)
+	return out
+}

pkg/controller/jenkins/client/jenkins.go

@@ -93,7 +93,7 @@ func New(url, user, passwordOrToken string) (Jenkins, error) {
 		return nil, err
 	}
 	if status != http.StatusOK {
-		return nil, fmt.Errorf("Invalid status code returned: %d", status)
+		return nil, fmt.Errorf("invalid status code returned: %d", status)
 	}
 
 	return jenkinsClient, nil

pkg/controller/jenkins/configuration/base/reconcile.go

@@ -45,59 +45,59 @@ func New(client client.Client, scheme *runtime.Scheme, logger logr.Logger,
 }
 
 // Reconcile takes care of base configuration
-func (r *ReconcileJenkinsBaseConfiguration) Reconcile() (*reconcile.Result, error) {
+func (r *ReconcileJenkinsBaseConfiguration) Reconcile() (*reconcile.Result, jenkinsclient.Jenkins, error) {
 	if !r.validate(r.jenkins) {
 		r.logger.V(log.VWarn).Info("Please correct Jenkins CR")
-		return &reconcile.Result{}, nil
+		return &reconcile.Result{}, nil, nil
 	}
 
 	metaObject := resources.NewResourceObjectMeta(r.jenkins)
 
 	if err := r.createOperatorCredentialsSecret(metaObject); err != nil {
-		return &reconcile.Result{}, err
+		return &reconcile.Result{}, nil, err
 	}
 	r.logger.V(log.VDebug).Info("Operator credentials secret is present")
 
 	if err := r.createScriptsConfigMap(metaObject); err != nil {
-		return &reconcile.Result{}, err
+		return &reconcile.Result{}, nil, err
 	}
 	r.logger.V(log.VDebug).Info("Scripts config map is present")
 
 	if err := r.createBaseConfigurationConfigMap(metaObject); err != nil {
-		return &reconcile.Result{}, err
+		return &reconcile.Result{}, nil, err
 	}
 	r.logger.V(log.VDebug).Info("Base configuration config map is present")
 
 	if err := r.createService(metaObject); err != nil {
-		return &reconcile.Result{}, err
+		return &reconcile.Result{}, nil, err
 	}
 	r.logger.V(log.VDebug).Info("Service is present")
 
 	result, err := r.createJenkinsMasterPod(metaObject)
 	if err != nil {
-		return &reconcile.Result{}, err
+		return &reconcile.Result{}, nil, err
 	}
 	if result != nil {
-		return result, nil
+		return result, nil, nil
 	}
 	r.logger.V(log.VDebug).Info("Jenkins master pod is present")
 
 	result, err = r.waitForJenkins(metaObject)
 	if err != nil {
-		return &reconcile.Result{}, err
+		return &reconcile.Result{}, nil, err
 	}
 	if result != nil {
-		return result, nil
+		return result, nil, nil
 	}
 	r.logger.V(log.VDebug).Info("Jenkins master pod is ready")
 
-	_, err = r.getJenkinsClient(metaObject)
+	jenkinsClient, err := r.getJenkinsClient(metaObject)
 	if err != nil {
-		return &reconcile.Result{}, err
+		return &reconcile.Result{}, nil, err
 	}
 	r.logger.V(log.VDebug).Info("Jenkins API client set")
 
-	return nil, nil
+	return nil, jenkinsClient, nil
 }
 
 func (r *ReconcileJenkinsBaseConfiguration) createOperatorCredentialsSecret(meta metav1.ObjectMeta) error {
@@ -240,6 +240,7 @@ func (r *ReconcileJenkinsBaseConfiguration) waitForJenkins(meta metav1.ObjectMet
 	return nil, nil
 }
 
+// FIXME(bantoniak) move jenkins client out of base.reconcile because it's needed for user.reconcile as well
 func (r *ReconcileJenkinsBaseConfiguration) getJenkinsClient(meta metav1.ObjectMeta) (jenkinsclient.Jenkins, error) {
 	jenkinsURL, err := jenkinsclient.BuildJenkinsAPIUrl(
 		r.jenkins.ObjectMeta.Namespace, meta.Name, resources.HTTPPortInt, r.local, r.minikube)

pkg/controller/jenkins/configuration/base/resources/base_configuration_configmap.go

@@ -5,6 +5,7 @@ import (
 	"text/template"
 
 	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
+	"github.com/VirtusLab/jenkins-operator/pkg/controller/render"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -40,7 +41,7 @@ func buildCreateJenkinsOperatorUserGroovyScript() (*string, error) {
 		OperatorPasswordFile:    OperatorCredentialsSecretPasswordKey,
 	}
 
-	output, err := renderTemplate(createOperatorUserGroovyFmtTemplate, data)
+	output, err := render.Render(createOperatorUserGroovyFmtTemplate, data)
 	if err != nil {
 		return nil, err
 	}

pkg/controller/jenkins/configuration/base/resources/scripts_configmap.go

@@ -6,6 +6,7 @@ import (
 
 	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
 
+	"github.com/VirtusLab/jenkins-operator/pkg/controller/render"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -18,6 +19,19 @@ set -x
 mkdir -p {{ .JenkinsHomePath }}/init.groovy.d
 cp -n {{ .BaseConfigurationPath }}/*.groovy {{ .JenkinsHomePath }}/init.groovy.d
 
+touch {{ .JenkinsHomePath }}/plugins.txt
+cat > {{ .JenkinsHomePath }}/plugins.txt <<EOL
+credentials:2.1.18
+ssh-credentials:1.14
+job-dsl:1.70
+git:3.9.1
+workflow-cps:2.61
+workflow-job:2.30
+workflow-aggregator:2.6
+EOL
+
+/usr/local/bin/install-plugins.sh < {{ .JenkinsHomePath }}/plugins.txt
+
 /sbin/tini -s -- /usr/local/bin/jenkins.sh
 `))
 
@@ -37,7 +51,7 @@ func buildInitBashScript() (*string, error) {
 		BaseConfigurationPath: jenkinsBaseConfigurationVolumePath,
 	}
 
-	output, err := renderTemplate(initBashTemplate, data)
+	output, err := render.Render(initBashTemplate, data)
 	if err != nil {
 		return nil, err
 	}

pkg/controller/jenkins/configuration/user/reconcile.go

@@ -0,0 +1,45 @@
+package user
+
+import (
+	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
+	jenkins "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/client"
+	"github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/user/seedjobs"
+	"github.com/VirtusLab/jenkins-operator/pkg/log"
+	"github.com/go-logr/logr"
+	k8s "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+// ReconcileUserConfiguration defines values required for Jenkins user configuration
+type ReconcileUserConfiguration struct {
+	k8sClient     k8s.Client
+	jenkinsClient jenkins.Jenkins
+	logger        logr.Logger
+	jenkins       *virtuslabv1alpha1.Jenkins
+}
+
+// New create structure which takes care of user configuration
+func New(k8sClient k8s.Client, jenkinsClient jenkins.Jenkins, logger logr.Logger,
+	jenkins *virtuslabv1alpha1.Jenkins) *ReconcileUserConfiguration {
+	return &ReconcileUserConfiguration{
+		k8sClient:     k8sClient,
+		jenkinsClient: jenkinsClient,
+		logger:        logger,
+		jenkins:       jenkins,
+	}
+}
+
+// Reconcile it's a main reconciliation loop for user supplied configuration
+func (r *ReconcileUserConfiguration) Reconcile() (*reconcile.Result, error) {
+	if !r.validate(r.jenkins) {
+		r.logger.V(log.VWarn).Info("Please correct Jenkins CR")
+		return &reconcile.Result{}, nil
+	}
+
+	err := seedjobs.ConfigureSeedJobs(r.jenkinsClient, r.k8sClient, r.jenkins)
+	if err != nil {
+		return &reconcile.Result{}, err
+	}
+
+	return nil, nil
+}

pkg/controller/jenkins/configuration/user/seedjobs/doc.go

@@ -0,0 +1,2 @@
+// Package seedjobs implements seed jobs configuration
+package seedjobs

pkg/controller/jenkins/configuration/user/seedjobs/seedjobs.go

@@ -0,0 +1,221 @@
+package seedjobs
+
+import (
+	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
+	jenkins "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/client"
+	k8s "sigs.k8s.io/controller-runtime/pkg/client"
+
+	"context"
+	"fmt"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"strings"
+)
+
+const (
+	// ConfigureSeedJobsName this is the job name
+	ConfigureSeedJobsName = "Configure Seed Jobs"
+
+	deployKeyIDParameterName      = "DEPLOY_KEY_ID"
+	privateKeyParameterName       = "PRIVATE_KEY"
+	repositoryURLParameterName    = "REPOSITORY_URL"
+	repositoryBranchParameterName = "REPOSITORY_BRANCH"
+	targetsParameterName          = "TARGETS"
+	displayNameParameterName      = "SEED_JOB_DISPLAY_NAME"
+)
+
+// ConfigureSeedJobs configures and triggers seed job pipeline for every Jenkins.Spec.SeedJobs entry
+func ConfigureSeedJobs(jenkinsClient jenkins.Jenkins, k8sClient k8s.Client, jenkins *virtuslabv1alpha1.Jenkins) error {
+	err := configureSeedJobsPipeline(jenkinsClient)
+	if err != nil {
+		return err
+	}
+
+	seedJobs := jenkins.Spec.SeedJobs
+	for _, seedJob := range seedJobs {
+		privateKey, err := extractPrivateKey(k8sClient, jenkins.Namespace, seedJob)
+		if err != nil {
+			return err
+		}
+		err = triggerConfigureSeedJobsPipeline(
+			jenkinsClient,
+			seedJob.ID,
+			privateKey,
+			seedJob.RepositoryURL,
+			seedJob.RepositoryBranch, seedJob.Targets, fmt.Sprintf("Seed Job from %s", seedJob.ID))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// configureSeedJobsPipeline configures seed jobs and deploy keys
+func configureSeedJobsPipeline(jenkinsClient jenkins.Jenkins) error {
+	// FIXME(bantoniak) implement CreateOrUpdateJob()
+	_, err := jenkinsClient.CreateJob(seedJobConfigXML, ConfigureSeedJobsName)
+	if err != nil && strings.Contains(err.Error(), "A job already exists") {
+		// skip, job already exists
+		return nil
+	}
+	return err
+}
+
+// triggerConfigureSeedJobsPipeline triggers and configures seed job for specific GitHub repository
+func triggerConfigureSeedJobsPipeline(jenkinsClient jenkins.Jenkins, deployKeyID, privateKey, repositoryURL, repositoryBranch, targets, displayName string) error {
+	options := map[string]string{
+		deployKeyIDParameterName:      deployKeyID,
+		privateKeyParameterName:       privateKey,
+		repositoryURLParameterName:    repositoryURL,
+		repositoryBranchParameterName: repositoryBranch,
+		targetsParameterName:          targets,
+		displayNameParameterName:      displayName,
+	}
+	// FIXME(bantoniak) implement EnsureJob()
+	_, err := jenkinsClient.BuildJob(ConfigureSeedJobsName, options)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func extractPrivateKey(k8sClient k8s.Client, namespace string, seedJob virtuslabv1alpha1.SeedJob) (string, error) {
+	if seedJob.PrivateKey.SecretKeyRef != nil {
+		deployKeySecret := &v1.Secret{}
+		namespaceName := types.NamespacedName{Namespace: namespace, Name: seedJob.PrivateKey.SecretKeyRef.Name}
+		err := k8sClient.Get(context.TODO(), namespaceName, deployKeySecret)
+		if err != nil {
+			return "", err
+		}
+		return string(deployKeySecret.Data[seedJob.PrivateKey.SecretKeyRef.Key]), nil
+	}
+	return "", nil
+}
+
+// FIXME use mask-password plugin for params.PRIVATE_KEY
+var seedJobConfigXML = `
+<flow-definition plugin="workflow-job@2.30">
+  <actions/>
+  <description></description>
+  <keepDependencies>false</keepDependencies>
+  <properties>
+    <hudson.model.ParametersDefinitionProperty>
+      <parameterDefinitions>
+        <hudson.model.StringParameterDefinition>
+          <name>DEPLOY_KEY_ID</name>
+          <description></description>
+          <defaultValue></defaultValue>
+          <trim>false</trim>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>PRIVATE_KEY</name>
+          <description></description>
+          <defaultValue></defaultValue>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>REPOSITORY_URL</name>
+          <description></description>
+          <defaultValue></defaultValue>
+          <trim>false</trim>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>REPOSITORY_BRANCH</name>
+          <description></description>
+          <defaultValue>master</defaultValue>
+          <trim>false</trim>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>SEED_JOB_DISPLAY_NAME</name>
+          <description></description>
+          <defaultValue></defaultValue>
+          <trim>false</trim>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>TARGETS</name>
+          <description></description>
+          <defaultValue>cicd/jobs/*.jenkins</defaultValue>
+          <trim>false</trim>
+        </hudson.model.StringParameterDefinition>
+      </parameterDefinitions>
+    </hudson.model.ParametersDefinitionProperty>
+  </properties>
+  <definition class="org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition" plugin="workflow-cps@2.61">
+    <script>import com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey
+import com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey.DirectEntryPrivateKeySource
+import com.cloudbees.plugins.credentials.CredentialsScope
+import com.cloudbees.plugins.credentials.SystemCredentialsProvider
+import com.cloudbees.plugins.credentials.domains.Domain
+import hudson.model.FreeStyleProject
+import hudson.model.labels.LabelAtom
+import hudson.plugins.git.BranchSpec
+import hudson.plugins.git.GitSCM
+import hudson.plugins.git.SubmoduleConfig
+import hudson.plugins.git.extensions.impl.CloneOption
+import javaposse.jobdsl.plugin.ExecuteDslScripts
+import javaposse.jobdsl.plugin.LookupStrategy
+import javaposse.jobdsl.plugin.RemovedJobAction
+import javaposse.jobdsl.plugin.RemovedViewAction
+import jenkins.model.Jenkins
+import javaposse.jobdsl.plugin.GlobalJobDslSecurityConfiguration
+import jenkins.model.GlobalConfiguration
+
+import static com.google.common.collect.Lists.newArrayList
+
+// https://javadoc.jenkins.io/plugin/ssh-credentials/com/cloudbees/jenkins/plugins/sshcredentials/impl/BasicSSHUserPrivateKey.html
+BasicSSHUserPrivateKey deployKeyPrivate = new BasicSSHUserPrivateKey(
+        CredentialsScope.GLOBAL,
+        &quot;${params.DEPLOY_KEY_ID}&quot;,
+        &quot;git&quot;,
+        new DirectEntryPrivateKeySource(&quot;${params.PRIVATE_KEY}&quot;),
+        &quot;&quot;,
+        &quot;${params.DEPLOY_KEY_ID}&quot;
+)
+
+// https://javadoc.jenkins.io/plugin/credentials/index.html?com/cloudbees/plugins/credentials/SystemCredentialsProvider.html
+SystemCredentialsProvider.getInstance().getStore().addCredentials(Domain.global(), deployKeyPrivate)
+
+Jenkins jenkins = Jenkins.instance
+
+
+def jobDslSeedName = &quot;${params.DEPLOY_KEY_ID}-job-dsl-seed&quot;
+def jobDslDeployKeyName = &quot;${params.DEPLOY_KEY_ID}&quot;
+def jobRef = jenkins.getItem(jobDslSeedName)
+
+def repoList = GitSCM.createRepoList(&quot;${params.REPOSITORY_URL}&quot;, jobDslDeployKeyName)
+def gitExtensions = [new CloneOption(true, true, &quot;&quot;, 10)]
+def scm = new GitSCM(
+        repoList,
+        newArrayList(new BranchSpec(&quot;${params.REPOSITORY_BRANCH}&quot;)),
+        false,
+        Collections.&lt;SubmoduleConfig&gt; emptyList(),
+        null,
+        null,
+        gitExtensions
+)
+
+def executeDslScripts = new ExecuteDslScripts()
+executeDslScripts.setTargets(&quot;${params.TARGETS}&quot;)
+executeDslScripts.setSandbox(false)
+executeDslScripts.setRemovedJobAction(RemovedJobAction.DELETE)
+executeDslScripts.setRemovedViewAction(RemovedViewAction.DELETE)
+executeDslScripts.setLookupStrategy(LookupStrategy.SEED_JOB)
+executeDslScripts.setAdditionalClasspath(&quot;src&quot;)
+
+if (jobRef == null) {
+        jobRef = jenkins.createProject(FreeStyleProject, jobDslSeedName)
+}
+jobRef.getBuildersList().clear()
+jobRef.getBuildersList().add(executeDslScripts)
+jobRef.setDisplayName(&quot;${params.SEED_JOB_DISPLAY_NAME}&quot;)
+jobRef.setScm(scm)
+jobRef.setAssignedLabel(new LabelAtom(&quot;master&quot;))
+
+// disable Job DSL script approval
+GlobalConfiguration.all().get(GlobalJobDslSecurityConfiguration.class).useScriptSecurity=false
+GlobalConfiguration.all().get(GlobalJobDslSecurityConfiguration.class).save()</script>
+    <sandbox>false</sandbox>
+  </definition>
+  <triggers/>
+  <disabled>false</disabled>
+</flow-definition>
+`

pkg/controller/jenkins/configuration/user/validate.go

@@ -0,0 +1,26 @@
+package user
+
+import (
+	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
+	"strings"
+)
+
+func (r *ReconcileUserConfiguration) validate(jenkins *virtuslabv1alpha1.Jenkins) bool {
+	// validate jenkins.Spec.SeedJobs
+	if jenkins.Spec.SeedJobs != nil {
+		for _, seedJob := range jenkins.Spec.SeedJobs {
+			if len(seedJob.ID) == 0 {
+				r.logger.V(0).Info("seed job id can't be empty")
+				return false
+			}
+
+			if strings.Contains(seedJob.RepositoryURL, "git@") {
+				if seedJob.PrivateKey.SecretKeyRef == nil {
+					r.logger.V(0).Info("private key can't be empty while using ssh repository url")
+					return false
+				}
+			}
+		}
+	}
+	return true
+}

pkg/controller/jenkins/jenkins_controller.go

@@ -5,6 +5,7 @@ import (
 
 	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
 	"github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/base"
+	"github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/user"
 	"github.com/VirtusLab/jenkins-operator/pkg/log"
 
 	"github.com/go-logr/logr"
@@ -74,8 +75,8 @@ type ReconcileJenkins struct {
 	local, minikube bool
 }
 
-// Reconcile reads that state of the cluster for a Jenkins object and makes changes based on the state read
+// Reconcile it's a main reconciliation loop which maintain desired state for on Jenkins.Spec
-// and what is in the Jenkins.Spec
+// including base and user supplied configuration
 func (r *ReconcileJenkins) Reconcile(request reconcile.Request) (reconcile.Result, error) {
 	logger := r.buildLogger(request.Name)
 	logger.Info("Reconciling Jenkins")
@@ -94,8 +95,9 @@ func (r *ReconcileJenkins) Reconcile(request reconcile.Request) (reconcile.Resul
 		return reconcile.Result{}, err
 	}
 
+	// Reconcile base configuration
 	baseConfiguration := base.New(r.client, r.scheme, logger, jenkins, r.local, r.minikube)
-	result, err := baseConfiguration.Reconcile()
+	result, jenkinsClient, err := baseConfiguration.Reconcile()
 	if err != nil {
 		return reconcile.Result{}, err
 	}
@@ -111,6 +113,24 @@ func (r *ReconcileJenkins) Reconcile(request reconcile.Request) (reconcile.Resul
 		}
 	}
 
+	// Reconcile user configuration
+	userConfiguration := user.New(r.client, jenkinsClient, logger, jenkins)
+	result, err = userConfiguration.Reconcile()
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+	if result != nil {
+		return *result, nil
+	}
+	if err == nil && result == nil && jenkins.Status.UserConfigurationCompletedTime == nil {
+		now := metav1.Now()
+		jenkins.Status.UserConfigurationCompletedTime = &now
+		err = r.client.Update(context.TODO(), jenkins)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+	}
+
 	return reconcile.Result{}, nil
 }
 

pkg/controller/render/render.go

@@ -1,11 +1,12 @@
-package resources
+package render
 
 import (
 	"bytes"
 	"text/template"
 )
 
-func renderTemplate(template *template.Template, data interface{}) (string, error) {
+// Render executes a parsed template (go-template) with configuration from data
+func Render(template *template.Template, data interface{}) (string, error) {
 	var buffer bytes.Buffer
 	if err := template.Execute(&buffer, data); err != nil {
 		return "", err

test/e2e/base_configuration_test.go

@@ -5,8 +5,6 @@ import (
 	"testing"
 
 	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
-
-	"github.com/bndr/gojenkins"
 )
 
 func TestBaseConfiguration(t *testing.T) {
@@ -22,16 +20,6 @@ func TestBaseConfiguration(t *testing.T) {
 	verifyJenkinsAPIConnection(t, jenkins)
 }
 
-func verifyJenkinsAPIConnection(t *testing.T, jenkins *virtuslabv1alpha1.Jenkins) *gojenkins.Jenkins {
-	client, err := createJenkinsAPIClient(jenkins)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Log("I can establish connection to Jenkins API")
-	return client
-}
-
 func verifyJenkinsMasterPodAttributes(t *testing.T, jenkins *virtuslabv1alpha1.Jenkins) {
 	jenkinsPod := getJenkinsMasterPod(t, jenkins)
 

test/e2e/jenkins.go

@@ -97,3 +97,54 @@ func createJenkinsCR(t *testing.T, namespace string) *virtuslabv1alpha1.Jenkins
 
 	return jenkins
 }
+
+func createJenkinsCRWithSeedJob(t *testing.T, namespace string) *virtuslabv1alpha1.Jenkins {
+	jenkins := &virtuslabv1alpha1.Jenkins{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "e2e",
+			Namespace: namespace,
+		},
+		Spec: virtuslabv1alpha1.JenkinsSpec{
+			Master: virtuslabv1alpha1.JenkinsMaster{
+				Image:       "jenkins/jenkins",
+				Annotations: map[string]string{"test": "label"},
+				Resources: corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("1"),
+						corev1.ResourceMemory: resource.MustParse("1Gi"),
+					},
+					Limits: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("2"),
+						corev1.ResourceMemory: resource.MustParse("2Gi"),
+					},
+				},
+			},
+			SeedJobs: []virtuslabv1alpha1.SeedJob{
+				{
+					ID:               "jenkins-operator-e2e",
+					Targets:          "cicd/jobs/*.jenkins",
+					Description:      "Jenkins Operator e2e tests repository",
+					RepositoryBranch: "master",
+					RepositoryURL:    "https://github.com/VirtusLab/jenkins-operator-e2e.git",
+				},
+			},
+		},
+	}
+
+	t.Logf("Jenkins CR %+v", *jenkins)
+	if err := framework.Global.Client.Create(context.TODO(), jenkins, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	return jenkins
+}
+
+func verifyJenkinsAPIConnection(t *testing.T, jenkins *virtuslabv1alpha1.Jenkins) *gojenkins.Jenkins {
+	client, err := createJenkinsAPIClient(jenkins)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log("I can establish connection to Jenkins API")
+	return client
+}

test/e2e/restart_pod_test.go

@@ -2,13 +2,12 @@ package e2e
 
 import (
 	"context"
-	"testing"
-	"time"
-
 	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
+	"testing"
 
 	framework "github.com/operator-framework/operator-sdk/pkg/test"
 	"k8s.io/apimachinery/pkg/types"
+	"time"
 )
 
 func TestJenkinsMasterPodRestart(t *testing.T) {
@@ -27,8 +26,8 @@ func TestJenkinsMasterPodRestart(t *testing.T) {
 
 func checkBaseConfigurationCompleteTimeIsNotSet(t *testing.T, jenkins *virtuslabv1alpha1.Jenkins) {
 	jenkinsStatus := &virtuslabv1alpha1.Jenkins{}
-	namespacedName := types.NamespacedName{Namespace: jenkins.Namespace, Name: jenkins.Name}
+	namespaceName := types.NamespacedName{Namespace: jenkins.Namespace, Name: jenkins.Name}
-	err := framework.Global.Client.Get(context.TODO(), namespacedName, jenkinsStatus)
+	err := framework.Global.Client.Get(context.TODO(), namespaceName, jenkinsStatus)
 	if err != nil {
 		t.Fatal(err)
 	}

test/e2e/user_configuration_test.go

@@ -0,0 +1,40 @@
+package e2e
+
+import (
+	"github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/user/seedjobs"
+	"github.com/bndr/gojenkins"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"testing"
+	"time"
+)
+
+func TestUserConfiguration(t *testing.T) {
+	t.Parallel()
+	namespace, ctx := setupTest(t)
+	// Deletes test namespace
+	defer ctx.Cleanup()
+
+	jenkins := createJenkinsCRWithSeedJob(t, namespace)
+	waitForJenkinsUserConfigurationToComplete(t, jenkins)
+	client := verifyJenkinsAPIConnection(t, jenkins)
+	verifyJenkinsSeedJobs(t, client)
+}
+
+func verifyJenkinsSeedJobs(t *testing.T, client *gojenkins.Jenkins) {
+	// check if job has been configured and executed successfully
+	err := wait.Poll(time.Second*10, time.Minute*2, func() (bool, error) {
+		t.Logf("Attempting to get seed job status '%v'", seedjobs.ConfigureSeedJobsName)
+		seedJob, err := client.GetJob(seedjobs.ConfigureSeedJobsName)
+		if err != nil || seedJob == nil {
+			return false, nil
+		}
+		build, err := seedJob.GetLastSuccessfulBuild()
+		if err != nil || build == nil {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatalf("couldn't get seed job '%v'", err)
+	}
+}

test/e2e/wait.go

@@ -15,7 +15,7 @@ import (
 
 var (
 	retryInterval = time.Second * 5
-	timeout       = time.Second * 30
+	timeout       = time.Second * 60
 )
 
 // checkConditionFunc is used to check if a condition for the jenkins CR is true
@@ -33,6 +33,18 @@ func waitForJenkinsBaseConfigurationToComplete(t *testing.T, jenkins *virtuslabv
 	t.Log("Jenkins pod is running")
 }
 
+func waitForJenkinsUserConfigurationToComplete(t *testing.T, jenkins *virtuslabv1alpha1.Jenkins) {
+	t.Log("Waiting for Jenkins user configuration to complete")
+	_, err := WaitUntilJenkinsConditionTrue(retryInterval, 30, jenkins, func(jenkins *virtuslabv1alpha1.Jenkins) bool {
+		t.Logf("Current Jenkins status '%+v'", jenkins.Status)
+		return jenkins.Status.UserConfigurationCompletedTime != nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("Jenkins pod is running")
+}
+
 // WaitUntilJenkinsConditionTrue retries until the specified condition check becomes true for the jenkins CR
 func WaitUntilJenkinsConditionTrue(retryInterval time.Duration, retries int, jenkins *virtuslabv1alpha1.Jenkins, checkCondition checkConditionFunc) (*virtuslabv1alpha1.Jenkins, error) {
 	jenkinsStatus := &virtuslabv1alpha1.Jenkins{}