#9 Add secret support in Configuration as Code plugin integration

docs/getting-started.md

@@ -213,7 +213,30 @@ You can verify if your pipelines were successfully configured in Jenkins Seed Jo
 ## Jenkins Customisation
 
 Jenkins can be customized using groovy scripts or configuration as code plugin. All custom configuration is stored in
-the **jenkins-operator-user-configuration-example** ConfigMap which is automatically created by **jenkins-operator**. 
+the **jenkins-operator-user-configuration-example** ConfigMap which is automatically created by **jenkins-operator**.
+
+**jenkins-operator** creates **jenkins-operator-user-configuration-example** secret where user can store sensitive 
+information used for custom configuration. If you have entry in secret named `PASSWORD` then you can use it in 
+Configuration as Plugin as `adminAddress: "${PASSWORD}"`.
+
+```
+kubectl get secret jenkins-operator-user-configuration-example -o yaml
+
+apiVersion: v1
+data:
+  SECRET_JENKINS_ADMIN_ADDRESS: YXNkZgo=
+kind: Secret
+metadata:
+  creationTimestamp: 2019-03-03T11:54:36Z
+  labels:
+    app: jenkins-operator
+    jenkins-cr: example
+    watch: "true"
+  name: jenkins-operator-user-configuration-example
+  namespace: default
+type: Opaque
+
+```
 
 ```
 kubectl get configmap jenkins-operator-user-configuration-example -o yaml
@@ -243,6 +266,7 @@ data:
   1-system-message.yaml: |2
     jenkins:
       systemMessage: "Configuration as Code integration works!!!"
+      adminAddress: "${SECRET_JENKINS_ADMIN_ADDRESS}"
 kind: ConfigMap
 metadata:
   labels:

pkg/controller/jenkins/configuration/base/reconcile.go

@@ -126,6 +126,11 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureResourcesRequiredForJenkinsPod
 	}
 	r.logger.V(log.VDebug).Info("User configuration config map is present")
 
+	if err := r.createUserConfigurationSecret(metaObject); err != nil {
+		return err
+	}
+	r.logger.V(log.VDebug).Info("User configuration secret is present")
+
 	if err := r.createRBAC(metaObject); err != nil {
 		return err
 	}
@@ -252,6 +257,23 @@ func (r *ReconcileJenkinsBaseConfiguration) createUserConfigurationConfigMap(met
 	return nil
 }
 
+func (r *ReconcileJenkinsBaseConfiguration) createUserConfigurationSecret(meta metav1.ObjectMeta) error {
+	currentSecret := &corev1.Secret{}
+	err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: resources.GetUserConfigurationSecretNameFromJenkins(r.jenkins), Namespace: r.jenkins.Namespace}, currentSecret)
+	if err != nil && errors.IsNotFound(err) {
+		return stackerr.WithStack(r.k8sClient.Create(context.TODO(), resources.NewUserConfigurationSecret(r.jenkins)))
+	} else if err != nil {
+		return stackerr.WithStack(err)
+	}
+	valid := r.verifyLabelsForWatchedResource(currentSecret)
+	if !valid {
+		currentSecret.ObjectMeta.Labels = resources.BuildLabelsForWatchedResources(r.jenkins)
+		return stackerr.WithStack(r.k8sClient.Update(context.TODO(), currentSecret))
+	}
+
+	return nil
+}
+
 func (r *ReconcileJenkinsBaseConfiguration) createRBAC(meta metav1.ObjectMeta) error {
 	serviceAccount := resources.NewServiceAccount(meta)
 	err := r.createResource(serviceAccount)

pkg/controller/jenkins/configuration/base/resources/pod.go

@@ -31,10 +31,14 @@ const (
 	JenkinsBaseConfigurationVolumePath = jenkinsPath + "/base-configuration"
 
 	jenkinsUserConfigurationVolumeName = "user-configuration"
-	// JenkinsUserConfigurationVolumePath is a path where are groovy scripts used to configure Jenkins
+	// JenkinsUserConfigurationVolumePath is a path where are groovy scripts and CasC configs used to configure Jenkins
-	// this scripts are provided by user
+	// this script is provided by user
 	JenkinsUserConfigurationVolumePath = jenkinsPath + "/user-configuration"
 
+	userConfigurationSecretVolumeName = "user-configuration-secrets"
+	// UserConfigurationSecretVolumePath is a path where are secrets used for groovy scripts and CasC configs
+	UserConfigurationSecretVolumePath = jenkinsPath + "/user-configuration-secrets"
+
 	httpPortName  = "http"
 	slavePortName = "slavelistener"
 	// HTTPPortInt defines Jenkins master HTTP port
@@ -121,6 +125,10 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
 							Name:  "JAVA_OPTS",
 							Value: "-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -Djenkins.install.runSetupWizard=false -Djava.awt.headless=true",
 						},
+						{
+							Name:  "SECRETS", // https://github.com/jenkinsci/configuration-as-code-plugin/blob/master/demos/kubernetes-secrets/README.md
+							Value: UserConfigurationSecretVolumePath,
+						},
 					},
 					Resources: jenkins.Spec.Master.Resources,
 					VolumeMounts: []corev1.VolumeMount{
@@ -154,6 +162,11 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
 							MountPath: jenkinsOperatorCredentialsVolumePath,
 							ReadOnly:  true,
 						},
+						{
+							Name:      userConfigurationSecretVolumeName,
+							MountPath: UserConfigurationSecretVolumePath,
+							ReadOnly:  true,
+						},
 					},
 				},
 			},
@@ -212,6 +225,14 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
 						},
 					},
 				},
+				{
+					Name: userConfigurationSecretVolumeName,
+					VolumeSource: corev1.VolumeSource{
+						Secret: &corev1.SecretVolumeSource{
+							SecretName: GetUserConfigurationSecretNameFromJenkins(jenkins),
+						},
+					},
+				},
 			},
 		},
 	}

pkg/controller/jenkins/configuration/base/resources/user_configuration_configmap.go

@@ -44,15 +44,13 @@ func GetUserConfigurationConfigMapName(jenkinsCRName string) string {
 
 // NewUserConfigurationConfigMap builds Kubernetes config map used to user configuration
 func NewUserConfigurationConfigMap(jenkins *v1alpha1.Jenkins) *corev1.ConfigMap {
-	meta := metav1.ObjectMeta{
-		Name:      GetUserConfigurationConfigMapNameFromJenkins(jenkins),
-		Namespace: jenkins.ObjectMeta.Namespace,
-		Labels:    BuildLabelsForWatchedResources(jenkins),
-	}
-
 	return &corev1.ConfigMap{
-		TypeMeta:   buildConfigMapTypeMeta(),
+		TypeMeta: buildConfigMapTypeMeta(),
-		ObjectMeta: meta,
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      GetUserConfigurationConfigMapNameFromJenkins(jenkins),
+			Namespace: jenkins.ObjectMeta.Namespace,
+			Labels:    BuildLabelsForWatchedResources(jenkins),
+		},
 		Data: map[string]string{
 			"1-configure-theme.groovy": configureTheme,
 		},

pkg/controller/jenkins/configuration/base/resources/user_configuration_secret.go

@@ -0,0 +1,33 @@
+package resources
+
+import (
+	"fmt"
+
+	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// GetUserConfigurationSecretNameFromJenkins returns name of Kubernetes secret used to store jenkins operator credentials
+func GetUserConfigurationSecretNameFromJenkins(jenkins *v1alpha1.Jenkins) string {
+	return fmt.Sprintf("%s-user-configuration-%s", constants.OperatorName, jenkins.Name)
+}
+
+// GetUserConfigurationSecretName returns name of Kubernetes secret used to store jenkins operator credentials
+func GetUserConfigurationSecretName(jenkinsCRName string) string {
+	return fmt.Sprintf("%s-user-configuration-%s", constants.OperatorName, jenkinsCRName)
+}
+
+// NewUserConfigurationSecret builds the Kubernetes secret resource which is used to store user sensitive data for Jenkins configuration
+func NewUserConfigurationSecret(jenkins *v1alpha1.Jenkins) *corev1.Secret {
+	return &corev1.Secret{
+		TypeMeta: buildServiceTypeMeta(),
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      GetUserConfigurationSecretNameFromJenkins(jenkins),
+			Namespace: jenkins.ObjectMeta.Namespace,
+			Labels:    BuildLabelsForWatchedResources(jenkins),
+		},
+	}
+}

pkg/controller/jenkins/configuration/user/casc/caac.go

@@ -1,6 +1,7 @@
 package casc
 
 import (
+	"context"
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
@@ -9,14 +10,19 @@ import (
 
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
 	jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/jobs"
 
 	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
 	k8s "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
-	jobHashParameterName = "hash"
+	userConfigurationHashParameterName       = "userConfigurationHash"
+	userConfigurationSecretHashParameterName = "userConfigurationSecretHash"
 )
 
 // ConfigurationAsCode defines API which configures Jenkins with help Configuration as a code plugin
@@ -25,23 +31,23 @@ type ConfigurationAsCode struct {
 	k8sClient     k8s.Client
 	logger        logr.Logger
 	jobName       string
-	configsPath   string
 }
 
 // New creates new instance of ConfigurationAsCode
-func New(jenkinsClient jenkinsclient.Jenkins, k8sClient k8s.Client, logger logr.Logger, jobName, configsPath string) *ConfigurationAsCode {
+func New(jenkinsClient jenkinsclient.Jenkins, k8sClient k8s.Client, logger logr.Logger, jobName string) *ConfigurationAsCode {
 	return &ConfigurationAsCode{
 		jenkinsClient: jenkinsClient,
 		k8sClient:     k8sClient,
 		logger:        logger,
 		jobName:       jobName,
-		configsPath:   configsPath,
 	}
 }
 
 // ConfigureJob configures jenkins job which configures Jenkins with help Configuration as a code plugin
 func (g *ConfigurationAsCode) ConfigureJob() error {
-	_, created, err := g.jenkinsClient.CreateOrUpdateJob(fmt.Sprintf(configurationJobXMLFmt, g.configsPath), g.jobName)
+	_, created, err := g.jenkinsClient.CreateOrUpdateJob(
+		fmt.Sprintf(configurationJobXMLFmt, resources.UserConfigurationSecretVolumePath, resources.JenkinsUserConfigurationVolumePath),
+		g.jobName)
 	if err != nil {
 		return err
 	}
@@ -52,29 +58,67 @@ func (g *ConfigurationAsCode) ConfigureJob() error {
 }
 
 // Ensure configures Jenkins with help Configuration as a code plugin
-func (g *ConfigurationAsCode) Ensure(secretOrConfigMapData map[string]string, jenkins *v1alpha1.Jenkins) (bool, error) {
+func (g *ConfigurationAsCode) Ensure(jenkins *v1alpha1.Jenkins) (bool, error) {
 	jobsClient := jobs.New(g.jenkinsClient, g.k8sClient, g.logger)
 
-	hash := g.calculateHash(secretOrConfigMapData)
+	configuration := &corev1.ConfigMap{}
-	done, err := jobsClient.EnsureBuildJob(g.jobName, hash, map[string]string{jobHashParameterName: hash}, jenkins, true)
+	namespaceName := types.NamespacedName{Namespace: jenkins.Namespace, Name: resources.GetUserConfigurationConfigMapNameFromJenkins(jenkins)}
+	err := g.k8sClient.Get(context.TODO(), namespaceName, configuration)
+	if err != nil {
+		return false, errors.WithStack(err)
+	}
+
+	secret := &corev1.Secret{}
+	namespaceName = types.NamespacedName{Namespace: jenkins.Namespace, Name: resources.GetUserConfigurationSecretNameFromJenkins(jenkins)}
+	err = g.k8sClient.Get(context.TODO(), namespaceName, configuration)
+	if err != nil {
+		return false, errors.WithStack(err)
+	}
+
+	userConfigurationSecretHash := g.calculateUserConfigurationSecretHash(secret)
+	userConfigurationHash := g.calculateUserConfigurationHash(configuration)
+	done, err := jobsClient.EnsureBuildJob(
+		g.jobName,
+		userConfigurationSecretHash+userConfigurationHash,
+		map[string]string{
+			userConfigurationHashParameterName:       userConfigurationHash,
+			userConfigurationSecretHashParameterName: userConfigurationSecretHash,
+		},
+		jenkins,
+		true)
 	if err != nil {
 		return false, err
 	}
 	return done, nil
 }
 
-func (g *ConfigurationAsCode) calculateHash(secretOrConfigMapData map[string]string) string {
+func (g *ConfigurationAsCode) calculateUserConfigurationSecretHash(userConfigurationSecret *corev1.Secret) string {
 	hash := sha256.New()
 
 	var keys []string
-	for key := range secretOrConfigMapData {
+	for key := range userConfigurationSecret.Data {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		hash.Write([]byte(key))
+		hash.Write([]byte(userConfigurationSecret.Data[key]))
+	}
+	return base64.StdEncoding.EncodeToString(hash.Sum(nil))
+}
+
+func (g *ConfigurationAsCode) calculateUserConfigurationHash(userConfiguration *corev1.ConfigMap) string {
+	hash := sha256.New()
+
+	var keys []string
+	for key := range userConfiguration.Data {
 		keys = append(keys, key)
 	}
 	sort.Strings(keys)
 	for _, key := range keys {
 		if strings.HasSuffix(key, ".yaml") {
 			hash.Write([]byte(key))
-			hash.Write([]byte(secretOrConfigMapData[key]))
+			hash.Write([]byte(userConfiguration.Data[key]))
 		}
 	}
 	return base64.StdEncoding.EncodeToString(hash.Sum(nil))
@@ -90,9 +134,15 @@ const configurationJobXMLFmt = `<?xml version='1.1' encoding='UTF-8'?>
     <hudson.model.ParametersDefinitionProperty>
       <parameterDefinitions>
         <hudson.model.StringParameterDefinition>
-          <name>` + jobHashParameterName + `</name>
+          <name>` + userConfigurationSecretHashParameterName + `</name>
-          <description></description>
+          <description/>
-          <defaultValue></defaultValue>
+          <defaultValue/>
+          <trim>false</trim>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>` + userConfigurationHashParameterName + `</name>
+          <description/>
+          <defaultValue/>
           <trim>false</trim>
         </hudson.model.StringParameterDefinition>
       </parameterDefinitions>
@@ -101,28 +151,23 @@ const configurationJobXMLFmt = `<?xml version='1.1' encoding='UTF-8'?>
   <definition class="org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition" plugin="workflow-cps@2.61.1">
     <script>import io.jenkins.plugins.casc.yaml.YamlSource;
 
+def secretsPath = &apos;%s&apos;
 def configsPath = &apos;%s&apos;
-def expectedHash = params.hash
+def userConfigurationSecretExpectedHash = params.` + userConfigurationSecretHashParameterName + `
+def userConfigurationExpectedHash = params.` + userConfigurationHashParameterName + `
 
 node(&apos;master&apos;) {
+    def secretsText = sh(script: &quot;ls ${secretsPath} | grep .yaml | sort&quot;, returnStdout: true).trim()
+    def secrets = []
+    secrets.addAll(secretsText.tokenize(&apos;\n&apos;))
+
     def configsText = sh(script: &quot;ls ${configsPath} | grep .yaml | sort&quot;, returnStdout: true).trim()
     def configs = []
     configs.addAll(configsText.tokenize(&apos;\n&apos;))
     
     stage(&apos;Synchronizing files&apos;) {
-        def complete = false
+		synchronizeFiles(secretsPath, (String[])secrets, userConfigurationSecretExpectedHash)
-        for(int i = 1; i &lt;= 10; i++) {
+		synchronizeFiles(configsPath, (String[])configs, userConfigurationExpectedHash)
-            def actualHash = calculateHash((String[])configs, configsPath)
-            println &quot;Expected hash &apos;${expectedHash}&apos;, actual hash &apos;${actualHash}&apos;&quot;
-            if(expectedHash == actualHash) {
-                complete = true
-                break
-            }
-            sleep 2
-        }
-        if(!complete) {
-            error(&quot;Timeout while synchronizing files&quot;)
-        }
     }
     
     for(config in configs) {
@@ -134,6 +179,23 @@ node(&apos;master&apos;) {
     }
 }
 
+def synchronizeFiles(String path, String[] files, String hash) {
+    def complete = false
+    for(int i = 1; i &lt;= 10; i++) {
+        def actualHash = calculateHash(files, path)
+        println &quot;Expected hash &apos;${hash}&apos;, actual hash &apos;${actualHash}&apos;, path &apos;${path}&apos;&quot;
+        if(hash == actualHash) {
+            complete = true
+            break
+        }
+        sleep 2
+    }
+    if(!complete) {
+        error(&quot;Timeout while synchronizing files&quot;)
+    }
+}
+
+
 @NonCPS
 def calculateHash(String[] configs, String configsPath) {
     def hash = java.security.MessageDigest.getInstance(&quot;SHA-256&quot;)

pkg/controller/jenkins/configuration/user/reconcile.go

@@ -105,12 +105,12 @@ func (r *ReconcileUserConfiguration) ensureUserConfiguration(jenkinsClient jenki
 		return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10}, nil
 	}
 
-	configurationAsCodeClient := casc.New(jenkinsClient, r.k8sClient, r.logger, constants.UserConfigurationCASCJobName, resources.JenkinsUserConfigurationVolumePath)
+	configurationAsCodeClient := casc.New(jenkinsClient, r.k8sClient, r.logger, constants.UserConfigurationCASCJobName)
 	err = configurationAsCodeClient.ConfigureJob()
 	if err != nil {
 		return reconcile.Result{}, err
 	}
-	done, err = configurationAsCodeClient.Ensure(configuration.Data, r.jenkins)
+	done, err = configurationAsCodeClient.Ensure(r.jenkins)
 	if err != nil {
 		return reconcile.Result{}, err
 	}

test/e2e/configuration_test.go

@@ -32,9 +32,11 @@ func TestConfiguration(t *testing.T) {
 	jenkinsCRName := "e2e"
 	numberOfExecutors := 6
 	systemMessage := "Configuration as Code integration works!!!"
+	systemMessageEnvName := "SYSTEM_MESSAGE"
 
 	// base
-	createUserConfigurationConfigMap(t, jenkinsCRName, namespace, numberOfExecutors, systemMessage)
+	createUserConfigurationSecret(t, jenkinsCRName, namespace, systemMessageEnvName, systemMessage)
+	createUserConfigurationConfigMap(t, jenkinsCRName, namespace, numberOfExecutors, fmt.Sprintf("${%s}", systemMessageEnvName))
 	jenkins := createJenkinsCR(t, jenkinsCRName, namespace)
 	createDefaultLimitsForContainersInNamespace(t, namespace)
 	waitForJenkinsBaseConfigurationToComplete(t, jenkins)
@@ -49,6 +51,23 @@ func TestConfiguration(t *testing.T) {
 	verifyUserConfiguration(t, client, numberOfExecutors, systemMessage)
 }
 
+func createUserConfigurationSecret(t *testing.T, jenkinsCRName string, namespace string, systemMessageEnvName, systemMessage string) {
+	userConfiguration := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      resources.GetUserConfigurationSecretName(jenkinsCRName),
+			Namespace: namespace,
+		},
+		StringData: map[string]string{
+			systemMessageEnvName: systemMessage,
+		},
+	}
+
+	t.Logf("User configuration secret %+v", *userConfiguration)
+	if err := framework.Global.Client.Create(context.TODO(), userConfiguration, nil); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func createUserConfigurationConfigMap(t *testing.T, jenkinsCRName string, namespace string, numberOfExecutors int, systemMessage string) {
 	userConfiguration := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{