#9 Add secret support in Configuration as Code plugin integration

docs/getting-started.md

@@ -215,6 +215,29 @@ You can verify if your pipelines were successfully configured in Jenkins Seed Jo
 Jenkins can be customized using groovy scripts or configuration as code plugin. All custom configuration is stored in
 the **jenkins-operator-user-configuration-example** ConfigMap which is automatically created by **jenkins-operator**.
 
+**jenkins-operator** creates **jenkins-operator-user-configuration-example** secret where user can store sensitive 
+information used for custom configuration. If you have entry in secret named `PASSWORD` then you can use it in 
+Configuration as Plugin as `adminAddress: "${PASSWORD}"`.
+
+```
+kubectl get secret jenkins-operator-user-configuration-example -o yaml
+
+apiVersion: v1
+data:
+  SECRET_JENKINS_ADMIN_ADDRESS: YXNkZgo=
+kind: Secret
+metadata:
+  creationTimestamp: 2019-03-03T11:54:36Z
+  labels:
+    app: jenkins-operator
+    jenkins-cr: example
+    watch: "true"
+  name: jenkins-operator-user-configuration-example
+  namespace: default
+type: Opaque
+
+```
+
 ```
 kubectl get configmap jenkins-operator-user-configuration-example -o yaml
 
@@ -243,6 +266,7 @@ data:
   1-system-message.yaml: |2
     jenkins:
       systemMessage: "Configuration as Code integration works!!!"
+      adminAddress: "${SECRET_JENKINS_ADMIN_ADDRESS}"
 kind: ConfigMap
 metadata:
   labels:

pkg/controller/jenkins/configuration/base/reconcile.go

@@ -126,6 +126,11 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureResourcesRequiredForJenkinsPod
 	}
 	r.logger.V(log.VDebug).Info("User configuration config map is present")
 
+	if err := r.createUserConfigurationSecret(metaObject); err != nil {
+		return err
+	}
+	r.logger.V(log.VDebug).Info("User configuration secret is present")
+
 	if err := r.createRBAC(metaObject); err != nil {
 		return err
 	}
@@ -252,6 +257,23 @@ func (r *ReconcileJenkinsBaseConfiguration) createUserConfigurationConfigMap(met
 	return nil
 }
 
+func (r *ReconcileJenkinsBaseConfiguration) createUserConfigurationSecret(meta metav1.ObjectMeta) error {
+	currentSecret := &corev1.Secret{}
+	err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: resources.GetUserConfigurationSecretNameFromJenkins(r.jenkins), Namespace: r.jenkins.Namespace}, currentSecret)
+	if err != nil && errors.IsNotFound(err) {
+		return stackerr.WithStack(r.k8sClient.Create(context.TODO(), resources.NewUserConfigurationSecret(r.jenkins)))
+	} else if err != nil {
+		return stackerr.WithStack(err)
+	}
+	valid := r.verifyLabelsForWatchedResource(currentSecret)
+	if !valid {
+		currentSecret.ObjectMeta.Labels = resources.BuildLabelsForWatchedResources(r.jenkins)
+		return stackerr.WithStack(r.k8sClient.Update(context.TODO(), currentSecret))
+	}
+
+	return nil
+}
+
 func (r *ReconcileJenkinsBaseConfiguration) createRBAC(meta metav1.ObjectMeta) error {
 	serviceAccount := resources.NewServiceAccount(meta)
 	err := r.createResource(serviceAccount)

pkg/controller/jenkins/configuration/base/resources/pod.go

@@ -31,10 +31,14 @@ const (
 	JenkinsBaseConfigurationVolumePath = jenkinsPath + "/base-configuration"
 
 	jenkinsUserConfigurationVolumeName = "user-configuration"
-	// JenkinsUserConfigurationVolumePath is a path where are groovy scripts used to configure Jenkins
+	// JenkinsUserConfigurationVolumePath is a path where are groovy scripts and CasC configs used to configure Jenkins
-	// this scripts are provided by user
+	// this script is provided by user
 	JenkinsUserConfigurationVolumePath = jenkinsPath + "/user-configuration"
 
+	userConfigurationSecretVolumeName = "user-configuration-secrets"
+	// UserConfigurationSecretVolumePath is a path where are secrets used for groovy scripts and CasC configs
+	UserConfigurationSecretVolumePath = jenkinsPath + "/user-configuration-secrets"
+
 	httpPortName  = "http"
 	slavePortName = "slavelistener"
 	// HTTPPortInt defines Jenkins master HTTP port
@@ -121,6 +125,10 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
 							Name:  "JAVA_OPTS",
 							Value: "-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -Djenkins.install.runSetupWizard=false -Djava.awt.headless=true",
 						},
+						{
+							Name:  "SECRETS", // https://github.com/jenkinsci/configuration-as-code-plugin/blob/master/demos/kubernetes-secrets/README.md
+							Value: UserConfigurationSecretVolumePath,
+						},
 					},
 					Resources: jenkins.Spec.Master.Resources,
 					VolumeMounts: []corev1.VolumeMount{
@@ -154,6 +162,11 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
 							MountPath: jenkinsOperatorCredentialsVolumePath,
 							ReadOnly:  true,
 						},
+						{
+							Name:      userConfigurationSecretVolumeName,
+							MountPath: UserConfigurationSecretVolumePath,
+							ReadOnly:  true,
+						},
 					},
 				},
 			},
@@ -212,6 +225,14 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
 						},
 					},
 				},
+				{
+					Name: userConfigurationSecretVolumeName,
+					VolumeSource: corev1.VolumeSource{
+						Secret: &corev1.SecretVolumeSource{
+							SecretName: GetUserConfigurationSecretNameFromJenkins(jenkins),
+						},
+					},
+				},
 			},
 		},
 	}

pkg/controller/jenkins/configuration/base/resources/user_configuration_configmap.go

@@ -44,15 +44,13 @@ func GetUserConfigurationConfigMapName(jenkinsCRName string) string {
 
 // NewUserConfigurationConfigMap builds Kubernetes config map used to user configuration
 func NewUserConfigurationConfigMap(jenkins *v1alpha1.Jenkins) *corev1.ConfigMap {
-	meta := metav1.ObjectMeta{
+	return &corev1.ConfigMap{
+		TypeMeta: buildConfigMapTypeMeta(),
+		ObjectMeta: metav1.ObjectMeta{
 			Name:      GetUserConfigurationConfigMapNameFromJenkins(jenkins),
 			Namespace: jenkins.ObjectMeta.Namespace,
 			Labels:    BuildLabelsForWatchedResources(jenkins),
-	}
+		},
-
-	return &corev1.ConfigMap{
-		TypeMeta:   buildConfigMapTypeMeta(),
-		ObjectMeta: meta,
 		Data: map[string]string{
 			"1-configure-theme.groovy": configureTheme,
 		},

pkg/controller/jenkins/configuration/base/resources/user_configuration_secret.go

@@ -0,0 +1,33 @@
+package resources
+
+import (
+	"fmt"
+
+	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// GetUserConfigurationSecretNameFromJenkins returns name of Kubernetes secret used to store jenkins operator credentials
+func GetUserConfigurationSecretNameFromJenkins(jenkins *v1alpha1.Jenkins) string {
+	return fmt.Sprintf("%s-user-configuration-%s", constants.OperatorName, jenkins.Name)
+}
+
+// GetUserConfigurationSecretName returns name of Kubernetes secret used to store jenkins operator credentials
+func GetUserConfigurationSecretName(jenkinsCRName string) string {
+	return fmt.Sprintf("%s-user-configuration-%s", constants.OperatorName, jenkinsCRName)
+}
+
+// NewUserConfigurationSecret builds the Kubernetes secret resource which is used to store user sensitive data for Jenkins configuration
+func NewUserConfigurationSecret(jenkins *v1alpha1.Jenkins) *corev1.Secret {
+	return &corev1.Secret{
+		TypeMeta: buildServiceTypeMeta(),
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      GetUserConfigurationSecretNameFromJenkins(jenkins),
+			Namespace: jenkins.ObjectMeta.Namespace,
+			Labels:    BuildLabelsForWatchedResources(jenkins),
+		},
+	}
+}

pkg/controller/jenkins/configuration/user/casc/caac.go

@@ -1,6 +1,7 @@
 package casc
 
 import (
+	"context"
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
@@ -9,14 +10,19 @@ import (
 
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
 	jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/jobs"
 
 	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
 	k8s "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
-	jobHashParameterName = "hash"
+	userConfigurationHashParameterName       = "userConfigurationHash"
+	userConfigurationSecretHashParameterName = "userConfigurationSecretHash"
 )
 
 // ConfigurationAsCode defines API which configures Jenkins with help Configuration as a code plugin
@@ -25,23 +31,23 @@ type ConfigurationAsCode struct {
 	k8sClient     k8s.Client
 	logger        logr.Logger
 	jobName       string
-	configsPath   string
 }
 
 // New creates new instance of ConfigurationAsCode
-func New(jenkinsClient jenkinsclient.Jenkins, k8sClient k8s.Client, logger logr.Logger, jobName, configsPath string) *ConfigurationAsCode {
+func New(jenkinsClient jenkinsclient.Jenkins, k8sClient k8s.Client, logger logr.Logger, jobName string) *ConfigurationAsCode {
 	return &ConfigurationAsCode{
 		jenkinsClient: jenkinsClient,
 		k8sClient:     k8sClient,
 		logger:        logger,
 		jobName:       jobName,
-		configsPath:   configsPath,
 	}
 }
 
 // ConfigureJob configures jenkins job which configures Jenkins with help Configuration as a code plugin
 func (g *ConfigurationAsCode) ConfigureJob() error {
-	_, created, err := g.jenkinsClient.CreateOrUpdateJob(fmt.Sprintf(configurationJobXMLFmt, g.configsPath), g.jobName)
+	_, created, err := g.jenkinsClient.CreateOrUpdateJob(
+		fmt.Sprintf(configurationJobXMLFmt, resources.UserConfigurationSecretVolumePath, resources.JenkinsUserConfigurationVolumePath),
+		g.jobName)
 	if err != nil {
 		return err
 	}
@@ -52,29 +58,67 @@ func (g *ConfigurationAsCode) ConfigureJob() error {
 }
 
 // Ensure configures Jenkins with help Configuration as a code plugin
-func (g *ConfigurationAsCode) Ensure(secretOrConfigMapData map[string]string, jenkins *v1alpha1.Jenkins) (bool, error) {
+func (g *ConfigurationAsCode) Ensure(jenkins *v1alpha1.Jenkins) (bool, error) {
 	jobsClient := jobs.New(g.jenkinsClient, g.k8sClient, g.logger)
 
-	hash := g.calculateHash(secretOrConfigMapData)
+	configuration := &corev1.ConfigMap{}
-	done, err := jobsClient.EnsureBuildJob(g.jobName, hash, map[string]string{jobHashParameterName: hash}, jenkins, true)
+	namespaceName := types.NamespacedName{Namespace: jenkins.Namespace, Name: resources.GetUserConfigurationConfigMapNameFromJenkins(jenkins)}
+	err := g.k8sClient.Get(context.TODO(), namespaceName, configuration)
+	if err != nil {
+		return false, errors.WithStack(err)
+	}
+
+	secret := &corev1.Secret{}
+	namespaceName = types.NamespacedName{Namespace: jenkins.Namespace, Name: resources.GetUserConfigurationSecretNameFromJenkins(jenkins)}
+	err = g.k8sClient.Get(context.TODO(), namespaceName, configuration)
+	if err != nil {
+		return false, errors.WithStack(err)
+	}
+
+	userConfigurationSecretHash := g.calculateUserConfigurationSecretHash(secret)
+	userConfigurationHash := g.calculateUserConfigurationHash(configuration)
+	done, err := jobsClient.EnsureBuildJob(
+		g.jobName,
+		userConfigurationSecretHash+userConfigurationHash,
+		map[string]string{
+			userConfigurationHashParameterName:       userConfigurationHash,
+			userConfigurationSecretHashParameterName: userConfigurationSecretHash,
+		},
+		jenkins,
+		true)
 	if err != nil {
 		return false, err
 	}
 	return done, nil
 }
 
-func (g *ConfigurationAsCode) calculateHash(secretOrConfigMapData map[string]string) string {
+func (g *ConfigurationAsCode) calculateUserConfigurationSecretHash(userConfigurationSecret *corev1.Secret) string {
 	hash := sha256.New()
 
 	var keys []string
-	for key := range secretOrConfigMapData {
+	for key := range userConfigurationSecret.Data {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		hash.Write([]byte(key))
+		hash.Write([]byte(userConfigurationSecret.Data[key]))
+	}
+	return base64.StdEncoding.EncodeToString(hash.Sum(nil))
+}
+
+func (g *ConfigurationAsCode) calculateUserConfigurationHash(userConfiguration *corev1.ConfigMap) string {
+	hash := sha256.New()
+
+	var keys []string
+	for key := range userConfiguration.Data {
 		keys = append(keys, key)
 	}
 	sort.Strings(keys)
 	for _, key := range keys {
 		if strings.HasSuffix(key, ".yaml") {
 			hash.Write([]byte(key))
-			hash.Write([]byte(secretOrConfigMapData[key]))
+			hash.Write([]byte(userConfiguration.Data[key]))
 		}
 	}
 	return base64.StdEncoding.EncodeToString(hash.Sum(nil))
@@ -90,9 +134,15 @@ const configurationJobXMLFmt = `<?xml version='1.1' encoding='UTF-8'?>
     <hudson.model.ParametersDefinitionProperty>
       <parameterDefinitions>
         <hudson.model.StringParameterDefinition>
-          <name>` + jobHashParameterName + `</name>
+          <name>` + userConfigurationSecretHashParameterName + `</name>
-          <description></description>
+          <description/>
-          <defaultValue></defaultValue>
+          <defaultValue/>
+          <trim>false</trim>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>` + userConfigurationHashParameterName + `</name>
+          <description/>
+          <defaultValue/>
           <trim>false</trim>
         </hudson.model.StringParameterDefinition>
       </parameterDefinitions>
@@ -101,28 +151,23 @@ const configurationJobXMLFmt = `<?xml version='1.1' encoding='UTF-8'?>
   <definition class="org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition" plugin="workflow-cps@2.61.1">
     <script>import io.jenkins.plugins.casc.yaml.YamlSource;
 
+def secretsPath = &apos;%s&apos;
 def configsPath = &apos;%s&apos;
-def expectedHash = params.hash
+def userConfigurationSecretExpectedHash = params.` + userConfigurationSecretHashParameterName + `
+def userConfigurationExpectedHash = params.` + userConfigurationHashParameterName + `
 
 node(&apos;master&apos;) {
+    def secretsText = sh(script: &quot;ls ${secretsPath} | grep .yaml | sort&quot;, returnStdout: true).trim()
+    def secrets = []
+    secrets.addAll(secretsText.tokenize(&apos;\n&apos;))
+
     def configsText = sh(script: &quot;ls ${configsPath} | grep .yaml | sort&quot;, returnStdout: true).trim()
     def configs = []
     configs.addAll(configsText.tokenize(&apos;\n&apos;))
     
     stage(&apos;Synchronizing files&apos;) {
-        def complete = false
+		synchronizeFiles(secretsPath, (String[])secrets, userConfigurationSecretExpectedHash)
-        for(int i = 1; i &lt;= 10; i++) {
+		synchronizeFiles(configsPath, (String[])configs, userConfigurationExpectedHash)
-            def actualHash = calculateHash((String[])configs, configsPath)
-            println &quot;Expected hash &apos;${expectedHash}&apos;, actual hash &apos;${actualHash}&apos;&quot;
-            if(expectedHash == actualHash) {
-                complete = true
-                break
-            }
-            sleep 2
-        }
-        if(!complete) {
-            error(&quot;Timeout while synchronizing files&quot;)
-        }
     }
     
     for(config in configs) {
@@ -134,6 +179,23 @@ node(&apos;master&apos;) {
     }
 }
 
+def synchronizeFiles(String path, String[] files, String hash) {
+    def complete = false
+    for(int i = 1; i &lt;= 10; i++) {
+        def actualHash = calculateHash(files, path)
+        println &quot;Expected hash &apos;${hash}&apos;, actual hash &apos;${actualHash}&apos;, path &apos;${path}&apos;&quot;
+        if(hash == actualHash) {
+            complete = true
+            break
+        }
+        sleep 2
+    }
+    if(!complete) {
+        error(&quot;Timeout while synchronizing files&quot;)
+    }
+}
+
+
 @NonCPS
 def calculateHash(String[] configs, String configsPath) {
     def hash = java.security.MessageDigest.getInstance(&quot;SHA-256&quot;)

pkg/controller/jenkins/configuration/user/reconcile.go

@@ -105,12 +105,12 @@ func (r *ReconcileUserConfiguration) ensureUserConfiguration(jenkinsClient jenki
 		return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10}, nil
 	}
 
-	configurationAsCodeClient := casc.New(jenkinsClient, r.k8sClient, r.logger, constants.UserConfigurationCASCJobName, resources.JenkinsUserConfigurationVolumePath)
+	configurationAsCodeClient := casc.New(jenkinsClient, r.k8sClient, r.logger, constants.UserConfigurationCASCJobName)
 	err = configurationAsCodeClient.ConfigureJob()
 	if err != nil {
 		return reconcile.Result{}, err
 	}
-	done, err = configurationAsCodeClient.Ensure(configuration.Data, r.jenkins)
+	done, err = configurationAsCodeClient.Ensure(r.jenkins)
 	if err != nil {
 		return reconcile.Result{}, err
 	}

test/e2e/configuration_test.go

@@ -32,9 +32,11 @@ func TestConfiguration(t *testing.T) {
 	jenkinsCRName := "e2e"
 	numberOfExecutors := 6
 	systemMessage := "Configuration as Code integration works!!!"
+	systemMessageEnvName := "SYSTEM_MESSAGE"
 
 	// base
-	createUserConfigurationConfigMap(t, jenkinsCRName, namespace, numberOfExecutors, systemMessage)
+	createUserConfigurationSecret(t, jenkinsCRName, namespace, systemMessageEnvName, systemMessage)
+	createUserConfigurationConfigMap(t, jenkinsCRName, namespace, numberOfExecutors, fmt.Sprintf("${%s}", systemMessageEnvName))
 	jenkins := createJenkinsCR(t, jenkinsCRName, namespace)
 	createDefaultLimitsForContainersInNamespace(t, namespace)
 	waitForJenkinsBaseConfigurationToComplete(t, jenkins)
@@ -49,6 +51,23 @@ func TestConfiguration(t *testing.T) {
 	verifyUserConfiguration(t, client, numberOfExecutors, systemMessage)
 }
 
+func createUserConfigurationSecret(t *testing.T, jenkinsCRName string, namespace string, systemMessageEnvName, systemMessage string) {
+	userConfiguration := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      resources.GetUserConfigurationSecretName(jenkinsCRName),
+			Namespace: namespace,
+		},
+		StringData: map[string]string{
+			systemMessageEnvName: systemMessage,
+		},
+	}
+
+	t.Logf("User configuration secret %+v", *userConfiguration)
+	if err := framework.Global.Client.Create(context.TODO(), userConfiguration, nil); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func createUserConfigurationConfigMap(t *testing.T, jenkinsCRName string, namespace string, numberOfExecutors int, systemMessage string) {
 	userConfiguration := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{