public_git
/
kubernetes-operator
mirror of https://github.com/jenkinsci/kubernetes-operator.git
1
0
Fork 0
Code Issues Releases Wiki Activity

#9 Add secret support in Configuration as Code plugin integration

This commit is contained in:
Tomasz Sęk 2019-03-03 17:59:22 +01:00
parent ebf1163b28
commit f817d2ad09
No known key found for this signature in database
GPG Key ID: DC356D23F6A644D0
8 changed files with 221 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
docs/getting-started.md
View File

@ -213,7 +213,30 @@ You can verify if your pipelines were successfully configured in Jenkins Seed Jo
## Jenkins Customisation
Jenkins can be customized using groovy scripts or configuration as code plugin. All custom configuration is stored in
the **jenkins-operator-user-configuration-example** ConfigMap which is automatically created by **jenkins-operator**.
the **jenkins-operator-user-configuration-example** ConfigMap which is automatically created by **jenkins-operator**.
**jenkins-operator** creates **jenkins-operator-user-configuration-example** secret where user can store sensitive
information used for custom configuration. If you have entry in secret named `PASSWORD` then you can use it in
Configuration as Plugin as `adminAddress: "${PASSWORD}"`.
```
kubectl get secret jenkins-operator-user-configuration-example -o yaml
apiVersion: v1
data:
SECRET_JENKINS_ADMIN_ADDRESS: YXNkZgo=
kind: Secret
metadata:
creationTimestamp: 2019-03-03T11:54:36Z
labels:
app: jenkins-operator
jenkins-cr: example
watch: "true"
name: jenkins-operator-user-configuration-example
namespace: default
type: Opaque
```
```
kubectl get configmap jenkins-operator-user-configuration-example -o yaml
@ -243,6 +266,7 @@ data:
1-system-message.yaml: |2
jenkins:
systemMessage: "Configuration as Code integration works!!!"
adminAddress: "${SECRET_JENKINS_ADMIN_ADDRESS}"
kind: ConfigMap
metadata:
labels:

22
pkg/controller/jenkins/configuration/base/reconcile.go
View File

@ -126,6 +126,11 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureResourcesRequiredForJenkinsPod
}
r.logger.V(log.VDebug).Info("User configuration config map is present")
if err := r.createUserConfigurationSecret(metaObject); err != nil {
return err
}
r.logger.V(log.VDebug).Info("User configuration secret is present")
if err := r.createRBAC(metaObject); err != nil {
return err
}
@ -252,6 +257,23 @@ func (r *ReconcileJenkinsBaseConfiguration) createUserConfigurationConfigMap(met
return nil
}
func (r *ReconcileJenkinsBaseConfiguration) createUserConfigurationSecret(meta metav1.ObjectMeta) error {
currentSecret := &corev1.Secret{}
err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: resources.GetUserConfigurationSecretNameFromJenkins(r.jenkins), Namespace: r.jenkins.Namespace}, currentSecret)
if err != nil && errors.IsNotFound(err) {
return stackerr.WithStack(r.k8sClient.Create(context.TODO(), resources.NewUserConfigurationSecret(r.jenkins)))
} else if err != nil {
return stackerr.WithStack(err)
}
valid := r.verifyLabelsForWatchedResource(currentSecret)
if !valid {
currentSecret.ObjectMeta.Labels = resources.BuildLabelsForWatchedResources(r.jenkins)
return stackerr.WithStack(r.k8sClient.Update(context.TODO(), currentSecret))
}
return nil
}
func (r *ReconcileJenkinsBaseConfiguration) createRBAC(meta metav1.ObjectMeta) error {
serviceAccount := resources.NewServiceAccount(meta)
err := r.createResource(serviceAccount)

25
pkg/controller/jenkins/configuration/base/resources/pod.go
View File

@ -31,10 +31,14 @@ const (
JenkinsBaseConfigurationVolumePath = jenkinsPath + "/base-configuration"
jenkinsUserConfigurationVolumeName = "user-configuration"
// JenkinsUserConfigurationVolumePath is a path where are groovy scripts used to configure Jenkins
// this scripts are provided by user
// JenkinsUserConfigurationVolumePath is a path where are groovy scripts and CasC configs used to configure Jenkins
// this script is provided by user
JenkinsUserConfigurationVolumePath = jenkinsPath + "/user-configuration"
userConfigurationSecretVolumeName = "user-configuration-secrets"
// UserConfigurationSecretVolumePath is a path where are secrets used for groovy scripts and CasC configs
UserConfigurationSecretVolumePath = jenkinsPath + "/user-configuration-secrets"
httpPortName = "http"
slavePortName = "slavelistener"
// HTTPPortInt defines Jenkins master HTTP port
@ -121,6 +125,10 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
Name: "JAVA_OPTS",
Value: "-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -Djenkins.install.runSetupWizard=false -Djava.awt.headless=true",
},
{
Name: "SECRETS", // https://github.com/jenkinsci/configuration-as-code-plugin/blob/master/demos/kubernetes-secrets/README.md
Value: UserConfigurationSecretVolumePath,
},
},
Resources: jenkins.Spec.Master.Resources,
VolumeMounts: []corev1.VolumeMount{
@ -154,6 +162,11 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
MountPath: jenkinsOperatorCredentialsVolumePath,
ReadOnly: true,
},
{
Name: userConfigurationSecretVolumeName,
MountPath: UserConfigurationSecretVolumePath,
ReadOnly: true,
},
},
},
},
@ -212,6 +225,14 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
},
},
},
{
Name: userConfigurationSecretVolumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: GetUserConfigurationSecretNameFromJenkins(jenkins),
},
},
},
},
},
}

14
pkg/controller/jenkins/configuration/base/resources/user_configuration_configmap.go
View File

@ -44,15 +44,13 @@ func GetUserConfigurationConfigMapName(jenkinsCRName string) string {
// NewUserConfigurationConfigMap builds Kubernetes config map used to user configuration
func NewUserConfigurationConfigMap(jenkins *v1alpha1.Jenkins) *corev1.ConfigMap {
meta := metav1.ObjectMeta{
Name: GetUserConfigurationConfigMapNameFromJenkins(jenkins),
Namespace: jenkins.ObjectMeta.Namespace,
Labels: BuildLabelsForWatchedResources(jenkins),
}
return &corev1.ConfigMap{
TypeMeta: buildConfigMapTypeMeta(),
ObjectMeta: meta,
TypeMeta: buildConfigMapTypeMeta(),
ObjectMeta: metav1.ObjectMeta{
Name: GetUserConfigurationConfigMapNameFromJenkins(jenkins),
Namespace: jenkins.ObjectMeta.Namespace,
Labels: BuildLabelsForWatchedResources(jenkins),
},
Data: map[string]string{
"1-configure-theme.groovy": configureTheme,
},

33
pkg/controller/jenkins/configuration/base/resources/user_configuration_secret.go Normal file
View File

@ -0,0 +1,33 @@
package resources
import (
"fmt"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetUserConfigurationSecretNameFromJenkins returns name of Kubernetes secret used to store jenkins operator credentials
func GetUserConfigurationSecretNameFromJenkins(jenkins *v1alpha1.Jenkins) string {
return fmt.Sprintf("%s-user-configuration-%s", constants.OperatorName, jenkins.Name)
}
// GetUserConfigurationSecretName returns name of Kubernetes secret used to store jenkins operator credentials
func GetUserConfigurationSecretName(jenkinsCRName string) string {
return fmt.Sprintf("%s-user-configuration-%s", constants.OperatorName, jenkinsCRName)
}
// NewUserConfigurationSecret builds the Kubernetes secret resource which is used to store user sensitive data for Jenkins configuration
func NewUserConfigurationSecret(jenkins *v1alpha1.Jenkins) *corev1.Secret {
return &corev1.Secret{
TypeMeta: buildServiceTypeMeta(),
ObjectMeta: metav1.ObjectMeta{
Name: GetUserConfigurationSecretNameFromJenkins(jenkins),
Namespace: jenkins.ObjectMeta.Namespace,
Labels: BuildLabelsForWatchedResources(jenkins),
},
}
}

118
pkg/controller/jenkins/configuration/user/casc/caac.go
View File

@ -1,6 +1,7 @@
package casc
import (
"context"
"crypto/sha256"
"encoding/base64"
"fmt"
@ -9,14 +10,19 @@ import (
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/jobs"
"github.com/go-logr/logr"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
k8s "sigs.k8s.io/controller-runtime/pkg/client"
)
const (
jobHashParameterName = "hash"
userConfigurationHashParameterName = "userConfigurationHash"
userConfigurationSecretHashParameterName = "userConfigurationSecretHash"
)
// ConfigurationAsCode defines API which configures Jenkins with help Configuration as a code plugin
@ -25,23 +31,23 @@ type ConfigurationAsCode struct {
k8sClient k8s.Client
logger logr.Logger
jobName string
configsPath string
}
// New creates new instance of ConfigurationAsCode
func New(jenkinsClient jenkinsclient.Jenkins, k8sClient k8s.Client, logger logr.Logger, jobName, configsPath string) *ConfigurationAsCode {
func New(jenkinsClient jenkinsclient.Jenkins, k8sClient k8s.Client, logger logr.Logger, jobName string) *ConfigurationAsCode {
return &ConfigurationAsCode{
jenkinsClient: jenkinsClient,
k8sClient: k8sClient,
logger: logger,
jobName: jobName,
configsPath: configsPath,
}
}
// ConfigureJob configures jenkins job which configures Jenkins with help Configuration as a code plugin
func (g *ConfigurationAsCode) ConfigureJob() error {
_, created, err := g.jenkinsClient.CreateOrUpdateJob(fmt.Sprintf(configurationJobXMLFmt, g.configsPath), g.jobName)
_, created, err := g.jenkinsClient.CreateOrUpdateJob(
fmt.Sprintf(configurationJobXMLFmt, resources.UserConfigurationSecretVolumePath, resources.JenkinsUserConfigurationVolumePath),
g.jobName)
if err != nil {
return err
}
@ -52,29 +58,67 @@ func (g *ConfigurationAsCode) ConfigureJob() error {
}
// Ensure configures Jenkins with help Configuration as a code plugin
func (g *ConfigurationAsCode) Ensure(secretOrConfigMapData map[string]string, jenkins *v1alpha1.Jenkins) (bool, error) {
func (g *ConfigurationAsCode) Ensure(jenkins *v1alpha1.Jenkins) (bool, error) {
jobsClient := jobs.New(g.jenkinsClient, g.k8sClient, g.logger)
hash := g.calculateHash(secretOrConfigMapData)
done, err := jobsClient.EnsureBuildJob(g.jobName, hash, map[string]string{jobHashParameterName: hash}, jenkins, true)
configuration := &corev1.ConfigMap{}
namespaceName := types.NamespacedName{Namespace: jenkins.Namespace, Name: resources.GetUserConfigurationConfigMapNameFromJenkins(jenkins)}
err := g.k8sClient.Get(context.TODO(), namespaceName, configuration)
if err != nil {
return false, errors.WithStack(err)
}
secret := &corev1.Secret{}
namespaceName = types.NamespacedName{Namespace: jenkins.Namespace, Name: resources.GetUserConfigurationSecretNameFromJenkins(jenkins)}
err = g.k8sClient.Get(context.TODO(), namespaceName, configuration)
if err != nil {
return false, errors.WithStack(err)
}
userConfigurationSecretHash := g.calculateUserConfigurationSecretHash(secret)
userConfigurationHash := g.calculateUserConfigurationHash(configuration)
done, err := jobsClient.EnsureBuildJob(
g.jobName,
userConfigurationSecretHash+userConfigurationHash,
map[string]string{
userConfigurationHashParameterName: userConfigurationHash,
userConfigurationSecretHashParameterName: userConfigurationSecretHash,
},
jenkins,
true)
if err != nil {
return false, err
}
return done, nil
}
func (g *ConfigurationAsCode) calculateHash(secretOrConfigMapData map[string]string) string {
func (g *ConfigurationAsCode) calculateUserConfigurationSecretHash(userConfigurationSecret *corev1.Secret) string {
hash := sha256.New()
var keys []string
for key := range secretOrConfigMapData {
for key := range userConfigurationSecret.Data {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
hash.Write([]byte(key))
hash.Write([]byte(userConfigurationSecret.Data[key]))
}
return base64.StdEncoding.EncodeToString(hash.Sum(nil))
}
func (g *ConfigurationAsCode) calculateUserConfigurationHash(userConfiguration *corev1.ConfigMap) string {
hash := sha256.New()
var keys []string
for key := range userConfiguration.Data {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if strings.HasSuffix(key, ".yaml") {
hash.Write([]byte(key))
hash.Write([]byte(secretOrConfigMapData[key]))
hash.Write([]byte(userConfiguration.Data[key]))
}
}
return base64.StdEncoding.EncodeToString(hash.Sum(nil))
@ -90,9 +134,15 @@ const configurationJobXMLFmt = `<?xml version='1.1' encoding='UTF-8'?>
<hudson.model.ParametersDefinitionProperty>
<parameterDefinitions>
<hudson.model.StringParameterDefinition>
<name>` + jobHashParameterName + `</name>
<description></description>
<defaultValue></defaultValue>
<name>` + userConfigurationSecretHashParameterName + `</name>
<description/>
<defaultValue/>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>` + userConfigurationHashParameterName + `</name>
<description/>
<defaultValue/>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
</parameterDefinitions>
@ -101,28 +151,23 @@ const configurationJobXMLFmt = `<?xml version='1.1' encoding='UTF-8'?>
<definition class="org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition" plugin="workflow-cps@2.61.1">
<script>import io.jenkins.plugins.casc.yaml.YamlSource;
def secretsPath = &apos;%s&apos;
def configsPath = &apos;%s&apos;
def expectedHash = params.hash
def userConfigurationSecretExpectedHash = params.` + userConfigurationSecretHashParameterName + `
def userConfigurationExpectedHash = params.` + userConfigurationHashParameterName + `
node(&apos;master&apos;) {
def secretsText = sh(script: &quot;ls ${secretsPath} | grep .yaml | sort&quot;, returnStdout: true).trim()
def secrets = []
secrets.addAll(secretsText.tokenize(&apos;\n&apos;))
def configsText = sh(script: &quot;ls ${configsPath} | grep .yaml | sort&quot;, returnStdout: true).trim()
def configs = []
configs.addAll(configsText.tokenize(&apos;\n&apos;))
stage(&apos;Synchronizing files&apos;) {
def complete = false
for(int i = 1; i &lt;= 10; i++) {
def actualHash = calculateHash((String[])configs, configsPath)
println &quot;Expected hash &apos;${expectedHash}&apos;, actual hash &apos;${actualHash}&apos;&quot;
if(expectedHash == actualHash) {
complete = true
break
}
sleep 2
}
if(!complete) {
error(&quot;Timeout while synchronizing files&quot;)
}
synchronizeFiles(secretsPath, (String[])secrets, userConfigurationSecretExpectedHash)
synchronizeFiles(configsPath, (String[])configs, userConfigurationExpectedHash)
}
for(config in configs) {
@ -134,6 +179,23 @@ node(&apos;master&apos;) {
}
}
def synchronizeFiles(String path, String[] files, String hash) {
def complete = false
for(int i = 1; i &lt;= 10; i++) {
def actualHash = calculateHash(files, path)
println &quot;Expected hash &apos;${hash}&apos;, actual hash &apos;${actualHash}&apos;, path &apos;${path}&apos;&quot;
if(hash == actualHash) {
complete = true
break
}
sleep 2
}
if(!complete) {
error(&quot;Timeout while synchronizing files&quot;)
}
}
@NonCPS
def calculateHash(String[] configs, String configsPath) {
def hash = java.security.MessageDigest.getInstance(&quot;SHA-256&quot;)

4
pkg/controller/jenkins/configuration/user/reconcile.go
View File

@ -105,12 +105,12 @@ func (r *ReconcileUserConfiguration) ensureUserConfiguration(jenkinsClient jenki
return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10}, nil
}
configurationAsCodeClient := casc.New(jenkinsClient, r.k8sClient, r.logger, constants.UserConfigurationCASCJobName, resources.JenkinsUserConfigurationVolumePath)
configurationAsCodeClient := casc.New(jenkinsClient, r.k8sClient, r.logger, constants.UserConfigurationCASCJobName)
err = configurationAsCodeClient.ConfigureJob()
if err != nil {
return reconcile.Result{}, err
}
done, err = configurationAsCodeClient.Ensure(configuration.Data, r.jenkins)
done, err = configurationAsCodeClient.Ensure(r.jenkins)
if err != nil {
return reconcile.Result{}, err
}

21
test/e2e/configuration_test.go
View File

@ -32,9 +32,11 @@ func TestConfiguration(t *testing.T) {
jenkinsCRName := "e2e"
numberOfExecutors := 6
systemMessage := "Configuration as Code integration works!!!"
systemMessageEnvName := "SYSTEM_MESSAGE"
// base
createUserConfigurationConfigMap(t, jenkinsCRName, namespace, numberOfExecutors, systemMessage)
createUserConfigurationSecret(t, jenkinsCRName, namespace, systemMessageEnvName, systemMessage)
createUserConfigurationConfigMap(t, jenkinsCRName, namespace, numberOfExecutors, fmt.Sprintf("${%s}", systemMessageEnvName))
jenkins := createJenkinsCR(t, jenkinsCRName, namespace)
createDefaultLimitsForContainersInNamespace(t, namespace)
waitForJenkinsBaseConfigurationToComplete(t, jenkins)
@ -49,6 +51,23 @@ func TestConfiguration(t *testing.T) {
verifyUserConfiguration(t, client, numberOfExecutors, systemMessage)
}
func createUserConfigurationSecret(t *testing.T, jenkinsCRName string, namespace string, systemMessageEnvName, systemMessage string) {
userConfiguration := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: resources.GetUserConfigurationSecretName(jenkinsCRName),
Namespace: namespace,
},
StringData: map[string]string{
systemMessageEnvName: systemMessage,
},
}
t.Logf("User configuration secret %+v", *userConfiguration)
if err := framework.Global.Client.Create(context.TODO(), userConfiguration, nil); err != nil {
t.Fatal(err)
}
}
func createUserConfigurationConfigMap(t *testing.T, jenkinsCRName string, namespace string, numberOfExecutors int, systemMessage string) {
userConfiguration := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{