public_git
/
kubernetes-operator
mirror of https://github.com/jenkinsci/kubernetes-operator.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Refactoring, additional tests and update README.md

This commit is contained in:
antoniaklja 2018-12-27 18:17:54 +01:00
parent 50656ceb5b
commit e7c55512a8
10 changed files with 315 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -332,7 +332,7 @@ endif
start-minikube: ## Start minikube
@echo "+ $@"
@minikube status && exit 0 || \
minikube start --kubernetes-version $(MINIKUBE_KUBERNETES_VERSION) --vm-driver=$(MINIKUBE_DRIVER)
minikube start --kubernetes-version $(MINIKUBE_KUBERNETES_VERSION) --vm-driver=$(MINIKUBE_DRIVER) --memory 2048
.PHONY: bump-version
BUMP := patch

41
README.md
View File

@ -76,23 +76,38 @@ Jenkins operator will automatically configure and trigger Seed Job Pipeline for
## TODO
Common:
- simple library for sending Kubernetes events using one common format
- decorate Jenkins API client and add more function for handling jobs e.g. Ensure, CreateOrUpdate
* VirtusLab docker registry (in-progress)
* simple library for sending Kubernetes events using one common format
* decorate Jenkins API client and add more function for handling jobs e.g. Ensure, CreateOrUpdate
* documentation [github pages with Hugo](https://gohugo.io/):
* Installation
* Getting Started
* Plugins
* Seed jobs
* Backup and Restore
* How it works
* architecture
* CR definition
* K8s events
* Debugging
* Base and User configuration
* Contributing
* VirtusLab flavored Jenkins [theme](http://afonsof.com/jenkins-material-theme/)
Base configuration:
- install configuration as a code Jenkins plugin
- handle Jenkins restart when base configuration has changed
- install and configure Kubernetes plugin
- e2e pipelines using Kubernetes plugin
- Jenkins hardening, disable insecure options
* install configuration as a code Jenkins plugin
* handle Jenkins restart when base configuration has changed
* install and configure Kubernetes plugin (in-progress)
* e2e pipelines using Kubernetes plugin
* Jenkins hardening, disable insecure options
User configuration:
- user reconciliation loop with CR validation (work in progress)
- configure seed jobs and deploy keys (work in progress)
- e2e tests for seed jobs (work in progress)
- backup and restore for Jenkins jobs running as standalone job
- trigger backup job before pod deletion using preStop k8s hooks
- verify Jenkins configuration events
* ~~user reconciliation loop with CR validation~~
* ~~configure seed jobs and deploy keys~~
* ~~e2e tests for seed jobs~~
* backup and restore for Jenkins jobs running as standalone job
* trigger backup job before pod deletion using preStop k8s hooks
* verify Jenkins configuration events
[developer_guide]:doc/developer-guide.md
[job-dsl]:https://github.com/jenkinsci/job-dsl-plugin

25
doc/developer-guide.md
View File

@ -1,6 +1,6 @@
# Developer guide
This document explains how to setup your dev environment.
This document explains how to setup your development environment.
## Prerequisites
@ -28,7 +28,7 @@ Build and run `jenkins-operator` locally:
make build && make docker-build && make minikube-run EXTRA_ARGS='--minikube --local'
```
Once minikube and `jenkins-operator` are up and running, apply CR file:
Once `minikube` and `jenkins-operator` are up and running, apply CR file:
```bash
kubectl apply -f jenkins-operator/deploy/crds/virtuslab_v1alpha1_jenkins_cr.yaml
@ -44,7 +44,7 @@ Run unit tests:
make test
```
Run e2e tests with minikube:
Run e2e tests with `minikube`:
```bash
make minikube-run
@ -64,7 +64,6 @@ Run:
make deepcopy-gen
```
### Getting Jenkins URL and basic credentials
```bash
@ -72,24 +71,6 @@ minikube service jenkins-operator-example --url
kubectl get secret jenkins-operator-credentials-example -o yaml
```
### Install custom plugins
Extend `initBashTemplate` in `jenkins-operator/pkg/controller/jenkins/configuration/base/resources/scripts_configmap.go`:
```
touch {{ .JenkinsHomePath }}/plugins.txt
cat > {{ .JenkinsHomePath }}/plugins.txt <<EOL
credentials:2.1.18
ssh-credentials:1.14
job-dsl:1.70
git:3.9.1
mask-passwords:2.12.0
workflow-cps:2.61
workflow-job:2.30
workflow-aggregator:2.6
EOL
```
[dep_tool]:https://golang.github.io/dep/docs/installation.html
[git_tool]:https://git-scm.com/downloads

30
pkg/controller/jenkins/configuration/user/validate.go
View File

@ -8,13 +8,14 @@ import (
virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
"github.com/VirtusLab/jenkins-operator/pkg/log"
"crypto/x509"
"encoding/pem"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
k8s "sigs.k8s.io/controller-runtime/pkg/client"
)
// Validate validates Jenkins CR Spec section
func (r *ReconcileUserConfiguration) Validate(k8sClient k8s.Client, jenkins *virtuslabv1alpha1.Jenkins) bool {
func (r *ReconcileUserConfiguration) Validate(jenkins *virtuslabv1alpha1.Jenkins) bool {
// validate jenkins.Spec.SeedJobs
if jenkins.Spec.SeedJobs != nil {
for _, seedJob := range jenkins.Spec.SeedJobs {
@ -38,7 +39,7 @@ func (r *ReconcileUserConfiguration) Validate(k8sClient k8s.Client, jenkins *vir
if seedJob.PrivateKey.SecretKeyRef != nil {
deployKeySecret := &v1.Secret{}
namespaceName := types.NamespacedName{Namespace: jenkins.Namespace, Name: seedJob.PrivateKey.SecretKeyRef.Name}
err := k8sClient.Get(context.TODO(), namespaceName, deployKeySecret)
err := r.k8sClient.Get(context.TODO(), namespaceName, deployKeySecret)
//TODO(bantoniak) handle error properly
if err != nil {
logger.Info("couldn't read private key secret")
@ -51,14 +52,31 @@ func (r *ReconcileUserConfiguration) Validate(k8sClient k8s.Client, jenkins *vir
return false
}
//TODO(bantoniak) load private key to validate it
if !strings.HasPrefix(privateKey, "-----BEGIN RSA PRIVATE KEY-----") {
logger.Info("private key has wrong prefix")
if !validatePrivateKey(privateKey) {
logger.Info("private key is invalid")
return false
}
}
}
}
return true
}
func validatePrivateKey(privateKey string) bool {
block, _ := pem.Decode([]byte(privateKey))
if block == nil {
return false
}
priv, err := x509.ParsePKCS1PrivateKey(block.Bytes)
if err != nil {
return false
}
err = priv.Validate()
if err != nil {
return false
}
return true
}

233
pkg/controller/jenkins/configuration/user/validate_test.go Normal file
View File

@ -0,0 +1,233 @@
package user
import (
"context"
"fmt"
"testing"
virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
)
func TestValidateUserConfiguration(t *testing.T) {
data := []struct {
description string
jenkins *virtuslabv1alpha1.Jenkins
secret *corev1.Secret
expectedResult bool
}{
{
description: "Valid with public repository and without private key",
jenkins: &virtuslabv1alpha1.Jenkins{
Spec: virtuslabv1alpha1.JenkinsSpec{
SeedJobs: []virtuslabv1alpha1.SeedJob{
{
ID: "jenkins-operator-e2e",
Targets: "cicd/jobs/*.jenkins",
Description: "Jenkins Operator e2e tests repository",
RepositoryBranch: "master",
RepositoryURL: "https://github.com/VirtusLab/jenkins-operator-e2e.git",
},
},
},
},
expectedResult: true,
},
{
description: "Invalid without id",
jenkins: &virtuslabv1alpha1.Jenkins{
Spec: virtuslabv1alpha1.JenkinsSpec{
SeedJobs: []virtuslabv1alpha1.SeedJob{
{
Targets: "cicd/jobs/*.jenkins",
Description: "Jenkins Operator e2e tests repository",
RepositoryBranch: "master",
RepositoryURL: "https://github.com/VirtusLab/jenkins-operator-e2e.git",
},
},
},
},
expectedResult: false,
},
{
description: "Valid with private key and secret",
jenkins: &virtuslabv1alpha1.Jenkins{
Spec: virtuslabv1alpha1.JenkinsSpec{
SeedJobs: []virtuslabv1alpha1.SeedJob{
{
ID: "jenkins-operator-e2e",
Targets: "cicd/jobs/*.jenkins",
Description: "Jenkins Operator e2e tests repository",
RepositoryBranch: "master",
RepositoryURL: "https://github.com/VirtusLab/jenkins-operator-e2e.git",
PrivateKey: virtuslabv1alpha1.PrivateKey{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: "deploy-keys",
},
Key: "jenkins-operator-e2e",
},
},
},
},
},
},
secret: &corev1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "deploy-keys",
Namespace: "default",
},
Data: map[string][]byte{
"jenkins-operator-e2e": []byte(fakePrivateKey),
},
},
expectedResult: true,
},
{
description: "Invalid private key in secret",
jenkins: &virtuslabv1alpha1.Jenkins{
Spec: virtuslabv1alpha1.JenkinsSpec{
SeedJobs: []virtuslabv1alpha1.SeedJob{
{
ID: "jenkins-operator-e2e",
Targets: "cicd/jobs/*.jenkins",
Description: "Jenkins Operator e2e tests repository",
RepositoryBranch: "master",
RepositoryURL: "https://github.com/VirtusLab/jenkins-operator-e2e.git",
PrivateKey: virtuslabv1alpha1.PrivateKey{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: "deploy-keys",
},
Key: "jenkins-operator-e2e",
},
},
},
},
},
},
secret: &corev1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "deploy-keys",
Namespace: "default",
},
Data: map[string][]byte{
"jenkins-operator-e2e": []byte(fakeInvalidPrivateKey),
},
},
expectedResult: false,
},
{
description: "Invalid with PrivateKey and empty Secret data",
jenkins: &virtuslabv1alpha1.Jenkins{
Spec: virtuslabv1alpha1.JenkinsSpec{
SeedJobs: []virtuslabv1alpha1.SeedJob{
{
ID: "jenkins-operator-e2e",
Targets: "cicd/jobs/*.jenkins",
Description: "Jenkins Operator e2e tests repository",
RepositoryBranch: "master",
RepositoryURL: "https://github.com/VirtusLab/jenkins-operator-e2e.git",
PrivateKey: virtuslabv1alpha1.PrivateKey{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: "deploy-keys",
},
Key: "jenkins-operator-e2e",
},
},
},
},
},
},
secret: &corev1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "deploy-keys",
Namespace: "default",
},
Data: map[string][]byte{
"jenkins-operator-e2e": []byte(""),
},
},
expectedResult: false,
},
{
description: "Invalid with ssh RepositoryURL and empty PrivateKey",
jenkins: &virtuslabv1alpha1.Jenkins{
Spec: virtuslabv1alpha1.JenkinsSpec{
SeedJobs: []virtuslabv1alpha1.SeedJob{
{
ID: "jenkins-operator-e2e",
Targets: "cicd/jobs/*.jenkins",
Description: "Jenkins Operator e2e tests repository",
RepositoryBranch: "master",
RepositoryURL: "git@github.com:VirtusLab/jenkins-operator.git",
},
},
},
},
expectedResult: false,
},
}
for _, testingData := range data {
t.Run(fmt.Sprintf("Testing '%s'", testingData.description), func(t *testing.T) {
fakeClient := fake.NewFakeClient()
fakeClient.Create(context.TODO(), testingData.secret)
userReconcileLoop := New(fakeClient, nil, logf.ZapLogger(false), nil)
result := userReconcileLoop.Validate(testingData.jenkins)
assert.Equal(t, testingData.expectedResult, result)
})
}
}
var fakePrivateKey = `-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEArK4ld6i2iqW6L3jaTZaKD/v7PjDn+Ik9MXp+kvLcUw/+wEGm
285UwqLnDDlBhSi9nDgJ+m1XU87VCpz/DXW23R/CQcMX2xunib4wWLQqoR3CWbk3
SwiLd8TWAvXkxdXm8fDOGAZbYK2alMV+M+9E2OpZsBUCxmb/3FAofF6JccKoJOH8
UveRNSOx7IXPKtHFiypBhWM4l6ZjgJKm+DRIEhyvoC+pHzcum2ZEPOv+ZJDy5jXK
ZHcNQXVnAZtCcojcjVUBw2rZms+fQ6Volv2JT71Gpykzx/rChhwNwxdAEwjLjKjL
nBWEh/WxsS3NbM7zb4B2XGMCeWVeb/niUwpy+wIDAQABAoIBAQCjGkJNidARmYQI
/u/DxWNWwb2H+o3BFW/1YixYBIjS9BK96cT/bR5mUZRG2XXnnpmqCsxx/AE2KfDU
e4H1ZrB4oFzN3MaVsMNIuZnUzyhM0l0WfnmZp9KEKCm01ilmLCpdcARacPaylIej
6f7QcznmYUShqtbaK8OUhyoWfvz3s0VLkpBlqm63uPtjAx6sAl399THxHVwbYgYy
TxPY8wdjOvNzQJ7ColUh05Zq6TsCGGFUFg7v4to/AXtDhcTMVONlapP+XxekRx8P
98BepIgzgvQhWak8gm+cKQYANk14Q8BDzUCDplYuIZVvKl+/ZHltjHGjrqxDrcDA
0U7REgtxAoGBAN+LAEf2o14ffs/ebVSxiv7LnuAxFh2L6i7RqtehpSf7BnYC65vB
6TMsc/0/KFkD5Az7nrJmA7HmM8J/NI2ks0Mbft+0XCRFx/zfU6pOvPinRKp/8Vtm
aUmNzhz8UMaQ1JXOvBOqvXKWYrN1WPha1+/BnUQrpTdhGxAoAh1FW4eHAoGBAMXA
mXTN5X8+mp9KW2bIpFsjrZ+EyhxO6a6oBMZY54rceeOzf5RcXY7EOiTrnmr+lQvp
fAKBeX5V8G96nSEIDmPhKGZ1C1vEP6hRWahJo1XkN5E1j6hRHCu3DQLtL2lxlyfG
Fx11fysgmLoPVVytLAEQwt4WxMp7OsM1NWqB+u3tAoGBAILUg3Gas7pejIV0FGDB
GCxPV8i2cc8RGBoWs/pHrLVdgUaIJwSd1LISjj/lOuP+FvZSPWsDsZ3osNpgQI21
mwTnjrW2hUblYEprGjhOpOKSYum2v7dSlMRrrfng4hWUphaXTBPmlcH+qf2F7HBO
GptDoZtIQAXNW111TOd8tDj5AoGAC1PO9nvcy38giENQHQEdOQNALMUEdr6mcBS7
wUjSaofai4p6olrwGP9wfTDp8CMJEpebPOGBvhTaIuiZG41ElcAN+mB1+Bmzs8aF
JjihnIfoDu9MfU24GWDw49wGPTn+eI7GQC+8yxGg7fd24kohHSaCowoW16pbYVco
6iLr5rkCgYBt0bcYJ3AOTH0UXS8kvJvnyce/RBIAMoUABwvdkZt9r5B4UzsoLq5e
WrrU6fSRsE6lSsBd83pOAQ46tv+vntQ+0EihD9/0INhkQM99lBw1TFdFTgGSAs1e
ns4JGP6f5uIuwqu/nbqPqMyDovjkGbX2znuGBcvki90Pi97XL7MMWw==
-----END RSA PRIVATE KEY-----
`
var fakeInvalidPrivateKey = `-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEArK4ld6i2iqW6L3jaTZaKD/v7PjDn+Ik9MXp+kvLcUw/+wEGm
285UwqLnDDlBhSi9nDgJ+m1XU87VCpz/DXW23R/CQcMX2xunib4wWLQqoR3CWbk3
SwiLd8TWAvXkxdXm8fDOGAZbYK2alMV+M+9E2OpZsBUCxmb/3FAofF6JccKoJOH8
`

2
pkg/controller/jenkins/jenkins_controller.go
View File

@ -125,7 +125,7 @@ func (r *ReconcileJenkins) Reconcile(request reconcile.Request) (reconcile.Resul
// Reconcile user configuration
userConfiguration := user.New(r.client, jenkinsClient, logger, jenkins)
if !userConfiguration.Validate(r.client, jenkins) {
if !userConfiguration.Validate(jenkins) {
logger.V(log.VWarn).Info("Please correct Jenkins CR")
return reconcile.Result{}, nil // don't requeue
}

8
test/e2e/base_configuration_test.go
View File

@ -45,7 +45,7 @@ func verifyJenkinsMasterPodAttributes(t *testing.T, jenkins *virtuslabv1alpha1.J
}
func verifyBasePlugins(t *testing.T, jenkinsClient *gojenkins.Jenkins) {
allPluginsInJenkins, err := jenkinsClient.GetPlugins(1)
installedPlugins, err := jenkinsClient.GetPlugins(1)
if err != nil {
t.Fatal(err)
}
@ -55,11 +55,11 @@ func verifyBasePlugins(t *testing.T, jenkinsClient *gojenkins.Jenkins) {
if err != nil {
t.Fatal(err)
}
if found, ok := isPluginValid(t, allPluginsInJenkins, *rootPlugin); !ok {
if found, ok := isPluginValid(installedPlugins, *rootPlugin); !ok {
t.Fatalf("Invalid plugin '%s', actual '%+v'", rootPlugin, found)
}
for _, requiredPlugin := range p {
if found, ok := isPluginValid(t, allPluginsInJenkins, requiredPlugin); !ok {
if found, ok := isPluginValid(installedPlugins, requiredPlugin); !ok {
t.Fatalf("Invalid plugin '%s', actual '%+v'", requiredPlugin, found)
}
}
@ -68,7 +68,7 @@ func verifyBasePlugins(t *testing.T, jenkinsClient *gojenkins.Jenkins) {
t.Log("Base plugins have been installed")
}
func isPluginValid(t *testing.T, plugins *gojenkins.Plugins, requiredPlugin plugin.Plugin) (*gojenkins.Plugin, bool) {
func isPluginValid(plugins *gojenkins.Plugins, requiredPlugin plugin.Plugin) (*gojenkins.Plugin, bool) {
p := plugins.Contains(requiredPlugin.Name)
if p == nil {
return p, false

8
test/e2e/jenkins.go
View File

@ -79,8 +79,8 @@ func createJenkinsCR(t *testing.T, namespace string) *virtuslabv1alpha1.Jenkins
Annotations: map[string]string{"test": "label"},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
corev1.ResourceCPU: resource.MustParse("300m"),
corev1.ResourceMemory: resource.MustParse("500Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),
@ -111,8 +111,8 @@ func createJenkinsCRWithSeedJob(t *testing.T, namespace string) *virtuslabv1alph
Annotations: map[string]string{"test": "label"},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
corev1.ResourceCPU: resource.MustParse("300m"),
corev1.ResourceMemory: resource.MustParse("500Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),

20
test/e2e/user_configuration_test.go
View File

@ -17,6 +17,7 @@ func TestUserConfiguration(t *testing.T) {
defer ctx.Cleanup()
jenkins := createJenkinsCRWithSeedJob(t, namespace)
waitForJenkinsBaseConfigurationToComplete(t, jenkins)
waitForJenkinsUserConfigurationToComplete(t, jenkins)
client := verifyJenkinsAPIConnection(t, jenkins)
verifyJenkinsSeedJobs(t, client)
@ -25,7 +26,7 @@ func TestUserConfiguration(t *testing.T) {
func verifyJenkinsSeedJobs(t *testing.T, client *gojenkins.Jenkins) {
// check if job has been configured and executed successfully
err := wait.Poll(time.Second*10, time.Minute*2, func() (bool, error) {
t.Logf("Attempting to get seed job status '%v'", seedjobs.ConfigureSeedJobsName)
t.Logf("Attempting to get configure seed job status '%v'", seedjobs.ConfigureSeedJobsName)
seedJob, err := client.GetJob(seedjobs.ConfigureSeedJobsName)
if err != nil || seedJob == nil {
return false, nil
@ -37,9 +38,20 @@ func verifyJenkinsSeedJobs(t *testing.T, client *gojenkins.Jenkins) {
return true, nil
})
if err != nil {
t.Fatalf("couldn't get seed job '%v'", err)
t.Fatalf("couldn't get configure seed job '%v'", err)
}
//TODO(bantoniak) verify if seed jobs have been created
//TODO(bantoniak) verify if jobs created by seed jobs have been created
// WARNING this use case depends on changes in https://github.com/VirtusLab/jenkins-operator-e2e/tree/master/cicd
seedJobName := "jenkins-operator-e2e-job-dsl-seed" // https://github.com/VirtusLab/jenkins-operator-e2e/blob/master/cicd/jobs/e2e_test_job.jenkins
err = wait.Poll(time.Second*10, time.Minute*2, func() (bool, error) {
t.Logf("Attempting to verify if seed job has been created '%v'", seedJobName)
seedJob, err := client.GetJob(seedJobName)
if err != nil || seedJob == nil {
return false, nil
}
return true, nil
})
if err != nil {
t.Fatalf("couldn't verify if seed job has been created '%v'", err)
}
}

2
test/e2e/wait.go
View File

@ -37,7 +37,7 @@ func waitForJenkinsBaseConfigurationToComplete(t *testing.T, jenkins *virtuslabv
}
func waitForRecreateJenkinsMasterPod(t *testing.T, jenkins *virtuslabv1alpha1.Jenkins) {
err := wait.Poll(retryInterval, 20*retryInterval, func() (bool, error) {
err := wait.Poll(retryInterval, 30*retryInterval, func() (bool, error) {
lo := metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(resources.BuildResourceLabels(jenkins)).String(),
}