From 68ffc69d418a0589e1712f6595d35ad87d98e6a8 Mon Sep 17 00:00:00 2001 From: Oleksandr Kovalchuk Date: Tue, 7 Apr 2020 13:36:52 +0300 Subject: [PATCH 1/4] Add possibility to set Jenkins master pod's priority Allow to set priorityClassName for master pod. This will make possible for jenkins master to either not be preempted on other's pod low resources or to preempt lesser priority pods on resources shortage. Ref: https://github.com/jenkinsci/kubernetes-operator/issues/321 --- pkg/apis/jenkins/v1alpha2/jenkins_types.go | 4 ++++ pkg/controller/jenkins/configuration/base/reconcile.go | 6 ++++++ pkg/controller/jenkins/configuration/base/resources/pod.go | 1 + 3 files changed, 11 insertions(+) diff --git a/pkg/apis/jenkins/v1alpha2/jenkins_types.go b/pkg/apis/jenkins/v1alpha2/jenkins_types.go index 1eb665d7..1e9eca80 100644 --- a/pkg/apis/jenkins/v1alpha2/jenkins_types.go +++ b/pkg/apis/jenkins/v1alpha2/jenkins_types.go @@ -374,6 +374,10 @@ type JenkinsMaster struct { // DisableCSRFProtection allows you to toggle CSRF Protection on Jenkins DisableCSRFProtection bool `json:"disableCSRFProtection"` + + // PriorityClassName for Jenkins master pod + // +optional + PriorityClassName string `json:"priorityClassName,omitempty"` } // Service defines Kubernetes service attributes diff --git a/pkg/controller/jenkins/configuration/base/reconcile.go b/pkg/controller/jenkins/configuration/base/reconcile.go index 64cada6a..420eb273 100644 --- a/pkg/controller/jenkins/configuration/base/reconcile.go +++ b/pkg/controller/jenkins/configuration/base/reconcile.go @@ -648,6 +648,12 @@ func (r *ReconcileJenkinsBaseConfiguration) checkForPodRecreation(currentJenkins len(currentJenkinsMasterPod.Spec.Containers), len(r.Configuration.Jenkins.Spec.Master.Containers))) } + if r.Configuration.Jenkins.Spec.Master.PriorityClassName != currentJenkinsMasterPod.Spec.PriorityClassName { + messages = append(messages, "Jenkins priorityClassName has changed") + verbose = append(verbose, fmt.Sprintf("Jenkins priorityClassName has changed, actual '%+v' required '%+v'", + currentJenkinsMasterPod.Spec.PriorityClassName, r.Configuration.Jenkins.Spec.Master.PriorityClassName)) + } + customResourceReplaced := (r.Configuration.Jenkins.Status.BaseConfigurationCompletedTime == nil || r.Configuration.Jenkins.Status.UserConfigurationCompletedTime == nil) && r.Configuration.Jenkins.Status.UserAndPasswordHash == "" diff --git a/pkg/controller/jenkins/configuration/base/resources/pod.go b/pkg/controller/jenkins/configuration/base/resources/pod.go index 22585cfd..dde4609b 100644 --- a/pkg/controller/jenkins/configuration/base/resources/pod.go +++ b/pkg/controller/jenkins/configuration/base/resources/pod.go @@ -328,6 +328,7 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha2.Jenkins SecurityContext: jenkins.Spec.Master.SecurityContext, ImagePullSecrets: jenkins.Spec.Master.ImagePullSecrets, Tolerations: jenkins.Spec.Master.Tolerations, + PriorityClassName: jenkins.Spec.Master.PriorityClassName, }, } } From b99468930e99166ad2bca6c7d18e9b90684d5521 Mon Sep 17 00:00:00 2001 From: Oleksandr Kovalchuk Date: Wed, 8 Apr 2020 20:37:38 +0300 Subject: [PATCH 2/4] Add e2e test with setting priorityClassName As it is quite difficult to set cluster-wide objects in test and don't break other tests, which are running in parallel, rely on pre-created priorityClass (which is already in the cluster by default) in test. --- test/e2e/configuration_test.go | 23 +++++++++++++++++++++-- test/e2e/jenkins.go | 5 +++-- test/e2e/restart_test.go | 4 ++-- test/e2e/seedjobs_test.go | 2 +- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/test/e2e/configuration_test.go b/test/e2e/configuration_test.go index 931f875d..184a3dd8 100644 --- a/test/e2e/configuration_test.go +++ b/test/e2e/configuration_test.go @@ -33,6 +33,7 @@ func TestConfiguration(t *testing.T) { numberOfExecutorsEnvName := "NUMBER_OF_EXECUTORS" systemMessage := "Configuration as Code integration works!!!" systemMessageEnvName := "SYSTEM_MESSAGE" + priorityClassName := "" mySeedJob := seedJobConfig{ SeedJob: v1alpha2.SeedJob{ ID: "jenkins-operator", @@ -84,7 +85,7 @@ func TestConfiguration(t *testing.T) { // base createUserConfigurationSecret(t, namespace, stringData) createUserConfigurationConfigMap(t, namespace, numberOfExecutorsEnvName, fmt.Sprintf("${%s}", systemMessageEnvName)) - jenkins := createJenkinsCR(t, jenkinsCRName, namespace, &[]v1alpha2.SeedJob{mySeedJob.SeedJob}, groovyScripts, casc) + jenkins := createJenkinsCR(t, jenkinsCRName, namespace, &[]v1alpha2.SeedJob{mySeedJob.SeedJob}, groovyScripts, casc, priorityClassName) createDefaultLimitsForContainersInNamespace(t, namespace) createKubernetesCredentialsProviderSecret(t, namespace, mySeedJob) waitForJenkinsBaseConfigurationToComplete(t, jenkins) @@ -107,6 +108,7 @@ func TestPlugins(t *testing.T) { jobID := "k8s-e2e" + priorityClassName := "" seedJobs := &[]v1alpha2.SeedJob{ { ID: "jenkins-operator", @@ -119,7 +121,7 @@ func TestPlugins(t *testing.T) { }, } - jenkins := createJenkinsCR(t, "k8s-e2e", namespace, seedJobs, v1alpha2.GroovyScripts{}, v1alpha2.ConfigurationAsCode{}) + jenkins := createJenkinsCR(t, "k8s-e2e", namespace, seedJobs, v1alpha2.GroovyScripts{}, v1alpha2.ConfigurationAsCode{}, priorityClassName) waitForJenkinsUserConfigurationToComplete(t, jenkins) jenkinsClient, cleanUpFunc := verifyJenkinsAPIConnection(t, jenkins, namespace) @@ -140,6 +142,22 @@ func TestPlugins(t *testing.T) { assert.True(t, build.IsGood()) } +func TestPriorityClassNameSetExisting(t *testing.T) { + t.Parallel() + namespace, ctx := setupTest(t) + defer showLogsAndCleanup(t, ctx) + + jenkinsCRName := "k8s-ete-priority-class-existing" + // One of the existing priority classes + priorityClassName := "system-cluster-critical" + + jenkins := createJenkinsCR(t, jenkinsCRName, namespace, nil, v1alpha2.GroovyScripts{}, v1alpha2.ConfigurationAsCode{}, priorityClassName) + waitForJenkinsBaseConfigurationToComplete(t, jenkins) + verifyJenkinsMasterPodAttributes(t, jenkins) + _, cleanUpFunc := verifyJenkinsAPIConnection(t, jenkins, namespace) + defer cleanUpFunc() +} + func createUserConfigurationSecret(t *testing.T, namespace string, stringData map[string]string) { userConfiguration := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -226,6 +244,7 @@ func verifyJenkinsMasterPodAttributes(t *testing.T, jenkins *v1alpha2.Jenkins) { assert.Equal(t, jenkins.Spec.Master.Containers[0].Command, jenkinsPod.Spec.Containers[0].Command) assert.Equal(t, resources.GetJenkinsMasterPodLabels(*jenkins), jenkinsPod.Labels) + assert.Equal(t, jenkins.Spec.Master.PriorityClassName, jenkinsPod.Spec.PriorityClassName) for _, actualContainer := range jenkinsPod.Spec.Containers { if actualContainer.Name == resources.JenkinsMasterContainerName { diff --git a/test/e2e/jenkins.go b/test/e2e/jenkins.go index f93b8848..e5e41ad2 100644 --- a/test/e2e/jenkins.go +++ b/test/e2e/jenkins.go @@ -51,7 +51,7 @@ func getJenkinsMasterPod(t *testing.T, jenkins *v1alpha2.Jenkins) *corev1.Pod { return &podList.Items[0] } -func createJenkinsCR(t *testing.T, name, namespace string, seedJob *[]v1alpha2.SeedJob, groovyScripts v1alpha2.GroovyScripts, casc v1alpha2.ConfigurationAsCode) *v1alpha2.Jenkins { +func createJenkinsCR(t *testing.T, name, namespace string, seedJob *[]v1alpha2.SeedJob, groovyScripts v1alpha2.GroovyScripts, casc v1alpha2.ConfigurationAsCode, priorityClassName string) *v1alpha2.Jenkins { var seedJobs []v1alpha2.SeedJob if seedJob != nil { seedJobs = append(seedJobs, *seedJob...) @@ -119,7 +119,8 @@ func createJenkinsCR(t *testing.T, name, namespace string, seedJob *[]v1alpha2.S {Name: "github", Version: "1.29.4"}, {Name: "devoptics", Version: "1.1863", DownloadURL: "https://jenkins-updates.cloudbees.com/download/plugins/devoptics/1.1863/devoptics.hpi"}, }, - NodeSelector: map[string]string{"kubernetes.io/os": "linux"}, + PriorityClassName: priorityClassName, + NodeSelector: map[string]string{"kubernetes.io/os": "linux"}, Volumes: []corev1.Volume{ { Name: "plugins-cache", diff --git a/test/e2e/restart_test.go b/test/e2e/restart_test.go index f2181d06..001611b5 100644 --- a/test/e2e/restart_test.go +++ b/test/e2e/restart_test.go @@ -20,7 +20,7 @@ func TestJenkinsMasterPodRestart(t *testing.T) { defer showLogsAndCleanup(t, ctx) - jenkins := createJenkinsCR(t, "e2e", namespace, nil, v1alpha2.GroovyScripts{}, v1alpha2.ConfigurationAsCode{}) + jenkins := createJenkinsCR(t, "e2e", namespace, nil, v1alpha2.GroovyScripts{}, v1alpha2.ConfigurationAsCode{}, "") waitForJenkinsBaseConfigurationToComplete(t, jenkins) restartJenkinsMasterPod(t, jenkins) waitForRecreateJenkinsMasterPod(t, jenkins) @@ -48,7 +48,7 @@ func TestSafeRestart(t *testing.T) { }, }, } - jenkins := createJenkinsCR(t, jenkinsCRName, namespace, nil, groovyScriptsConfig, v1alpha2.ConfigurationAsCode{}) + jenkins := createJenkinsCR(t, jenkinsCRName, namespace, nil, groovyScriptsConfig, v1alpha2.ConfigurationAsCode{}, "") waitForJenkinsBaseConfigurationToComplete(t, jenkins) waitForJenkinsUserConfigurationToComplete(t, jenkins) jenkinsClient, cleanUpFunc := verifyJenkinsAPIConnection(t, jenkins, namespace) diff --git a/test/e2e/seedjobs_test.go b/test/e2e/seedjobs_test.go index 4ac6e45f..e4dfbc23 100644 --- a/test/e2e/seedjobs_test.go +++ b/test/e2e/seedjobs_test.go @@ -54,7 +54,7 @@ func TestSeedJobs(t *testing.T) { createKubernetesCredentialsProviderSecret(t, namespace, seedJobConfig) seedJobs = append(seedJobs, seedJobConfig.SeedJob) } - jenkins := createJenkinsCR(t, jenkinsCRName, namespace, &seedJobs, v1alpha2.GroovyScripts{}, v1alpha2.ConfigurationAsCode{}) + jenkins := createJenkinsCR(t, jenkinsCRName, namespace, &seedJobs, v1alpha2.GroovyScripts{}, v1alpha2.ConfigurationAsCode{}, "") waitForJenkinsBaseConfigurationToComplete(t, jenkins) verifyJenkinsMasterPodAttributes(t, jenkins) From 7c5f41fde56d6e20ffbfde064351237d430eae01 Mon Sep 17 00:00:00 2001 From: Oleksandr Kovalchuk Date: Sat, 11 Apr 2020 13:38:25 +0300 Subject: [PATCH 3/4] Bump kubernetes version --- config.minikube.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.minikube.env b/config.minikube.env index 0e0d230f..309181c9 100644 --- a/config.minikube.env +++ b/config.minikube.env @@ -1,10 +1,10 @@ KUBERNETES_PROVIDER=minikube -MINIKUBE_KUBERNETES_VERSION=v1.16.0 +MINIKUBE_KUBERNETES_VERSION=v1.17.4 MINIKUBE_DRIVER=virtualbox MINIKUBE_VERSION=1.4.0 KUBECTL_CONTEXT=minikube JENKINS_API_HOSTNAME_COMMAND=minikube ip JENKINS_API_PORT=0 -JENKINS_API_USE_NODEPORT=true \ No newline at end of file +JENKINS_API_USE_NODEPORT=true From c90969256eab00da0030fc92992a19cae99b871f Mon Sep 17 00:00:00 2001 From: Oleksandr Kovalchuk Date: Sat, 11 Apr 2020 15:06:04 +0300 Subject: [PATCH 4/4] Use existing version of envoyproxy image --- test/e2e/jenkins.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/jenkins.go b/test/e2e/jenkins.go index e5e41ad2..28cf81d8 100644 --- a/test/e2e/jenkins.go +++ b/test/e2e/jenkins.go @@ -110,7 +110,7 @@ func createJenkinsCR(t *testing.T, name, namespace string, seedJob *[]v1alpha2.S }, { Name: "envoyproxy", - Image: "envoyproxy/envoy-alpine", + Image: "envoyproxy/envoy-alpine:v1.14.1", }, }, Plugins: []v1alpha2.Plugin{