From 7d48aa30793df98f128b34f3a15d2b8f1a9c517b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20S=C4=99k?= <31410181+tomaszsek@users.noreply.github.com> Date: Sun, 29 Mar 2020 18:40:07 +0200 Subject: [PATCH] #190 Allow set quay.io/openshift/origin-jenkins with OAuth image in e2e tests (#307) --- go.mod | 4 + test/e2e/jenkins.go | 93 +++++++++++------- test/e2e/{cr.go => mode_kubernetes.go} | 4 + .../{cr_openshift.go => mode_openshift.go} | 4 + test/e2e/mode_openshift_oauth.go | 95 +++++++++++++++++++ test/e2e/restart_test.go | 3 + test/e2e/restorebackup_test.go | 1 + test/e2e/wait.go | 4 +- 8 files changed, 174 insertions(+), 34 deletions(-) rename test/e2e/{cr.go => mode_kubernetes.go} (70%) rename test/e2e/{cr_openshift.go => mode_openshift.go} (92%) create mode 100644 test/e2e/mode_openshift_oauth.go diff --git a/go.mod b/go.mod index ee21ded6..6284f174 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 // indirect github.com/emersion/go-sasl v0.0.0-20190817083125-240c8404624e // indirect github.com/emersion/go-smtp v0.11.2 + github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-logr/logr v0.1.0 github.com/go-logr/zapr v0.1.1 github.com/go-openapi/spec v0.19.4 @@ -16,8 +17,10 @@ require ( github.com/golang/mock v1.3.1 github.com/mailgun/mailgun-go/v3 v3.6.0 github.com/operator-framework/operator-sdk v0.15.1 + github.com/pborman/uuid v1.2.0 github.com/pkg/errors v0.8.1 github.com/robfig/cron v1.2.0 + github.com/sirupsen/logrus v1.4.2 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.4.0 go.uber.org/zap v1.10.0 @@ -26,6 +29,7 @@ require ( gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df k8s.io/api v0.0.0 + k8s.io/apiextensions-apiserver v0.0.0 k8s.io/apimachinery v0.0.0 k8s.io/cli-runtime v0.0.0 k8s.io/client-go v12.0.0+incompatible diff --git a/test/e2e/jenkins.go b/test/e2e/jenkins.go index d9f83a5e..daa69554 100644 --- a/test/e2e/jenkins.go +++ b/test/e2e/jenkins.go @@ -6,15 +6,20 @@ import ( "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2" jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base" "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources" "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants" + framework "github.com/operator-framework/operator-sdk/pkg/test" + "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" ) const ( @@ -46,37 +51,6 @@ func getJenkinsMasterPod(t *testing.T, jenkins *v1alpha2.Jenkins) *corev1.Pod { return &podList.Items[0] } -func createJenkinsAPIClient(jenkins *v1alpha2.Jenkins, hostname string, port int) (jenkinsclient.Jenkins, error) { - adminSecret := &corev1.Secret{} - namespaceName := types.NamespacedName{Namespace: jenkins.Namespace, Name: resources.GetOperatorCredentialsSecretName(jenkins)} - if err := framework.Global.Client.Get(context.TODO(), namespaceName, adminSecret); err != nil { - return nil, err - } - - var service corev1.Service - - err := framework.Global.Client.Get(context.TODO(), types.NamespacedName{ - Namespace: jenkins.Namespace, - Name: resources.GetJenkinsHTTPServiceName(jenkins), - }, &service) - - if err != nil { - return nil, err - } - - jenkinsAPIURL := jenkinsclient.JenkinsAPIConnectionSettings{ - Hostname: hostname, - Port: port, - UseNodePort: false, - }.BuildJenkinsAPIUrl(service.Name, service.Namespace, service.Spec.Ports[0].Port, service.Spec.Ports[0].NodePort) - - return jenkinsclient.NewUserAndPasswordAuthorization( - jenkinsAPIURL, - string(adminSecret.Data[resources.OperatorCredentialsSecretUserNameKey]), - string(adminSecret.Data[resources.OperatorCredentialsSecretTokenKey]), - ) -} - func createJenkinsCR(t *testing.T, name, namespace string, seedJob *[]v1alpha2.SeedJob, groovyScripts v1alpha2.GroovyScripts, casc v1alpha2.ConfigurationAsCode) *v1alpha2.Jenkins { var seedJobs []v1alpha2.SeedJob if seedJob != nil { @@ -178,7 +152,49 @@ func createJenkinsCR(t *testing.T, name, namespace string, seedJob *[]v1alpha2.S return jenkins } +func createJenkinsAPIClientFromServiceAccount(t *testing.T, jenkins *v1alpha2.Jenkins, jenkinsAPIURL string) (jenkinsclient.Jenkins, error) { + t.Log("Creating Jenkins API client from service account") + podName := resources.GetJenkinsMasterPodName(*jenkins) + + clientSet, err := kubernetes.NewForConfig(framework.Global.KubeConfig) + if err != nil { + return nil, err + } + config := configuration.Configuration{Jenkins: jenkins, ClientSet: *clientSet, Config: framework.Global.KubeConfig} + r := base.New(config, nil, jenkinsclient.JenkinsAPIConnectionSettings{}) + + token, _, err := r.Configuration.Exec(podName, resources.JenkinsMasterContainerName, []string{"cat", "/var/run/secrets/kubernetes.io/serviceaccount/token"}) + if err != nil { + return nil, err + } + + return jenkinsclient.NewBearerTokenAuthorization(jenkinsAPIURL, token.String()) +} + +func createJenkinsAPIClientFromSecret(t *testing.T, jenkins *v1alpha2.Jenkins, jenkinsAPIURL string) (jenkinsclient.Jenkins, error) { + t.Log("Creating Jenkins API client from secret") + + adminSecret := &corev1.Secret{} + namespaceName := types.NamespacedName{Namespace: jenkins.Namespace, Name: resources.GetOperatorCredentialsSecretName(jenkins)} + if err := framework.Global.Client.Get(context.TODO(), namespaceName, adminSecret); err != nil { + return nil, err + } + + return jenkinsclient.NewUserAndPasswordAuthorization( + jenkinsAPIURL, + string(adminSecret.Data[resources.OperatorCredentialsSecretUserNameKey]), + string(adminSecret.Data[resources.OperatorCredentialsSecretTokenKey]), + ) +} + func verifyJenkinsAPIConnection(t *testing.T, jenkins *v1alpha2.Jenkins, namespace string) (jenkinsclient.Jenkins, func()) { + var service corev1.Service + err := framework.Global.Client.Get(context.TODO(), types.NamespacedName{ + Namespace: jenkins.Namespace, + Name: resources.GetJenkinsHTTPServiceName(jenkins), + }, &service) + require.NoError(t, err) + podName := resources.GetJenkinsMasterPodName(*jenkins) port, cleanUpFunc, waitFunc, portForwardFunc, err := setupPortForwardToPod(t, namespace, podName, int(constants.DefaultHTTPPortInt32)) if err != nil { @@ -186,7 +202,20 @@ func verifyJenkinsAPIConnection(t *testing.T, jenkins *v1alpha2.Jenkins, namespa } go portForwardFunc() waitFunc() - client, err := createJenkinsAPIClient(jenkins, "localhost", port) + + jenkinsAPIURL := jenkinsclient.JenkinsAPIConnectionSettings{ + Hostname: "localhost", + Port: port, + UseNodePort: false, + }.BuildJenkinsAPIUrl(service.Name, service.Namespace, service.Spec.Ports[0].Port, service.Spec.Ports[0].NodePort) + + var client jenkinsclient.Jenkins + if jenkins.Spec.JenkinsAPISettings.AuthorizationStrategy == v1alpha2.ServiceAccountAuthorizationStrategy { + client, err = createJenkinsAPIClientFromServiceAccount(t, jenkins, jenkinsAPIURL) + } else { + client, err = createJenkinsAPIClientFromSecret(t, jenkins, jenkinsAPIURL) + } + if err != nil { defer cleanUpFunc() t.Fatal(err) diff --git a/test/e2e/cr.go b/test/e2e/mode_kubernetes.go similarity index 70% rename from test/e2e/cr.go rename to test/e2e/mode_kubernetes.go index 48639680..d2c27c79 100644 --- a/test/e2e/cr.go +++ b/test/e2e/mode_kubernetes.go @@ -1,4 +1,5 @@ // +build !OpenShift +// +build !OpenShiftOAuth package e2e @@ -8,6 +9,9 @@ import ( "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2" ) +const skipTestSafeRestart = false + func updateJenkinsCR(t *testing.T, jenkins *v1alpha2.Jenkins) { + t.Log("Update Jenkins CR") // do nothing } diff --git a/test/e2e/cr_openshift.go b/test/e2e/mode_openshift.go similarity index 92% rename from test/e2e/cr_openshift.go rename to test/e2e/mode_openshift.go index 3dc724cd..5b52723d 100644 --- a/test/e2e/cr_openshift.go +++ b/test/e2e/mode_openshift.go @@ -11,7 +11,11 @@ import ( corev1 "k8s.io/api/core/v1" ) +const skipTestSafeRestart = false + func updateJenkinsCR(t *testing.T, jenkins *v1alpha2.Jenkins) { + t.Log("Update Jenkins CR: OpenShift") + jenkins.Spec.Master.Containers[0].Image = "quay.io/openshift/origin-jenkins" jenkins.Spec.Master.Containers[0].Command = []string{ "bash", diff --git a/test/e2e/mode_openshift_oauth.go b/test/e2e/mode_openshift_oauth.go new file mode 100644 index 00000000..5c688129 --- /dev/null +++ b/test/e2e/mode_openshift_oauth.go @@ -0,0 +1,95 @@ +// +build OpenShiftOAuth + +package e2e + +import ( + "context" + "testing" + + "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants" + + framework "github.com/operator-framework/operator-sdk/pkg/test" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const skipTestSafeRestart = true + +func updateJenkinsCR(t *testing.T, jenkins *v1alpha2.Jenkins) { + t.Log("Update Jenkins CR: OpenShiftOAuth") + + adminRoleBinding := &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + Kind: "RoleBinding", + APIVersion: "rbac.authorization.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "admin", + Namespace: jenkins.Namespace, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: "admin", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: constants.OperatorName, + Namespace: jenkins.Namespace, + }, + }, + } + if err := framework.Global.Client.Create(context.TODO(), adminRoleBinding, nil); err != nil { + t.Fatal(err) + } + + jenkins.Spec.JenkinsAPISettings = v1alpha2.JenkinsAPISettings{AuthorizationStrategy: v1alpha2.ServiceAccountAuthorizationStrategy} + jenkins.Spec.Master.Containers[0].Image = "quay.io/openshift/origin-jenkins" + jenkins.Spec.Master.Containers[0].Command = []string{ + "bash", + "-c", + "/var/jenkins/scripts/init.sh && exec /usr/bin/go-init -main /usr/libexec/s2i/run", + } + jenkins.Spec.Roles = []rbacv1.RoleRef{ + { + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: "admin", + }, + } + jenkins.Spec.Master.DisableCSRFProtection = true + jenkins.Spec.Master.Containers[0].Env = append(jenkins.Spec.Master.Containers[0].Env, + corev1.EnvVar{ + Name: "OPENSHIFT_ENABLE_OAUTH", + Value: "true", + }, + corev1.EnvVar{ + Name: "OPENSHIFT_ENABLE_REDIRECT_PROMPT", + Value: "true", + }, + corev1.EnvVar{ + Name: "DISABLE_ADMINISTRATIVE_MONITORS", + Value: "false", + }, + corev1.EnvVar{ + Name: "KUBERNETES_TRUST_CERTIFICATES", + Value: "true", + }, + corev1.EnvVar{ + Name: "JENKINS_UC_INSECURE", + Value: "false", + }, + corev1.EnvVar{ + Name: "JENKINS_SERVICE_NAME", + Value: resources.GetJenkinsHTTPServiceName(jenkins), + }, + corev1.EnvVar{ + Name: "JNLP_SERVICE_NAME", + Value: resources.GetJenkinsSlavesServiceName(jenkins), + }, + ) +} diff --git a/test/e2e/restart_test.go b/test/e2e/restart_test.go index 4f586a5c..f2181d06 100644 --- a/test/e2e/restart_test.go +++ b/test/e2e/restart_test.go @@ -29,6 +29,9 @@ func TestJenkinsMasterPodRestart(t *testing.T) { } func TestSafeRestart(t *testing.T) { + if skipTestSafeRestart { + t.Skip() + } t.Parallel() namespace, ctx := setupTest(t) // Deletes test namespace diff --git a/test/e2e/restorebackup_test.go b/test/e2e/restorebackup_test.go index 2a8cb204..394c0cf9 100644 --- a/test/e2e/restorebackup_test.go +++ b/test/e2e/restorebackup_test.go @@ -183,6 +183,7 @@ func createJenkinsWithBackupAndRestoreConfigured(t *testing.T, name, namespace s }, }, } + updateJenkinsCR(t, jenkins) t.Logf("Jenkins CR %+v", *jenkins) err := framework.Global.Client.Create(context.TODO(), jenkins, nil) diff --git a/test/e2e/wait.go b/test/e2e/wait.go index c7620309..fbf7a673 100644 --- a/test/e2e/wait.go +++ b/test/e2e/wait.go @@ -58,7 +58,7 @@ func waitForJobToFinish(t *testing.T, job *gojenkins.Job, tick, timeout time.Dur func waitForJenkinsBaseConfigurationToComplete(t *testing.T, jenkins *v1alpha2.Jenkins) { t.Log("Waiting for Jenkins base configuration to complete") - _, err := WaitUntilJenkinsConditionSet(retryInterval, 150, jenkins, func(jenkins *v1alpha2.Jenkins, err error) bool { + _, err := WaitUntilJenkinsConditionSet(retryInterval, 170, jenkins, func(jenkins *v1alpha2.Jenkins, err error) bool { t.Logf("Current Jenkins status: '%+v', error '%s'", jenkins.Status, err) return err == nil && jenkins.Status.BaseConfigurationCompletedTime != nil }) @@ -96,7 +96,7 @@ func waitForRecreateJenkinsMasterPod(t *testing.T, jenkins *v1alpha2.Jenkins) { func waitForJenkinsUserConfigurationToComplete(t *testing.T, jenkins *v1alpha2.Jenkins) { t.Log("Waiting for Jenkins user configuration to complete") - _, err := WaitUntilJenkinsConditionSet(retryInterval, 75, jenkins, func(jenkins *v1alpha2.Jenkins, err error) bool { + _, err := WaitUntilJenkinsConditionSet(retryInterval, 110, jenkins, func(jenkins *v1alpha2.Jenkins, err error) bool { t.Logf("Current Jenkins status: '%+v', error '%s'", jenkins.Status, err) return err == nil && jenkins.Status.UserConfigurationCompletedTime != nil })