From a612939f338b42dec1a60b01a93134a1f937610d Mon Sep 17 00:00:00 2001 From: Jakub Al-Khalili Date: Fri, 22 Nov 2019 11:52:19 +0100 Subject: [PATCH] Add support for local development kubernetes cluster software (#207) #203 Add support for local development kubernetes cluster --- Makefile | 15 +++-- cmd/manager/main.go | 14 ++++- config.env => config.base.env | 4 -- config.minikube.env | 10 ++++ pkg/controller/jenkins/client/jenkins.go | 56 ++++++++++--------- .../jenkins/configuration/base/reconcile.go | 34 ++++++----- .../configuration/base/reconcile_test.go | 6 +- .../configuration/base/validate_test.go | 55 +++++++++--------- pkg/controller/jenkins/jenkins_controller.go | 43 ++++++-------- test/e2e/configuration_test.go | 6 +- test/e2e/jenkins.go | 26 +++++++-- test/e2e/main_test.go | 10 ++++ test/e2e/restart_test.go | 2 +- test/e2e/restorebackup_test.go | 9 ++- test/e2e/seedjobs_test.go | 2 +- 15 files changed, 171 insertions(+), 121 deletions(-) rename config.env => config.base.env (75%) create mode 100644 config.minikube.env diff --git a/Makefile b/Makefile index 9bb0c038..fd280500 100644 --- a/Makefile +++ b/Makefile @@ -15,9 +15,11 @@ else endif endif +include config.base.env + # Import config # You can change the default config with `make config="config_special.env" build` -config ?= config.env +config ?= config.minikube.env include $(config) # Set an output prefix, which is the local directory if not specified @@ -154,6 +156,7 @@ test: ## Runs the go tests .PHONY: e2e CURRENT_DIRECTORY := $(shell pwd) +JENKINS_API_HOSTNAME := $(shell $(JENKINS_API_HOSTNAME_COMMAND)) e2e: docker-build ## Runs e2e tests, you can use EXTRA_ARGS @echo "+ $@" @echo "Docker image: $(DOCKER_REGISTRY):$(GITCOMMIT)" @@ -164,22 +167,22 @@ e2e: docker-build ## Runs e2e tests, you can use EXTRA_ARGS cat deploy/operator.yaml >> deploy/namespace-init.yaml ifeq ($(OSFLAG), LINUX) sed -i 's|\(image:\).*|\1 $(DOCKER_REGISTRY):$(GITCOMMIT)|g' deploy/namespace-init.yaml -ifeq ($(KUBECTL_CONTEXT),minikube) +ifeq ($(KUBERNETES_PROVIDER),minikube) sed -i 's|\(imagePullPolicy\): IfNotPresent|\1: Never|g' deploy/namespace-init.yaml - sed -i 's|\(args:\).*|\1\ ["--minikube"\]|' deploy/namespace-init.yaml endif endif ifeq ($(OSFLAG), OSX) sed -i '' 's|\(image:\).*|\1 $(DOCKER_REGISTRY):$(GITCOMMIT)|g' deploy/namespace-init.yaml -ifeq ($(KUBECTL_CONTEXT),minikube) +ifeq ($(KUBERNETES_PROVIDER),minikube) sed -i '' 's|\(imagePullPolicy\): IfNotPresent|\1: Never|g' deploy/namespace-init.yaml - sed -i '' 's|\(args:\).*|\1\ ["--minikube"\]|' deploy/namespace-init.yaml endif endif @RUNNING_TESTS=1 go test -parallel=1 "./test/e2e/" -tags "$(BUILDTAGS) cgo" -v -timeout 45m -run "$(E2E_TEST_SELECTOR)" \ - -root=$(CURRENT_DIRECTORY) -kubeconfig=$(HOME)/.kube/config -globalMan deploy/crds/jenkins_$(API_VERSION)_jenkins_crd.yaml -namespacedMan deploy/namespace-init.yaml $(EXTRA_ARGS) + -root=$(CURRENT_DIRECTORY) -kubeconfig=$(HOME)/.kube/config -globalMan deploy/crds/jenkins_$(API_VERSION)_jenkins_crd.yaml \ + -jenkins-api-hostname=$(JENKINS_API_HOSTNAME) -jenkins-api-port=$(JENKINS_API_PORT) -jenkins-api-use-nodeport=$(JENKINS_API_USE_NODEPORT) \ + -namespacedMan deploy/namespace-init.yaml $(EXTRA_ARGS) .PHONY: vet vet: ## Verifies `go vet` passes diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 6831f396..be6082df 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -12,6 +12,7 @@ import ( "github.com/jenkinsci/kubernetes-operator/pkg/apis" "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client" "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants" "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/notifications" e "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/notifications/event" @@ -64,8 +65,9 @@ func main() { // controller-runtime) pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - minikube := pflag.Bool("minikube", false, "Use minikube as a Kubernetes platform") - local := pflag.Bool("local", false, "Run operator locally") + hostname := pflag.String("jenkins-api-hostname", "", "Hostname or IP of Jenkins API. It can be service name, node IP or localhost.") + port := pflag.Int("jenkins-api-port", 0, "The port on which Jenkins API is running. Note: If you want to use nodePort don't set this setting and --jenkins-api-use-nodeport must be true.") + useNodePort := pflag.Bool("jenkins-api-use-nodeport", false, "Connect to Jenkins API using the service nodePort instead of service port. If you want to set this as true - don't set --jenkins-api-port.") debug := pflag.Bool("debug", false, "Set log level to debug") pflag.Parse() @@ -123,8 +125,14 @@ func main() { c := make(chan e.Event) go notifications.Listen(c, events, mgr.GetClient()) + // validate jenkins API connection + jenkinsAPIConnectionSettings := client.JenkinsAPIConnectionSettings{Hostname: *hostname, Port: *port, UseNodePort: *useNodePort} + if err := jenkinsAPIConnectionSettings.Validate(); err != nil { + fatal(errors.Wrap(err, "invalid command line parameters"), *debug) + } + // setup Jenkins controller - if err := jenkins.Add(mgr, *local, *minikube, *clientSet, *cfg, &c); err != nil { + if err := jenkins.Add(mgr, jenkinsAPIConnectionSettings, *clientSet, *cfg, &c); err != nil { fatal(errors.Wrap(err, "failed to setup controllers"), *debug) } diff --git a/config.env b/config.base.env similarity index 75% rename from config.env rename to config.base.env index 53037d69..bfb2aab8 100644 --- a/config.env +++ b/config.base.env @@ -7,9 +7,5 @@ DOCKER_ORGANIZATION=virtuslab DOCKER_REGISTRY=jenkins-operator NAMESPACE=default API_VERSION=v1alpha2 -MINIKUBE_KUBERNETES_VERSION=v1.16.0 -MINIKUBE_DRIVER=virtualbox -MINIKUBE_VERSION=1.4.0 -KUBECTL_CONTEXT=minikube ALL_IN_ONE_DEPLOY_FILE_PREFIX=all-in-one GEN_CRD_API=gen-crd-api-reference-docs diff --git a/config.minikube.env b/config.minikube.env new file mode 100644 index 00000000..6c6ff315 --- /dev/null +++ b/config.minikube.env @@ -0,0 +1,10 @@ +KUBERNETES_PROVIDER=minikube + +MINIKUBE_KUBERNETES_VERSION=v1.16.0 +MINIKUBE_DRIVER=virtualbox +MINIKUBE_VERSION=1.4.0 +KUBECTL_CONTEXT=minikube + +JENKINS_API_HOSTNAME_COMMAND=minikube ip +JENKINS_API_PORT=-1 +JENKINS_API_USE_NODEPORT=true \ No newline at end of file diff --git a/pkg/controller/jenkins/client/jenkins.go b/pkg/controller/jenkins/client/jenkins.go index e757073b..802850d8 100644 --- a/pkg/controller/jenkins/client/jenkins.go +++ b/pkg/controller/jenkins/client/jenkins.go @@ -1,10 +1,8 @@ package client import ( - "bytes" "fmt" "net/http" - "os/exec" "regexp" "strings" @@ -63,6 +61,13 @@ type jenkins struct { gojenkins.Jenkins } +// JenkinsAPIConnectionSettings is struct that handle information about Jenkins API connection +type JenkinsAPIConnectionSettings struct { + Hostname string + Port int + UseNodePort bool +} + // CreateOrUpdateJob creates or updates a job from config func (jenkins *jenkins) CreateOrUpdateJob(config, jobName string) (job *gojenkins.Job, created bool, err error) { // create or update @@ -80,34 +85,33 @@ func (jenkins *jenkins) CreateOrUpdateJob(config, jobName string) (job *gojenkin } // BuildJenkinsAPIUrl returns Jenkins API URL -func BuildJenkinsAPIUrl(namespace, serviceName string, portNumber int32, local, minikube bool) (string, error) { - // Get Jenkins URL from minikube command - if local && minikube { - cmd := exec.Command("minikube", "service", "--url", "-n", namespace, serviceName) - var out bytes.Buffer - cmd.Stdout = &out - err := cmd.Run() - if err != nil { - return "", errors.WithStack(err) - } - lines := strings.Split(out.String(), "\n") - // First is for http, the second one is for Jenkins slaves communication - // see pkg/controller/jenkins/configuration/base/resources/service.go - url := lines[0] - if strings.HasPrefix(url, "* ") { - return url[2:], nil - } - - return url, nil +func (j JenkinsAPIConnectionSettings) BuildJenkinsAPIUrl(serviceName string, serviceNamespace string, servicePort int32, serviceNodePort int32) string { + if j.Hostname == "" && j.Port == 0 { + return fmt.Sprintf("http://%s.%s:%d", serviceName, serviceNamespace, servicePort) } - if local { - // When run locally make port-forward to jenkins pod ('kubectl -n default port-forward jenkins-operator-example 8080') - return fmt.Sprintf("http://localhost:%d", portNumber), nil + if j.Hostname != "" && j.UseNodePort { + return fmt.Sprintf("http://%s:%d", j.Hostname, serviceNodePort) } - // Connect through Kubernetes service, operator has to be run inside cluster - return fmt.Sprintf("http://%s.%s:%d", serviceName, namespace, portNumber), nil + return fmt.Sprintf("http://%s:%d", j.Hostname, j.Port) +} + +// Validate validates jenkins API connection settings +func (j JenkinsAPIConnectionSettings) Validate() error { + if j.Port >= 0 && j.UseNodePort { + return errors.New("can't use service port and nodePort both. Please use port or nodePort") + } + + if j.Port < 0 { + return errors.New("service port cannot be lower than 0") + } + + if (j.Hostname == "" && j.Port > 0) || (j.Hostname == "" && j.UseNodePort) { + return errors.New("empty hostname is now allowed. Please provide hostname") + } + + return nil } // New creates Jenkins API client diff --git a/pkg/controller/jenkins/configuration/base/reconcile.go b/pkg/controller/jenkins/configuration/base/reconcile.go index 77f6ba7b..42b17b54 100644 --- a/pkg/controller/jenkins/configuration/base/reconcile.go +++ b/pkg/controller/jenkins/configuration/base/reconcile.go @@ -41,19 +41,18 @@ const ( // ReconcileJenkinsBaseConfiguration defines values required for Jenkins base configuration type ReconcileJenkinsBaseConfiguration struct { configuration.Configuration - logger logr.Logger - local, minikube bool - config *rest.Config + logger logr.Logger + jenkinsAPIConnectionSettings jenkinsclient.JenkinsAPIConnectionSettings + config *rest.Config } // New create structure which takes care of base configuration -func New(config configuration.Configuration, logger logr.Logger, local, minikube bool, restConfig *rest.Config) *ReconcileJenkinsBaseConfiguration { +func New(config configuration.Configuration, logger logr.Logger, jenkinsAPIConnectionSettings jenkinsclient.JenkinsAPIConnectionSettings, restConfig *rest.Config) *ReconcileJenkinsBaseConfiguration { return &ReconcileJenkinsBaseConfiguration{ - Configuration: config, - logger: logger, - local: local, - minikube: minikube, - config: restConfig, + Configuration: config, + logger: logger, + jenkinsAPIConnectionSettings: jenkinsAPIConnectionSettings, + config: restConfig, } } @@ -800,16 +799,23 @@ func (r *ReconcileJenkinsBaseConfiguration) waitForJenkins() (reconcile.Result, } func (r *ReconcileJenkinsBaseConfiguration) ensureJenkinsClient() (jenkinsclient.Jenkins, error) { - jenkinsURL, err := jenkinsclient.BuildJenkinsAPIUrl( - r.Configuration.Jenkins.ObjectMeta.Namespace, resources.GetJenkinsHTTPServiceName(r.Configuration.Jenkins), r.Configuration.Jenkins.Spec.Service.Port, r.local, r.minikube) + var service corev1.Service + + err := r.Client.Get(context.TODO(), types.NamespacedName{ + Namespace: r.Configuration.Jenkins.ObjectMeta.Namespace, + Name: resources.GetJenkinsHTTPServiceName(r.Configuration.Jenkins), + }, &service) + + if err != nil { + return nil, err + } + + jenkinsURL := r.jenkinsAPIConnectionSettings.BuildJenkinsAPIUrl(service.Name, service.Namespace, service.Spec.Ports[0].Port, service.Spec.Ports[0].NodePort) if prefix, ok := GetJenkinsOpts(*r.Configuration.Jenkins)["prefix"]; ok { jenkinsURL = jenkinsURL + prefix } - if err != nil { - return nil, err - } r.logger.V(log.VDebug).Info(fmt.Sprintf("Jenkins API URL '%s'", jenkinsURL)) credentialsSecret := &corev1.Secret{} diff --git a/pkg/controller/jenkins/configuration/base/reconcile_test.go b/pkg/controller/jenkins/configuration/base/reconcile_test.go index 67088a71..2b6a073a 100644 --- a/pkg/controller/jenkins/configuration/base/reconcile_test.go +++ b/pkg/controller/jenkins/configuration/base/reconcile_test.go @@ -234,7 +234,7 @@ func TestCompareVolumes(t *testing.T) { Volumes: resources.GetJenkinsMasterPodBaseVolumes(jenkins), }, } - reconciler := New(configuration.Configuration{Jenkins: jenkins}, nil, false, false, nil) + reconciler := New(configuration.Configuration{Jenkins: jenkins}, nil, client.JenkinsAPIConnectionSettings{}, nil) got := reconciler.compareVolumes(pod) @@ -258,7 +258,7 @@ func TestCompareVolumes(t *testing.T) { Volumes: resources.GetJenkinsMasterPodBaseVolumes(jenkins), }, } - reconciler := New(configuration.Configuration{Jenkins: jenkins}, nil, false, false, nil) + reconciler := New(configuration.Configuration{Jenkins: jenkins}, nil, client.JenkinsAPIConnectionSettings{}, nil) got := reconciler.compareVolumes(pod) @@ -282,7 +282,7 @@ func TestCompareVolumes(t *testing.T) { Volumes: append(resources.GetJenkinsMasterPodBaseVolumes(jenkins), corev1.Volume{Name: "added"}), }, } - reconciler := New(configuration.Configuration{Jenkins: jenkins}, nil, false, false, nil) + reconciler := New(configuration.Configuration{Jenkins: jenkins}, nil, client.JenkinsAPIConnectionSettings{}, nil) got := reconciler.compareVolumes(pod) diff --git a/pkg/controller/jenkins/configuration/base/validate_test.go b/pkg/controller/jenkins/configuration/base/validate_test.go index fff7101f..b6d4415e 100644 --- a/pkg/controller/jenkins/configuration/base/validate_test.go +++ b/pkg/controller/jenkins/configuration/base/validate_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client" "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration" "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources" "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants" @@ -23,7 +24,7 @@ import ( func TestValidatePlugins(t *testing.T) { log.SetupLogger(true) - baseReconcileLoop := New(configuration.Configuration{}, log.Log, false, false, nil) + baseReconcileLoop := New(configuration.Configuration{}, log.Log, client.JenkinsAPIConnectionSettings{}, nil) t.Run("empty", func(t *testing.T) { var requiredBasePlugins []plugins.Plugin var basePlugins []v1alpha2.Plugin @@ -166,7 +167,7 @@ func TestReconcileJenkinsBaseConfiguration_validateImagePullSecrets(t *testing.T baseReconcileLoop := New(configuration.Configuration{ Client: fakeClient, Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got, err := baseReconcileLoop.validateImagePullSecrets() fmt.Println(got) @@ -190,7 +191,7 @@ func TestReconcileJenkinsBaseConfiguration_validateImagePullSecrets(t *testing.T baseReconcileLoop := New(configuration.Configuration{ Client: fakeClient, Jenkins: &jenkins, - }, nil, false, false, nil) + }, nil, client.JenkinsAPIConnectionSettings{}, nil) got, _ := baseReconcileLoop.validateImagePullSecrets() @@ -226,7 +227,7 @@ func TestReconcileJenkinsBaseConfiguration_validateImagePullSecrets(t *testing.T baseReconcileLoop := New(configuration.Configuration{ Client: fakeClient, Jenkins: &jenkins, - }, nil, false, false, nil) + }, nil, client.JenkinsAPIConnectionSettings{}, nil) got, _ := baseReconcileLoop.validateImagePullSecrets() @@ -262,7 +263,7 @@ func TestReconcileJenkinsBaseConfiguration_validateImagePullSecrets(t *testing.T baseReconcileLoop := New(configuration.Configuration{ Client: fakeClient, Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got, _ := baseReconcileLoop.validateImagePullSecrets() @@ -298,7 +299,7 @@ func TestReconcileJenkinsBaseConfiguration_validateImagePullSecrets(t *testing.T baseReconcileLoop := New(configuration.Configuration{ Client: fakeClient, Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got, _ := baseReconcileLoop.validateImagePullSecrets() @@ -334,7 +335,7 @@ func TestReconcileJenkinsBaseConfiguration_validateImagePullSecrets(t *testing.T baseReconcileLoop := New(configuration.Configuration{ Client: fakeClient, Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got, _ := baseReconcileLoop.validateImagePullSecrets() @@ -367,7 +368,7 @@ func TestValidateJenkinsMasterPodEnvs(t *testing.T) { } baseReconcileLoop := New(configuration.Configuration{ Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got := baseReconcileLoop.validateJenkinsMasterPodEnvs() assert.Nil(t, got) }) @@ -390,7 +391,7 @@ func TestValidateJenkinsMasterPodEnvs(t *testing.T) { } baseReconcileLoop := New(configuration.Configuration{ Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got := baseReconcileLoop.validateJenkinsMasterPodEnvs() assert.Equal(t, got, []string{"Jenkins Master container env 'JAVA_OPTS' doesn't have required flag '-Djava.awt.headless=true'"}) @@ -414,7 +415,7 @@ func TestValidateJenkinsMasterPodEnvs(t *testing.T) { } baseReconcileLoop := New(configuration.Configuration{ Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got := baseReconcileLoop.validateJenkinsMasterPodEnvs() assert.Equal(t, got, []string{"Jenkins Master container env 'JAVA_OPTS' doesn't have required flag '-Djenkins.install.runSetupWizard=false'"}) @@ -436,7 +437,7 @@ func TestValidateReservedVolumes(t *testing.T) { } baseReconcileLoop := New(configuration.Configuration{ Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got := baseReconcileLoop.validateReservedVolumes() assert.Nil(t, got) }) @@ -454,7 +455,7 @@ func TestValidateReservedVolumes(t *testing.T) { } baseReconcileLoop := New(configuration.Configuration{ Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got := baseReconcileLoop.validateReservedVolumes() assert.Equal(t, got, []string{"Jenkins Master pod volume 'jenkins-home' is reserved please choose different one"}) @@ -470,7 +471,7 @@ func TestValidateContainerVolumeMounts(t *testing.T) { } baseReconcileLoop := New(configuration.Configuration{ Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got := baseReconcileLoop.validateContainerVolumeMounts(v1alpha2.Container{}) assert.Nil(t, got) }) @@ -498,7 +499,7 @@ func TestValidateContainerVolumeMounts(t *testing.T) { } baseReconcileLoop := New(configuration.Configuration{ Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got := baseReconcileLoop.validateContainerVolumeMounts(jenkins.Spec.Master.Containers[0]) assert.Nil(t, got) }) @@ -526,7 +527,7 @@ func TestValidateContainerVolumeMounts(t *testing.T) { } baseReconcileLoop := New(configuration.Configuration{ Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got := baseReconcileLoop.validateContainerVolumeMounts(jenkins.Spec.Master.Containers[0]) assert.Equal(t, got, []string{"mountPath not set for 'example' volume mount in container ''"}) }) @@ -549,7 +550,7 @@ func TestValidateContainerVolumeMounts(t *testing.T) { } baseReconcileLoop := New(configuration.Configuration{ Jenkins: &jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got := baseReconcileLoop.validateContainerVolumeMounts(jenkins.Spec.Master.Containers[0]) assert.Equal(t, got, []string{"Not found volume for 'missing-volume' volume mount in container ''"}) @@ -571,7 +572,7 @@ func TestValidateConfigMapVolume(t *testing.T) { fakeClient := fake.NewFakeClient() baseReconcileLoop := New(configuration.Configuration{ Client: fakeClient, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got, err := baseReconcileLoop.validateConfigMapVolume(volume) @@ -599,7 +600,7 @@ func TestValidateConfigMapVolume(t *testing.T) { baseReconcileLoop := New(configuration.Configuration{ Client: fakeClient, Jenkins: jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got, err := baseReconcileLoop.validateConfigMapVolume(volume) @@ -625,7 +626,7 @@ func TestValidateConfigMapVolume(t *testing.T) { baseReconcileLoop := New(configuration.Configuration{ Client: fakeClient, Jenkins: jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got, err := baseReconcileLoop.validateConfigMapVolume(volume) @@ -650,7 +651,7 @@ func TestValidateSecretVolume(t *testing.T) { fakeClient := fake.NewFakeClient() baseReconcileLoop := New(configuration.Configuration{ Client: fakeClient, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got, err := baseReconcileLoop.validateSecretVolume(volume) @@ -676,7 +677,7 @@ func TestValidateSecretVolume(t *testing.T) { baseReconcileLoop := New(configuration.Configuration{ Client: fakeClient, Jenkins: jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got, err := baseReconcileLoop.validateSecretVolume(volume) @@ -700,7 +701,7 @@ func TestValidateSecretVolume(t *testing.T) { baseReconcileLoop := New(configuration.Configuration{ Client: fakeClient, Jenkins: jenkins, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got, err := baseReconcileLoop.validateSecretVolume(volume) assert.NoError(t, err) @@ -724,7 +725,7 @@ func TestValidateCustomization(t *testing.T) { baseReconcileLoop := New(configuration.Configuration{ Jenkins: jenkins, Client: fakeClient, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) got, err := baseReconcileLoop.validateCustomization(customization, "spec.groovyScripts") @@ -746,7 +747,7 @@ func TestValidateCustomization(t *testing.T) { baseReconcileLoop := New(configuration.Configuration{ Jenkins: jenkins, Client: fakeClient, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) err := fakeClient.Create(context.TODO(), secret) require.NoError(t, err) @@ -777,7 +778,7 @@ func TestValidateCustomization(t *testing.T) { baseReconcileLoop := New(configuration.Configuration{ Jenkins: jenkins, Client: fakeClient, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) err := fakeClient.Create(context.TODO(), secret) require.NoError(t, err) err = fakeClient.Create(context.TODO(), configMap) @@ -804,7 +805,7 @@ func TestValidateCustomization(t *testing.T) { baseReconcileLoop := New(configuration.Configuration{ Jenkins: jenkins, Client: fakeClient, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) err := fakeClient.Create(context.TODO(), configMap) require.NoError(t, err) @@ -829,7 +830,7 @@ func TestValidateCustomization(t *testing.T) { baseReconcileLoop := New(configuration.Configuration{ Jenkins: jenkins, Client: fakeClient, - }, logf.ZapLogger(false), false, false, nil) + }, logf.ZapLogger(false), client.JenkinsAPIConnectionSettings{}, nil) err := fakeClient.Create(context.TODO(), secret) require.NoError(t, err) diff --git a/pkg/controller/jenkins/jenkins_controller.go b/pkg/controller/jenkins/jenkins_controller.go index f0885dd0..79607a44 100644 --- a/pkg/controller/jenkins/jenkins_controller.go +++ b/pkg/controller/jenkins/jenkins_controller.go @@ -47,20 +47,19 @@ var reconcileErrors = map[string]reconcileError{} // Add creates a new Jenkins Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, local, minikube bool, clientSet kubernetes.Clientset, config rest.Config, notificationEvents *chan event.Event) error { - return add(mgr, newReconciler(mgr, local, minikube, clientSet, config, notificationEvents)) +func Add(mgr manager.Manager, jenkinsAPIConnectionSettings jenkinsclient.JenkinsAPIConnectionSettings, clientSet kubernetes.Clientset, config rest.Config, notificationEvents *chan event.Event) error { + return add(mgr, newReconciler(mgr, jenkinsAPIConnectionSettings, clientSet, config, notificationEvents)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, local, minikube bool, clientSet kubernetes.Clientset, config rest.Config, notificationEvents *chan event.Event) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, jenkinsAPIConnectionSettings jenkinsclient.JenkinsAPIConnectionSettings, clientSet kubernetes.Clientset, config rest.Config, notificationEvents *chan event.Event) reconcile.Reconciler { return &ReconcileJenkins{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - local: local, - minikube: minikube, - clientSet: clientSet, - config: config, - notificationEvents: notificationEvents, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + jenkinsAPIConnectionSettings: jenkinsAPIConnectionSettings, + clientSet: clientSet, + config: config, + notificationEvents: notificationEvents, } } @@ -114,12 +113,12 @@ var _ reconcile.Reconciler = &ReconcileJenkins{} // ReconcileJenkins reconciles a Jenkins object type ReconcileJenkins struct { - client client.Client - scheme *runtime.Scheme - local, minikube bool - clientSet kubernetes.Clientset - config rest.Config - notificationEvents *chan event.Event + client client.Client + scheme *runtime.Scheme + jenkinsAPIConnectionSettings jenkinsclient.JenkinsAPIConnectionSettings + clientSet kubernetes.Clientset + config rest.Config + notificationEvents *chan event.Event } // Reconcile it's a main reconciliation loop which maintain desired state based on Jenkins.Spec @@ -236,7 +235,7 @@ func (r *ReconcileJenkins) reconcile(request reconcile.Request, logger logr.Logg } // Reconcile base configuration - baseConfiguration := base.New(config, logger, r.local, r.minikube, &r.config) + baseConfiguration := base.New(config, logger, r.jenkinsAPIConnectionSettings, &r.config) var baseMessages []string baseMessages, err = baseConfiguration.Validate(jenkins) @@ -446,16 +445,8 @@ func (r *ReconcileJenkins) setDefaults(jenkins *v1alpha2.Jenkins, logger logr.Lo if reflect.DeepEqual(jenkins.Spec.Service, v1alpha2.Service{}) { logger.Info("Setting default Jenkins master service") changed = true - var serviceType corev1.ServiceType - if r.minikube { - // When running locally with minikube cluster Jenkins Service have to be exposed via node port - // to allow communication operator -> Jenkins API - serviceType = corev1.ServiceTypeNodePort - } else { - serviceType = corev1.ServiceTypeClusterIP - } jenkins.Spec.Service = v1alpha2.Service{ - Type: serviceType, + Type: corev1.ServiceTypeClusterIP, Port: constants.DefaultHTTPPortInt32, } } diff --git a/test/e2e/configuration_test.go b/test/e2e/configuration_test.go index b21fe118..a8f1b319 100644 --- a/test/e2e/configuration_test.go +++ b/test/e2e/configuration_test.go @@ -89,7 +89,7 @@ func TestConfiguration(t *testing.T) { createKubernetesCredentialsProviderSecret(t, namespace, mySeedJob) waitForJenkinsBaseConfigurationToComplete(t, jenkins) verifyJenkinsMasterPodAttributes(t, jenkins) - client := verifyJenkinsAPIConnection(t, jenkins) + client := verifyJenkinsAPIConnection(t, jenkins, *hostname, *port, *useNodePort) verifyPlugins(t, client, jenkins) // user @@ -121,7 +121,7 @@ func TestPlugins(t *testing.T) { jenkins := createJenkinsCR(t, "k8s-e2e", namespace, seedJobs, v1alpha2.GroovyScripts{}, v1alpha2.ConfigurationAsCode{}) waitForJenkinsUserConfigurationToComplete(t, jenkins) - jenkinsClient := verifyJenkinsAPIConnection(t, jenkins) + jenkinsClient := verifyJenkinsAPIConnection(t, jenkins, *hostname, *port, *useNodePort) waitForJob(t, jenkinsClient, jobID) job, err := jenkinsClient.GetJob(jobID) @@ -267,7 +267,7 @@ func verifyJenkinsMasterPodAttributes(t *testing.T, jenkins *v1alpha2.Jenkins) { func verifyContainer(t *testing.T, expected corev1.Container, actual corev1.Container) { assert.Equal(t, expected.Args, actual.Args, expected.Name, expected.Name) assert.Equal(t, expected.Command, actual.Command, expected.Name) - assert.Equal(t, expected.Env, actual.Env, expected.Name) + assert.ElementsMatch(t, expected.Env, actual.Env, expected.Name) assert.Equal(t, expected.EnvFrom, actual.EnvFrom, expected.Name) assert.Equal(t, expected.Image, actual.Image, expected.Name) assert.Equal(t, expected.ImagePullPolicy, actual.ImagePullPolicy, expected.Name) diff --git a/test/e2e/jenkins.go b/test/e2e/jenkins.go index e842e9f1..df1b5784 100644 --- a/test/e2e/jenkins.go +++ b/test/e2e/jenkins.go @@ -7,7 +7,7 @@ import ( "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2" jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client" "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources" - + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants" framework "github.com/operator-framework/operator-sdk/pkg/test" "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" @@ -46,18 +46,30 @@ func getJenkinsMasterPod(t *testing.T, jenkins *v1alpha2.Jenkins) *v1.Pod { return &podList.Items[0] } -func createJenkinsAPIClient(jenkins *v1alpha2.Jenkins) (jenkinsclient.Jenkins, error) { +func createJenkinsAPIClient(jenkins *v1alpha2.Jenkins, hostname string, port int, useNodePort bool) (jenkinsclient.Jenkins, error) { adminSecret := &v1.Secret{} namespaceName := types.NamespacedName{Namespace: jenkins.Namespace, Name: resources.GetOperatorCredentialsSecretName(jenkins)} if err := framework.Global.Client.Get(context.TODO(), namespaceName, adminSecret); err != nil { return nil, err } - jenkinsAPIURL, err := jenkinsclient.BuildJenkinsAPIUrl(jenkins.ObjectMeta.Namespace, resources.GetJenkinsHTTPServiceName(jenkins), resources.HTTPPortInt, true, true) + var service corev1.Service + + err := framework.Global.Client.Get(context.TODO(), types.NamespacedName{ + Namespace: jenkins.Namespace, + Name: resources.GetJenkinsHTTPServiceName(jenkins), + }, &service) + if err != nil { return nil, err } + jenkinsAPIURL := jenkinsclient.JenkinsAPIConnectionSettings{ + Hostname: hostname, + Port: port, + UseNodePort: useNodePort, + }.BuildJenkinsAPIUrl(service.Name, service.Namespace, service.Spec.Ports[0].Port, service.Spec.Ports[0].NodePort) + return jenkinsclient.New( jenkinsAPIURL, string(adminSecret.Data[resources.OperatorCredentialsSecretUserNameKey]), @@ -129,6 +141,10 @@ func createJenkinsCR(t *testing.T, name, namespace string, seedJob *[]v1alpha2.S NodeSelector: map[string]string{"kubernetes.io/hostname": "minikube"}, }, SeedJobs: seedJobs, + Service: v1alpha2.Service{ + Type: corev1.ServiceTypeNodePort, + Port: constants.DefaultHTTPPortInt32, + }, }, } @@ -140,8 +156,8 @@ func createJenkinsCR(t *testing.T, name, namespace string, seedJob *[]v1alpha2.S return jenkins } -func verifyJenkinsAPIConnection(t *testing.T, jenkins *v1alpha2.Jenkins) jenkinsclient.Jenkins { - client, err := createJenkinsAPIClient(jenkins) +func verifyJenkinsAPIConnection(t *testing.T, jenkins *v1alpha2.Jenkins, hostname string, port int, useNodePort bool) jenkinsclient.Jenkins { + client, err := createJenkinsAPIClient(jenkins, hostname, port, useNodePort) if err != nil { t.Fatal(err) } diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 154a6894..d0764078 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -17,14 +17,24 @@ import ( const ( jenkinsOperatorDeploymentName = constants.OperatorName seedJobConfigurationParameterName = "seed-job-config" + hostnameParameterName = "jenkins-api-hostname" + portParameterName = "jenkins-api-port" + nodePortParameterName = "jenkins-api-use-nodeport" ) var ( seedJobConfigurationFile *string + hostname *string + port *int + useNodePort *bool ) func TestMain(m *testing.M) { seedJobConfigurationFile = flag.String(seedJobConfigurationParameterName, "", "path to seed job config") + hostname = flag.String(hostnameParameterName, "", "Hostname or IP of Jenkins API. It can be service name, node IP or localhost.") + port = flag.Int(portParameterName, -1, "The port on which Jenkins API is working. Note: If you want to use nodePort don't set this setting and --jenkins-api-use-nodeport must be false.") + useNodePort = flag.Bool(nodePortParameterName, false, "Connect to Jenkins API using the nodePort instead of service port. If you want to set this as true - don't set --jenkins-api-port.") + f.MainEntry(m) } diff --git a/test/e2e/restart_test.go b/test/e2e/restart_test.go index 5f4896df..66e1f606 100644 --- a/test/e2e/restart_test.go +++ b/test/e2e/restart_test.go @@ -48,7 +48,7 @@ func TestSafeRestart(t *testing.T) { jenkins := createJenkinsCR(t, jenkinsCRName, namespace, nil, groovyScriptsConfig, v1alpha2.ConfigurationAsCode{}) waitForJenkinsBaseConfigurationToComplete(t, jenkins) waitForJenkinsUserConfigurationToComplete(t, jenkins) - jenkinsClient := verifyJenkinsAPIConnection(t, jenkins) + jenkinsClient := verifyJenkinsAPIConnection(t, jenkins, *hostname, *port, *useNodePort) checkIfAuthorizationStrategyUnsecuredIsSet(t, jenkinsClient) err := jenkinsClient.SafeRestart() diff --git a/test/e2e/restorebackup_test.go b/test/e2e/restorebackup_test.go index 07ab5f9f..0dd41f9d 100644 --- a/test/e2e/restorebackup_test.go +++ b/test/e2e/restorebackup_test.go @@ -9,6 +9,7 @@ import ( "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2" "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client" "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources" + "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants" framework "github.com/operator-framework/operator-sdk/pkg/test" "github.com/stretchr/testify/assert" @@ -31,7 +32,7 @@ func TestBackupAndRestore(t *testing.T) { jenkins := createJenkinsWithBackupAndRestoreConfigured(t, "e2e", namespace) waitForJenkinsUserConfigurationToComplete(t, jenkins) - jenkinsClient := verifyJenkinsAPIConnection(t, jenkins) + jenkinsClient := verifyJenkinsAPIConnection(t, jenkins, *hostname, *port, *useNodePort) waitForJob(t, jenkinsClient, jobID) job, err := jenkinsClient.GetJob(jobID) require.NoError(t, err, job) @@ -43,7 +44,7 @@ func TestBackupAndRestore(t *testing.T) { restartJenkinsMasterPod(t, jenkins) waitForRecreateJenkinsMasterPod(t, jenkins) waitForJenkinsUserConfigurationToComplete(t, jenkins) - jenkinsClient = verifyJenkinsAPIConnection(t, jenkins) + jenkinsClient = verifyJenkinsAPIConnection(t, jenkins, *hostname, *port, *useNodePort) waitForJob(t, jenkinsClient, jobID) verifyJobBuildsAfterRestoreBackup(t, jenkinsClient, jobID) } @@ -162,6 +163,10 @@ func createJenkinsWithBackupAndRestoreConfigured(t *testing.T, name, namespace s RepositoryURL: "https://github.com/jenkinsci/kubernetes-operator.git", }, }, + Service: v1alpha2.Service{ + Type: corev1.ServiceTypeNodePort, + Port: constants.DefaultHTTPPortInt32, + }, }, } diff --git a/test/e2e/seedjobs_test.go b/test/e2e/seedjobs_test.go index 428a38df..adc73416 100644 --- a/test/e2e/seedjobs_test.go +++ b/test/e2e/seedjobs_test.go @@ -58,7 +58,7 @@ func TestSeedJobs(t *testing.T) { waitForJenkinsBaseConfigurationToComplete(t, jenkins) verifyJenkinsMasterPodAttributes(t, jenkins) - client := verifyJenkinsAPIConnection(t, jenkins) + client := verifyJenkinsAPIConnection(t, jenkins, *hostname, *port, *useNodePort) verifyPlugins(t, client, jenkins) // user