#12 Allow configure Jenkins and Jenkins slave services

pkg/apis/jenkinsio/v1alpha1/jenkins_types.go

@@ -14,6 +14,8 @@ type JenkinsSpec struct {
 	// Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file
 	Master       JenkinsMaster `json:"master,omitempty"`
 	SeedJobs     []SeedJob     `json:"seedJobs,omitempty"`
+	Service      Service       `json:"service,omitempty"`
+	SlaveService Service       `json:"slaveService,omitempty"`
 }
 
 // JenkinsMaster defines the Jenkins master pod attributes and plugins,
@@ -29,6 +31,17 @@ type JenkinsMaster struct {
 	Plugins map[string][]string `json:"plugins,omitempty"`
 }
 
+// Service defines Kubernetes service attributes which Operator will manage
+type Service struct {
+	Annotations              map[string]string  `json:"annotations,omitempty"`
+	Labels                   map[string]string  `json:"labels,omitempty"`
+	Type                     corev1.ServiceType `json:"type,omitempty"`
+	Port                     int32              `json:"port,omitempty"`
+	NodePort                 int32              `json:"nodePort,omitempty"`
+	LoadBalancerSourceRanges []string           `json:"loadBalancerSourceRanges,omitempty"`
+	LoadBalancerIP           string             `json:"loadBalancerIP,omitempty"`
+}
+
 // JenkinsStatus defines the observed state of Jenkins
 type JenkinsStatus struct {
 	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster

pkg/apis/jenkinsio/v1alpha1/zz_generated.deepcopy.go

@@ -182,6 +182,8 @@ func (in *JenkinsSpec) DeepCopyInto(out *JenkinsSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	in.Service.DeepCopyInto(&out.Service)
+	in.SlaveService.DeepCopyInto(&out.SlaveService)
 	return
 }
 
@@ -267,3 +269,38 @@ func (in *SeedJob) DeepCopy() *SeedJob {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Service) DeepCopyInto(out *Service) {
+	*out = *in
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.LoadBalancerSourceRanges != nil {
+		in, out := &in.LoadBalancerSourceRanges, &out.LoadBalancerSourceRanges
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Service.
+func (in *Service) DeepCopy() *Service {
+	if in == nil {
+		return nil
+	}
+	out := new(Service)
+	in.DeepCopyInto(out)
+	return out
+}

pkg/controller/jenkins/client/jenkins.go

@@ -75,7 +75,7 @@ func (jenkins *jenkins) CreateOrUpdateJob(config, jobName string) (job *gojenkin
 }
 
 // BuildJenkinsAPIUrl returns Jenkins API URL
-func BuildJenkinsAPIUrl(namespace, serviceName string, portNumber int, local, minikube bool) (string, error) {
+func BuildJenkinsAPIUrl(namespace, serviceName string, portNumber int32, local, minikube bool) (string, error) {
 	// Get Jenkins URL from minikube command
 	if local && minikube {
 		cmd := exec.Command("minikube", "service", "--url", "-n", namespace, serviceName)

pkg/controller/jenkins/configuration/base/reconcile.go

@@ -136,10 +136,14 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureResourcesRequiredForJenkinsPod
 	}
 	r.logger.V(log.VDebug).Info("Service account, role and role binding are present")
 
-	if err := r.createService(metaObject); err != nil {
+	if err := r.createService(metaObject, resources.GetJenkinsHTTPServiceName(r.jenkins), r.jenkins.Spec.Service); err != nil {
 		return err
 	}
-	r.logger.V(log.VDebug).Info("Service is present")
+	r.logger.V(log.VDebug).Info("Jenkins HTTP Service is present")
+	if err := r.createService(metaObject, resources.GetJenkinsSlavesServiceName(r.jenkins), r.jenkins.Spec.SlaveService); err != nil {
+		return err
+	}
+	r.logger.V(log.VDebug).Info("Jenkins slave Service is present")
 
 	return nil
 }
@@ -296,13 +300,29 @@ func (r *ReconcileJenkinsBaseConfiguration) createRBAC(meta metav1.ObjectMeta) e
 	return nil
 }
 
-func (r *ReconcileJenkinsBaseConfiguration) createService(meta metav1.ObjectMeta) error {
+func (r *ReconcileJenkinsBaseConfiguration) createService(meta metav1.ObjectMeta, name string, config v1alpha1.Service) error {
-	err := r.createResource(resources.NewService(meta, r.minikube))
+	service := corev1.Service{}
-	if err != nil && !apierrors.IsAlreadyExists(err) {
+	err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: meta.Namespace}, &service)
+	if err != nil && errors.IsNotFound(err) {
+		service = resources.UpdateService(corev1.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      name,
+				Namespace: meta.Namespace,
+				Labels:    meta.Labels,
+			},
+			Spec: corev1.ServiceSpec{
+				Selector: meta.Labels,
+			},
+		}, config)
+		if err = r.createResource(&service); err != nil {
+			return stackerr.WithStack(err)
+		}
+	} else if err != nil {
 		return stackerr.WithStack(err)
 	}
 
-	return nil
+	service = resources.UpdateService(service, config)
+	return stackerr.WithStack(r.updateResource(&service))
 }
 
 func (r *ReconcileJenkinsBaseConfiguration) getJenkinsMasterPod(meta metav1.ObjectMeta) (*corev1.Pod, error) {
@@ -425,7 +445,7 @@ func (r *ReconcileJenkinsBaseConfiguration) waitForJenkins(meta metav1.ObjectMet
 
 func (r *ReconcileJenkinsBaseConfiguration) ensureJenkinsClient(meta metav1.ObjectMeta) (jenkinsclient.Jenkins, error) {
 	jenkinsURL, err := jenkinsclient.BuildJenkinsAPIUrl(
-		r.jenkins.ObjectMeta.Namespace, meta.Name, resources.HTTPPortInt, r.local, r.minikube)
+		r.jenkins.ObjectMeta.Namespace, resources.GetJenkinsHTTPServiceName(r.jenkins), r.jenkins.Spec.Service.Port, r.local, r.minikube)
 	if err != nil {
 		return nil, err
 	}

pkg/controller/jenkins/configuration/base/resources/base_configuration_configmap.go

@@ -115,13 +115,21 @@ ServiceAccountCredential serviceAccountCredential = new ServiceAccountCredential
 )
 SystemCredentialsProvider.getInstance().getStore().addCredentials(Domain.global(), serviceAccountCredential)
 
-KubernetesCloud kubernetes = new KubernetesCloud("kubernetes")
+def kubernetes = Jenkins.instance.clouds.getByName("kubernetes")
+def add = false
+if (kubernetes == null) {
+    add = true
+	kubernetes = new KubernetesCloud("kubernetes")
+}
 kubernetes.setServerUrl("https://kubernetes.default")
 kubernetes.setNamespace("%s")
 kubernetes.setCredentialsId(kubernetesCredentialsId)
-kubernetes.setJenkinsUrl("http://%s:%d")
+kubernetes.setJenkinsUrl("%s")
+kubernetes.setJenkinsTunnel("%s")
 kubernetes.setRetentionTimeout(15)
+if (add) {
 	jenkins.clouds.add(kubernetes)
+}
 
 jenkins.save()
 `
@@ -176,7 +184,10 @@ func NewBaseConfigurationConfigMap(meta metav1.ObjectMeta, jenkins *v1alpha1.Jen
 			"4-enable-master-access-control.groovy": enableMasterAccessControl,
 			"5-disable-insecure-features.groovy":    disableInsecureFeatures,
 			"6-configure-kubernetes-plugin.groovy": fmt.Sprintf(configureKubernetesPluginFmt,
-				jenkins.ObjectMeta.Namespace, GetResourceName(jenkins), HTTPPortInt),
+				jenkins.ObjectMeta.Namespace,
+				fmt.Sprintf("http://%s.%s:%d", GetJenkinsHTTPServiceName(jenkins), jenkins.ObjectMeta.Namespace, jenkins.Spec.Service.Port),
+				fmt.Sprintf("%s.%s:%d", GetJenkinsSlavesServiceName(jenkins), jenkins.ObjectMeta.Namespace, jenkins.Spec.SlaveService.Port),
+			),
 			"7-configure-views.groovy": configureViews,
 		},
 	}

pkg/controller/jenkins/configuration/base/resources/pod.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -43,9 +44,6 @@ const (
 	slavePortName = "slavelistener"
 	// HTTPPortInt defines Jenkins master HTTP port
 	HTTPPortInt = 8080
-	slavePortInt   = 50000
-	httpPortInt32  = int32(8080)
-	slavePortInt32 = int32(50000)
 
 	jenkinsUserUID = int64(1000) // build in Docker image jenkins user UID
 )
@@ -109,12 +107,12 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
 					},
 					Ports: []corev1.ContainerPort{
 						{
-							Name:          slavePortName,
+							Name:          httpPortName,
-							ContainerPort: slavePortInt32,
+							ContainerPort: constants.DefaultHTTPPortInt32,
 						},
 						{
-							Name:          httpPortName,
+							Name:          slavePortName,
-							ContainerPort: httpPortInt32,
+							ContainerPort: constants.DefaultSlavePortInt32,
 						},
 					},
 					Env: []corev1.EnvVar{

pkg/controller/jenkins/configuration/base/resources/service.go

@@ -1,9 +1,13 @@
 package resources
 
 import (
+	"fmt"
+
+	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
+
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
 func buildServiceTypeMeta() metav1.TypeMeta {
@@ -13,37 +17,32 @@ func buildServiceTypeMeta() metav1.TypeMeta {
 	}
 }
 
-// NewService builds the Kubernetes service resource
+// UpdateService returns new service with override fields from config
-func NewService(meta metav1.ObjectMeta, minikube bool) *corev1.Service {
+func UpdateService(actual corev1.Service, config v1alpha1.Service) corev1.Service {
-	service := &corev1.Service{
+	actual.ObjectMeta.Annotations = config.Annotations
-		TypeMeta:   buildServiceTypeMeta(),
+	for key, value := range config.Labels {
-		ObjectMeta: meta,
+		actual.ObjectMeta.Labels[key] = value
-		Spec: corev1.ServiceSpec{
+	}
-			Selector: meta.Labels,
+	actual.Spec.Type = config.Type
-			// The first port have to be Jenkins http port because when run with minikube
+	actual.Spec.LoadBalancerIP = config.LoadBalancerIP
-			// command 'minikube service' returns endpoints in the same sequence
+	actual.Spec.LoadBalancerSourceRanges = config.LoadBalancerSourceRanges
-			Ports: []corev1.ServicePort{
+	if len(actual.Spec.Ports) == 0 {
-				{
+		actual.Spec.Ports = []corev1.ServicePort{{}}
-					Name:       httpPortName,
+	}
-					Port:       httpPortInt32,
+	actual.Spec.Ports[0].Port = config.Port
-					TargetPort: intstr.FromInt(HTTPPortInt),
+	if config.NodePort != 0 {
-				},
+		actual.Spec.Ports[0].NodePort = config.NodePort
-				{
-					Name:       slavePortName,
-					Port:       slavePortInt32,
-					TargetPort: intstr.FromInt(slavePortInt),
-				},
-			},
-		},
 	}
 
-	if minikube {
+	return actual
-		// When running locally with minikube cluster Jenkins Service have to be exposed via node port
-		// to allow communication operator -> Jenkins API
-		service.Spec.Type = corev1.ServiceTypeNodePort
-	} else {
-		service.Spec.Type = corev1.ServiceTypeClusterIP
 }
 
-	return service
+// GetJenkinsHTTPServiceName returns Kubernetes service name used for expose Jenkins HTTP endpoint
+func GetJenkinsHTTPServiceName(jenkins *v1alpha1.Jenkins) string {
+	return fmt.Sprintf("%s-http-%s", constants.OperatorName, jenkins.ObjectMeta.Name)
+}
+
+// GetJenkinsSlavesServiceName returns Kubernetes service name used for expose Jenkins slave endpoint
+func GetJenkinsSlavesServiceName(jenkins *v1alpha1.Jenkins) string {
+	return fmt.Sprintf("%s-slave-%s", constants.OperatorName, jenkins.ObjectMeta.Name)
 }

pkg/controller/jenkins/constants/constants.go

@@ -13,4 +13,8 @@ const (
 	UserConfigurationJobName = OperatorName + "-user-configuration"
 	// UserConfigurationCASCJobName is the Jenkins job name used to configure Jenkins by Configuration as code yaml configs provided by user
 	UserConfigurationCASCJobName = OperatorName + "-user-configuration-casc"
+	// DefaultHTTPPortInt32 is the default Jenkins HTTP port
+	DefaultHTTPPortInt32 = int32(8080)
+	// DefaultSlavePortInt32 is the default Jenkins port for slaves
+	DefaultSlavePortInt32 = int32(50000)
 )

pkg/controller/jenkins/jenkins_controller.go

@@ -3,6 +3,7 @@ package jenkins
 import (
 	"context"
 	"fmt"
+	"reflect"
 
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base"
@@ -246,6 +247,30 @@ func (r *ReconcileJenkins) setDefaults(jenkins *v1alpha1.Jenkins, logger logr.Lo
 			},
 		}
 	}
+	if reflect.DeepEqual(jenkins.Spec.Service, v1alpha1.Service{}) {
+		logger.Info("Setting default Jenkins master service")
+		changed = true
+		var serviceType corev1.ServiceType
+		if r.minikube {
+			// When running locally with minikube cluster Jenkins Service have to be exposed via node port
+			// to allow communication operator -> Jenkins API
+			serviceType = corev1.ServiceTypeNodePort
+		} else {
+			serviceType = corev1.ServiceTypeClusterIP
+		}
+		jenkins.Spec.Service = v1alpha1.Service{
+			Type: serviceType,
+			Port: constants.DefaultHTTPPortInt32,
+		}
+	}
+	if reflect.DeepEqual(jenkins.Spec.SlaveService, v1alpha1.Service{}) {
+		logger.Info("Setting default Jenkins slave service")
+		changed = true
+		jenkins.Spec.SlaveService = v1alpha1.Service{
+			Type: corev1.ServiceTypeClusterIP,
+			Port: constants.DefaultSlavePortInt32,
+		}
+	}
 
 	if changed {
 		return errors.WithStack(r.client.Update(context.TODO(), jenkins))

test/e2e/jenkins.go

@@ -46,7 +46,7 @@ func createJenkinsAPIClient(jenkins *v1alpha1.Jenkins) (jenkinsclient.Jenkins, e
 		return nil, err
 	}
 
-	jenkinsAPIURL, err := jenkinsclient.BuildJenkinsAPIUrl(jenkins.ObjectMeta.Namespace, resources.GetResourceName(jenkins), resources.HTTPPortInt, true, true)
+	jenkinsAPIURL, err := jenkinsclient.BuildJenkinsAPIUrl(jenkins.ObjectMeta.Namespace, resources.GetJenkinsHTTPServiceName(jenkins), resources.HTTPPortInt, true, true)
 	if err != nil {
 		return nil, err
 	}

test/e2e/wait.go

@@ -14,6 +14,7 @@ import (
 
 	framework "github.com/operator-framework/operator-sdk/pkg/test"
 	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -35,10 +36,13 @@ func waitForJenkinsBaseConfigurationToComplete(t *testing.T, jenkins *v1alpha1.J
 		t.Logf("Current Jenkins status '%+v'", jenkins.Status)
 		return jenkins.Status.BaseConfigurationCompletedTime != nil
 	})
-	if err != nil {
+	assert.NoError(t, err)
-		t.Fatal(err)
-	}
 	t.Log("Jenkins pod is running")
+
+	// update jenkins CR because Operator sets default values
+	namespacedName := types.NamespacedName{Namespace: jenkins.Namespace, Name: jenkins.Name}
+	err = framework.Global.Client.Get(goctx.TODO(), namespacedName, jenkins)
+	assert.NoError(t, err)
 }
 
 func waitForRecreateJenkinsMasterPod(t *testing.T, jenkins *v1alpha1.Jenkins) {