diff --git a/test/e2e/configuration_test.go b/test/e2e/configuration_test.go index a8f1b319..b102a5ac 100644 --- a/test/e2e/configuration_test.go +++ b/test/e2e/configuration_test.go @@ -89,13 +89,14 @@ func TestConfiguration(t *testing.T) { createKubernetesCredentialsProviderSecret(t, namespace, mySeedJob) waitForJenkinsBaseConfigurationToComplete(t, jenkins) verifyJenkinsMasterPodAttributes(t, jenkins) - client := verifyJenkinsAPIConnection(t, jenkins, *hostname, *port, *useNodePort) - verifyPlugins(t, client, jenkins) + jenkinsClient, cleanUpFunc := verifyJenkinsAPIConnection(t, jenkins, namespace) + defer cleanUpFunc() + verifyPlugins(t, jenkinsClient, jenkins) // user waitForJenkinsUserConfigurationToComplete(t, jenkins) - verifyUserConfiguration(t, client, numberOfExecutors, systemMessage) - verifyJenkinsSeedJobs(t, client, []seedJobConfig{mySeedJob}) + verifyUserConfiguration(t, jenkinsClient, numberOfExecutors, systemMessage) + verifyJenkinsSeedJobs(t, jenkinsClient, []seedJobConfig{mySeedJob}) } func TestPlugins(t *testing.T) { @@ -121,7 +122,8 @@ func TestPlugins(t *testing.T) { jenkins := createJenkinsCR(t, "k8s-e2e", namespace, seedJobs, v1alpha2.GroovyScripts{}, v1alpha2.ConfigurationAsCode{}) waitForJenkinsUserConfigurationToComplete(t, jenkins) - jenkinsClient := verifyJenkinsAPIConnection(t, jenkins, *hostname, *port, *useNodePort) + jenkinsClient, cleanUpFunc := verifyJenkinsAPIConnection(t, jenkins, namespace) + defer cleanUpFunc() waitForJob(t, jenkinsClient, jobID) job, err := jenkinsClient.GetJob(jobID) diff --git a/test/e2e/jenkins.go b/test/e2e/jenkins.go index c09d5ab1..1b6326f7 100644 --- a/test/e2e/jenkins.go +++ b/test/e2e/jenkins.go @@ -46,7 +46,7 @@ func getJenkinsMasterPod(t *testing.T, jenkins *v1alpha2.Jenkins) *corev1.Pod { return &podList.Items[0] } -func createJenkinsAPIClient(jenkins *v1alpha2.Jenkins, hostname string, port int, useNodePort bool) (jenkinsclient.Jenkins, error) { +func createJenkinsAPIClient(jenkins *v1alpha2.Jenkins, hostname string, port int) (jenkinsclient.Jenkins, error) { adminSecret := &corev1.Secret{} namespaceName := types.NamespacedName{Namespace: jenkins.Namespace, Name: resources.GetOperatorCredentialsSecretName(jenkins)} if err := framework.Global.Client.Get(context.TODO(), namespaceName, adminSecret); err != nil { @@ -67,7 +67,7 @@ func createJenkinsAPIClient(jenkins *v1alpha2.Jenkins, hostname string, port int jenkinsAPIURL := jenkinsclient.JenkinsAPIConnectionSettings{ Hostname: hostname, Port: port, - UseNodePort: useNodePort, + UseNodePort: false, }.BuildJenkinsAPIUrl(service.Name, service.Namespace, service.Spec.Ports[0].Port, service.Spec.Ports[0].NodePort) return jenkinsclient.NewUserAndPasswordAuthorization( @@ -163,14 +163,22 @@ func createJenkinsCR(t *testing.T, name, namespace string, seedJob *[]v1alpha2.S return jenkins } -func verifyJenkinsAPIConnection(t *testing.T, jenkins *v1alpha2.Jenkins, hostname string, port int, useNodePort bool) jenkinsclient.Jenkins { - client, err := createJenkinsAPIClient(jenkins, hostname, port, useNodePort) +func verifyJenkinsAPIConnection(t *testing.T, jenkins *v1alpha2.Jenkins, namespace string) (jenkinsclient.Jenkins, func()) { + podName := resources.GetJenkinsMasterPodName(*jenkins) + port, cleanUpFunc, waitFunc, portForwardFunc, err := setupPortForwardToPod(t, namespace, podName, int(constants.DefaultHTTPPortInt32)) if err != nil { t.Fatal(err) } + go portForwardFunc() + waitFunc() + client, err := createJenkinsAPIClient(jenkins, "localhost", port) + if err != nil { + defer cleanUpFunc() + t.Fatal(err) + } t.Log("I can establish connection to Jenkins API") - return client + return client, cleanUpFunc } func restartJenkinsMasterPod(t *testing.T, jenkins *v1alpha2.Jenkins) { diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 2a9adc75..c697caae 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -16,23 +16,14 @@ import ( const ( jenkinsOperatorDeploymentName = constants.OperatorName seedJobConfigurationParameterName = "seed-job-config" - hostnameParameterName = "jenkins-api-hostname" - portParameterName = "jenkins-api-port" - nodePortParameterName = "jenkins-api-use-nodeport" ) var ( seedJobConfigurationFile *string - hostname *string - port *int - useNodePort *bool ) func TestMain(m *testing.M) { seedJobConfigurationFile = flag.String(seedJobConfigurationParameterName, "", "path to seed job config") - hostname = flag.String(hostnameParameterName, "", "Hostname or IP of Jenkins API. It can be service name, node IP or localhost.") - port = flag.Int(portParameterName, -1, "The port on which Jenkins API is working. Note: If you want to use nodePort don't set this setting and --jenkins-api-use-nodeport must be false.") - useNodePort = flag.Bool(nodePortParameterName, false, "Connect to Jenkins API using the nodePort instead of service port. If you want to set this as true - don't set --jenkins-api-port.") framework.MainEntry(m) } diff --git a/test/e2e/port_forward.go b/test/e2e/port_forward.go new file mode 100644 index 00000000..49efcd78 --- /dev/null +++ b/test/e2e/port_forward.go @@ -0,0 +1,122 @@ +package e2e + +import ( + "fmt" + "net" + "net/http" + "net/url" + "os" + "strings" + "testing" + + framework "github.com/operator-framework/operator-sdk/pkg/test" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" +) + +type portForwardToPodRequest struct { + // config is the kubernetes config + config *rest.Config + // pod is the selected pod for this port forwarding + pod v1.Pod + // localPort is the local port that will be selected to expose the podPort + localPort int + // podPort is the target port for the pod + podPort int + // Steams configures where to write or read input from + streams genericclioptions.IOStreams + // stopCh is the channel used to manage the port forward lifecycle + stopCh <-chan struct{} + // readyCh communicates when the tunnel is ready to receive traffic + readyCh chan struct{} +} + +func getFreePort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", ":0") + if err != nil { + return 0, err + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return 0, err + } + _ = l.Close() + return l.Addr().(*net.TCPAddr).Port, nil +} + +func setupPortForwardToPod(t *testing.T, namespace, podName string, podPort int) (port int, cleanUpFunc func(), waitFunc func(), portForwardFunc func(), err error) { + port, err = getFreePort() + if err != nil { + t.Fatal(err) + } + + stream := genericclioptions.IOStreams{ + In: os.Stdin, + Out: os.Stdout, + ErrOut: os.Stderr, + } + + // stopCh control the port forwarding lifecycle. When it gets closed the + // port forward will terminate + stopCh := make(chan struct{}, 1) + // readyCh communicate when the port forward is ready to get traffic + readyCh := make(chan struct{}) + + req := portForwardToPodRequest{ + config: framework.Global.KubeConfig, + pod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: namespace, + }, + }, + localPort: port, + podPort: podPort, + streams: stream, + stopCh: stopCh, + readyCh: readyCh, + } + + waitFunc = func() { + t.Log("Waiting for the port-forward.") + <-readyCh + t.Log("The port-forward is established.") + } + + portForwardFunc = func() { + err := portForwardToPod(req) + if err != nil { + panic(err) + } + } + + cleanUpFunc = func() { + t.Log("Closing port-forward") + close(stopCh) + } + + return +} + +func portForwardToPod(req portForwardToPodRequest) error { + path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", + req.pod.Namespace, req.pod.Name) + hostIP := strings.TrimLeft(req.config.Host, "htps:/") + + transport, upgrader, err := spdy.RoundTripperFor(req.config) + if err != nil { + return err + } + + dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, http.MethodPost, &url.URL{Scheme: "https", Path: path, Host: hostIP}) + fw, err := portforward.New(dialer, []string{fmt.Sprintf("%d:%d", req.localPort, req.podPort)}, req.stopCh, req.readyCh, req.streams.Out, req.streams.ErrOut) + if err != nil { + return err + } + return fw.ForwardPorts() +} diff --git a/test/e2e/restart_test.go b/test/e2e/restart_test.go index 66e1f606..4f586a5c 100644 --- a/test/e2e/restart_test.go +++ b/test/e2e/restart_test.go @@ -48,7 +48,8 @@ func TestSafeRestart(t *testing.T) { jenkins := createJenkinsCR(t, jenkinsCRName, namespace, nil, groovyScriptsConfig, v1alpha2.ConfigurationAsCode{}) waitForJenkinsBaseConfigurationToComplete(t, jenkins) waitForJenkinsUserConfigurationToComplete(t, jenkins) - jenkinsClient := verifyJenkinsAPIConnection(t, jenkins, *hostname, *port, *useNodePort) + jenkinsClient, cleanUpFunc := verifyJenkinsAPIConnection(t, jenkins, namespace) + defer cleanUpFunc() checkIfAuthorizationStrategyUnsecuredIsSet(t, jenkinsClient) err := jenkinsClient.SafeRestart() diff --git a/test/e2e/restorebackup_test.go b/test/e2e/restorebackup_test.go index 0dd41f9d..b1db9c0d 100644 --- a/test/e2e/restorebackup_test.go +++ b/test/e2e/restorebackup_test.go @@ -32,7 +32,8 @@ func TestBackupAndRestore(t *testing.T) { jenkins := createJenkinsWithBackupAndRestoreConfigured(t, "e2e", namespace) waitForJenkinsUserConfigurationToComplete(t, jenkins) - jenkinsClient := verifyJenkinsAPIConnection(t, jenkins, *hostname, *port, *useNodePort) + jenkinsClient, cleanUpFunc := verifyJenkinsAPIConnection(t, jenkins, namespace) + defer cleanUpFunc() waitForJob(t, jenkinsClient, jobID) job, err := jenkinsClient.GetJob(jobID) require.NoError(t, err, job) @@ -44,9 +45,10 @@ func TestBackupAndRestore(t *testing.T) { restartJenkinsMasterPod(t, jenkins) waitForRecreateJenkinsMasterPod(t, jenkins) waitForJenkinsUserConfigurationToComplete(t, jenkins) - jenkinsClient = verifyJenkinsAPIConnection(t, jenkins, *hostname, *port, *useNodePort) - waitForJob(t, jenkinsClient, jobID) - verifyJobBuildsAfterRestoreBackup(t, jenkinsClient, jobID) + jenkinsClient2, cleanUpFunc2 := verifyJenkinsAPIConnection(t, jenkins, namespace) + defer cleanUpFunc2() + waitForJob(t, jenkinsClient2, jobID) + verifyJobBuildsAfterRestoreBackup(t, jenkinsClient2, jobID) } func waitForJob(t *testing.T, jenkinsClient client.Jenkins, jobID string) { diff --git a/test/e2e/seedjobs_test.go b/test/e2e/seedjobs_test.go index adc73416..4ac6e45f 100644 --- a/test/e2e/seedjobs_test.go +++ b/test/e2e/seedjobs_test.go @@ -58,12 +58,13 @@ func TestSeedJobs(t *testing.T) { waitForJenkinsBaseConfigurationToComplete(t, jenkins) verifyJenkinsMasterPodAttributes(t, jenkins) - client := verifyJenkinsAPIConnection(t, jenkins, *hostname, *port, *useNodePort) - verifyPlugins(t, client, jenkins) + jenkinsClient, cleanUpFunc := verifyJenkinsAPIConnection(t, jenkins, namespace) + defer cleanUpFunc() + verifyPlugins(t, jenkinsClient, jenkins) // user waitForJenkinsUserConfigurationToComplete(t, jenkins) - verifyJenkinsSeedJobs(t, client, seedJobsConfig.SeedJobs) + verifyJenkinsSeedJobs(t, jenkinsClient, seedJobsConfig.SeedJobs) } func loadSeedJobsConfig(t *testing.T) seedJobsConfig {