Improve e2e tests

Gopkg.lock

@@ -368,6 +368,14 @@
   revision = "5f041e8faa004a95c88a202771f4cc3e991971e6"
   version = "v2.0.1"
 
+[[projects]]
+  digest = "1:14715f705ff5dfe0ffd6571d7d201dd8e921030f8070321a79380d8ca4ec1a24"
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  pruneopts = "NT"
+  revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
+  version = "v0.8.1"
+
 [[projects]]
   digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
   name = "github.com/pmezard/go-difflib"
@@ -872,6 +880,7 @@
     "github.com/operator-framework/operator-sdk/pkg/test",
     "github.com/operator-framework/operator-sdk/pkg/test/e2eutil",
     "github.com/operator-framework/operator-sdk/version",
+    "github.com/pkg/errors",
     "github.com/stretchr/testify/assert",
     "k8s.io/api/core/v1",
     "k8s.io/api/rbac/v1",

Makefile

@@ -144,15 +144,12 @@ test: ## Runs the go tests
 CURRENT_DIRECTORY := $(shell pwd)
 e2e: ## Runs e2e tests
 	@echo "+ $@"
-	@echo "E2E_IMAGE: $(E2E_IMAGE)"
+	@echo "Docker image: $(REPO):$(GITCOMMIT)"
-ifeq ($(E2E_IMAGE),)
-	$(error You must provide an image to e2e tests)
-endif
 	cp deploy/service_account.yaml deploy/namespace-init.yaml
 	cat deploy/role.yaml >> deploy/namespace-init.yaml
 	cat deploy/role_binding.yaml >> deploy/namespace-init.yaml
 	cat deploy/operator.yaml >> deploy/namespace-init.yaml
-	sed -i 's|REPLACE_IMAGE|$(E2E_IMAGE)|g' deploy/namespace-init.yaml
+	sed -i 's|REPLACE_IMAGE|$(REPO):$(GITCOMMIT)|g' deploy/namespace-init.yaml
 ifeq ($(ENVIRONMENT),minikube)
 	sed -i 's|imagePullPolicy: IfNotPresent|imagePullPolicy: Never|g' deploy/namespace-init.yaml
 	sed -i 's|REPLACE_ARGS|args: ["--minikube"]|g' deploy/namespace-init.yaml
@@ -160,7 +157,7 @@ else
 	sed -i 's|REPLACE_ARGS||g' deploy/namespace-init.yaml
 endif
 
-	@RUNNING_TESTS=1 go test -parallel=1 "./test/e2e/" -tags "$(BUILDTAGS) cgo" -v \
+	@RUNNING_TESTS=1 go test -parallel=1 "./test/e2e/" -tags "$(BUILDTAGS) cgo" -v -timeout 30m \
 		-root=$(CURRENT_DIRECTORY) -kubeconfig=$(HOME)/.kube/config -globalMan deploy/crds/virtuslab_v1alpha1_jenkins_crd.yaml -namespacedMan deploy/namespace-init.yaml
 
 .PHONY: vet
@@ -261,12 +258,6 @@ docker-build: check-env ## Build the container
 	@echo "+ $@"
 	docker build . -t $(REPO):$(GITCOMMIT) --file build/Dockerfile
 
-.PHONY: docker-build-e2e
-docker-build-e2e: check-env ## Build the container for e2e tests
-	@echo "+ $@"
-	eval $(minikube docker-env)
-	docker build . -t $(REPO):$(GITCOMMIT) --file build/Dockerfile
-
 .PHONY: docker-images
 docker-images: ## List all local containers
 	@echo "+ $@"
@@ -328,7 +319,7 @@ deepcopy-gen: ## Generate deepcopy golang code
 start-minikube: ## Start minikube
 	@echo "+ $@"
 	@minikube status && exit 0 || \
-	minikube start --kubernetes-version $(MINIKUBE_KUBERNETES_VERSION) --vm-driver=$(MINIKUBE_DRIVER) --memory 4096
+	minikube start --kubernetes-version $(MINIKUBE_KUBERNETES_VERSION) --vm-driver=$(MINIKUBE_DRIVER) --memory 4096 --cpus 3
 
 .PHONY: bump-version
 BUMP := patch
@@ -347,7 +338,7 @@ bump-version: ## Bump the version in the version file. Set BUMP to [ patch | maj
 .PHONY: tag
 tag: ## Create a new git tag to prepare to build a release
 	@echo "+ $@"
-	git tag -a $(VERSION) -m "$(VERSION)"
+	git tag -s -a $(VERSION) -m "$(VERSION)"
 	git push origin $(VERSION)
 
 .PHONY: help

docs/developer-guide.md

@@ -50,8 +50,9 @@ Run e2e tests with minikube:
 ```bash
 make start-minikube
 eval $(minikube docker-env)
-make docker-build-e2e
+make build
-make e2e E2E_IMAGE=jenkins-operator
+make docker-build
+make e2e
 ```
 
 ## Tips & Tricks

test/e2e/jenkins.go

@@ -123,11 +123,11 @@ func createJenkinsCRWithSeedJob(t *testing.T, namespace string) *virtuslabv1alph
 			//TODO(bantoniak) add seed job with private key
 			SeedJobs: []virtuslabv1alpha1.SeedJob{
 				{
-					ID:               "jenkins-operator-e2e",
+					ID:               "jenkins-operator",
 					Targets:          "cicd/jobs/*.jenkins",
-					Description:      "Jenkins Operator e2e tests repository",
+					Description:      "Jenkins Operator repository",
 					RepositoryBranch: "master",
-					RepositoryURL:    "https://github.com/VirtusLab/jenkins-operator-e2e.git",
+					RepositoryURL:    "https://github.com/VirtusLab/jenkins-operator.git",
 				},
 			},
 		},
@@ -150,3 +150,13 @@ func verifyJenkinsAPIConnection(t *testing.T, jenkins *virtuslabv1alpha1.Jenkins
 	t.Log("I can establish connection to Jenkins API")
 	return client
 }
+
+func restartJenkinsMasterPod(t *testing.T, jenkins *virtuslabv1alpha1.Jenkins) {
+	t.Log("Restarting Jenkins master pod")
+	jenkinsPod := getJenkinsMasterPod(t, jenkins)
+	err := framework.Global.Client.Delete(context.TODO(), jenkinsPod)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("Jenkins master pod has been restarted")
+}

test/e2e/restart_pod_test.go

@@ -35,11 +35,3 @@ func checkBaseConfigurationCompleteTimeIsNotSet(t *testing.T, jenkins *virtuslab
 		t.Fatalf("Status.BaseConfigurationCompletedTime is set after restart of pod, status %+v", jenkinsStatus.Status)
 	}
 }
-
-func restartJenkinsMasterPod(t *testing.T, jenkins *virtuslabv1alpha1.Jenkins) {
-	jenkinsPod := getJenkinsMasterPod(t, jenkins)
-	err := framework.Global.Client.Delete(context.TODO(), jenkinsPod)
-	if err != nil {
-		t.Fatal(err)
-	}
-}

test/e2e/user_configuration_test.go

@@ -32,41 +32,34 @@ func TestUserConfiguration(t *testing.T) {
 }
 
 func verifyJenkinsSeedJobs(t *testing.T, client *gojenkins.Jenkins, jenkins *virtuslabv1alpha1.Jenkins) {
-	// check if job has been configured and executed successfully
-	err := wait.Poll(time.Second*10, time.Minute*2, func() (bool, error) {
 	t.Logf("Attempting to get configure seed job status '%v'", seedjobs.ConfigureSeedJobsName)
-		seedJob, err := client.GetJob(seedjobs.ConfigureSeedJobsName)
+	configureSeedJobs, err := client.GetJob(seedjobs.ConfigureSeedJobsName)
-		if err != nil || seedJob == nil {
+	assert.NoError(t, err)
-			return false, nil
+	assert.NotNil(t, configureSeedJobs)
-		}
+	build, err := configureSeedJobs.GetLastSuccessfulBuild()
-		build, err := seedJob.GetLastSuccessfulBuild()
+	assert.NotNil(t, build)
-		if err != nil || build == nil {
-			return false, nil
-		}
-		return true, nil
-	})
-	assert.NoError(t, err, "couldn't get jenkins job")
 
-	// WARNING this use case depends on changes in https://github.com/VirtusLab/jenkins-operator-e2e/tree/master/cicd
+	seedJobName := "jenkins-operator-configure-seed-job"
-	seedJobName := "jenkins-operator-e2e-job-dsl-seed" // https://github.com/VirtusLab/jenkins-operator-e2e/blob/master/cicd/jobs/e2e_test_job.jenkins
-	err = wait.Poll(time.Second*10, time.Minute*2, func() (bool, error) {
 	t.Logf("Attempting to verify if seed job has been created '%v'", seedJobName)
 	seedJob, err := client.GetJob(seedJobName)
+	assert.NoError(t, err)
+	assert.NotNil(t, seedJob)
+	build, err = seedJob.GetLastSuccessfulBuild()
+	assert.NotNil(t, build)
+
+	err = framework.Global.Client.Get(context.TODO(), types.NamespacedName{Namespace: jenkins.Namespace, Name: jenkins.Name}, jenkins)
+	assert.NoError(t, err, "couldn't get jenkins custom resource")
+	assert.NotNil(t, jenkins.Status.Builds)
+	assert.NotEmpty(t, jenkins.Status.Builds)
+
+	jobCreatedByDSLPluginName := "build-jenkins-operator"
+	err = wait.Poll(time.Second*10, time.Minute*2, func() (bool, error) {
+		t.Logf("Attempting to verify if job '%s' has been created ", jobCreatedByDSLPluginName)
+		seedJob, err := client.GetJob(jobCreatedByDSLPluginName)
 		if err != nil || seedJob == nil {
 			return false, nil
 		}
 		return true, nil
 	})
-	assert.NoError(t, err, "couldn't verify if seed job has been created")
+	assert.NoError(t, err)
-
-	// verify Jenkins.Status.Builds
-	// WARNING this use case depends on changes in https://github.com/VirtusLab/jenkins-operator-e2e/tree/master/cicd
-	err = framework.Global.Client.Get(context.TODO(), types.NamespacedName{Namespace: jenkins.Namespace, Name: jenkins.Name}, jenkins)
-	assert.NoError(t, err, "couldn't get jenkins custom resource")
-
-	assert.NotNil(t, jenkins.Status.Builds)
-	assert.NotEmpty(t, jenkins.Status.Builds)
-	assert.Equal(t, len(jenkins.Status.Builds), 1)
-	build := jenkins.Status.Builds[0]
-	assert.Equal(t, build.JobName, seedjobs.ConfigureSeedJobsName)
 }

test/e2e/wait.go

@@ -26,7 +26,7 @@ type checkConditionFunc func(*virtuslabv1alpha1.Jenkins) bool
 
 func waitForJenkinsBaseConfigurationToComplete(t *testing.T, jenkins *virtuslabv1alpha1.Jenkins) {
 	t.Log("Waiting for Jenkins base configuration to complete")
-	_, err := WaitUntilJenkinsConditionTrue(retryInterval, 50, jenkins, func(jenkins *virtuslabv1alpha1.Jenkins) bool {
+	_, err := WaitUntilJenkinsConditionTrue(retryInterval, 150, jenkins, func(jenkins *virtuslabv1alpha1.Jenkins) bool {
 		t.Logf("Current Jenkins status '%+v'", jenkins.Status)
 		return jenkins.Status.BaseConfigurationCompletedTime != nil
 	})
@@ -45,13 +45,16 @@ func waitForRecreateJenkinsMasterPod(t *testing.T, jenkins *virtuslabv1alpha1.Je
 		if err != nil {
 			return false, err
 		}
+		if len(podList.Items) != 1 {
+			return false, nil
+		}
 
 		return podList.Items[0].DeletionTimestamp == nil, nil
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	t.Log("Jenkins pod has bee recreated")
+	t.Log("Jenkins pod has been recreated")
 }
 
 func waitForJenkinsUserConfigurationToComplete(t *testing.T, jenkins *virtuslabv1alpha1.Jenkins) {