feat: make ginkgo tests as matrix (#979)

- make ginkgo tests as matrix for easily re-run failing tests or checking logs
- fix pod restart check

.github/workflows/auto-tests-bats.yaml

@@ -1,4 +1,4 @@
-name: Run bats tests
+name: Tests BATS
 on:
   push:
     branches:
@@ -22,7 +22,7 @@ on:
 jobs:
   run-tests:
     if: github.event.pull_request.draft == false
-    name: Run automated bats tests
+    name: BATS Run tests
     runs-on: ubuntu-latest
     steps:
       - name: Check out code

.github/workflows/auto-tests-e2e.yaml

@@ -1,4 +1,4 @@
-name: Run e2e tests
+name: Tests E2E
 on:
   push:
     branches:
@@ -20,20 +20,29 @@ on:
       - '*.md'
 
 jobs:
-  run-tests:
+  create-e2e-list:
-    if: github.event.pull_request.draft == false
+    name: E2E Create tests list
-    name: Run automated tests
     runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
     steps:
-      - name: Check out code
+      - uses: actions/checkout@v4
-        uses: actions/checkout@v4
+      - id: matrix
+        run: |
+          script=$(./test/make_matrix_ginkgo.sh e2e)
+          echo "matrix=${script}" >> $GITHUB_OUTPUT
+
+  verify-code:
+    name: E2E Verify code before tests
+    runs-on: ubuntu-latest
+    needs: [create-e2e-list]
+    steps:
+      - uses: actions/checkout@v4
 
       - name: Set up env vars
         run: |
           echo "GO111MODULE=on" >> $GITHUB_ENV
           echo "GO_VERSION=v$(sed -n 's/GO_VERSION=//p' config.base.env | tr -d '\n' | tr -d '"')" >> $GITHUB_ENV
-          echo "HELM_VERSION=v$(sed -n 's/HELM_VERSION=//p' config.base.env | tr -d '\n' | tr -d '"')" >> $GITHUB_ENV
-          echo "KIND_CLUSTER_NAME=$(sed -n 's/KIND_CLUSTER_NAME=//p' config.base.env | tr -d '\n' | tr -d '"')" >> $GITHUB_ENV
           echo "GOPATH=/home/runner/go" >> $GITHUB_ENV
 
       - name: Prepare go environment
@@ -47,6 +56,33 @@ jobs:
       - name: Verify code formatting
         run: make verify
 
+  run-e2e-tests:
+    runs-on: ubuntu-latest
+    needs: [create-e2e-list, verify-code]
+    if: github.event.pull_request.draft == false
+    name: E2E ${{ matrix.test }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.create-e2e-list.outputs.matrix) }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up env vars
+        run: |
+          echo "GO111MODULE=on" >> $GITHUB_ENV
+          echo "GO_VERSION=v$(sed -n 's/GO_VERSION=//p' config.base.env | tr -d '\n' | tr -d '"')" >> $GITHUB_ENV
+          echo "KIND_CLUSTER_NAME=$(sed -n 's/KIND_CLUSTER_NAME=//p' config.base.env | tr -d '\n' | tr -d '"')" >> $GITHUB_ENV
+          echo "GOPATH=/home/runner/go" >> $GITHUB_ENV
+
+      - name: Prepare go environment
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Ensure Golang runtime dependencies
+        run: make go-dependencies
+
       - name: Kind setup
         uses: helm/kind-action@v1.9.0
         with:
@@ -55,16 +91,19 @@ jobs:
 
       - name: Prepare environment for e2e
         run: |
-          sudo apt-get update
+          sudo apt-get update && sudo apt-get install -y socat
-          sudo apt-get install socat
           sudo mkdir -p $HOME/.kube
           sudo chown -R $USER $HOME/.kube
 
-      - name: Jenkins Operator - e2e - list tests
+      - name: Jenkins Operator - e2e Chart tests
-        run: make e2e E2E_TEST_ARGS='-ginkgo.v -ginkgo.dryRun'
+        env:
-
+          TNAME: ${{ matrix.test }}
-      - name: Jenkins Operator - e2e
+          TFILE: ${{ matrix.file }}
-        run: make e2e E2E_TEST_ARGS='-ginkgo.v'
+          TLINE: ${{ matrix.line }}
+        run: |
+          git reset --hard
+          printf "\n \n > Running test: %s from file: $s line: %s\n" "${TNAME}" "${TFILE}" "${TLINE}"
+          make e2e E2E_TEST_ARGS='-ginkgo.v -ginkgo.focus="${TNAME}"'
 
       - name: Debug
         if: failure()

.github/workflows/auto-tests-helm.yaml

@@ -1,4 +1,4 @@
-name: Run Helm e2e tests
+name: Tests HELM
 on:
   push:
     branches:
@@ -20,10 +20,50 @@ on:
       - '*.md'
 
 jobs:
-  run-tests:
+  create-helm-list:
-    if: github.event.pull_request.draft == false
+    name: HELM Create tests list
-    name: Run automated tests
     runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+    - uses: actions/checkout@v4
+    - id: matrix
+      run: |
+        script=$(./test/make_matrix_ginkgo.sh helm)
+        echo "matrix=${script}" >> $GITHUB_OUTPUT
+
+  verify-code:
+    name: HELM Verify code before tests
+    runs-on: ubuntu-latest
+    needs: [create-helm-list]
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up env vars
+      run: |
+        echo "GO111MODULE=on" >> $GITHUB_ENV
+        echo "GO_VERSION=v$(sed -n 's/GO_VERSION=//p' config.base.env | tr -d '\n' | tr -d '"')" >> $GITHUB_ENV
+        echo "GOPATH=/home/runner/go" >> $GITHUB_ENV
+
+    - name: Prepare go environment
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ env.GO_VERSION }}
+
+    - name: Ensure Golang runtime dependencies
+      run: make go-dependencies
+
+    - name: Verify code formatting
+      run: make verify
+
+  run-helm-tests:
+    runs-on: ubuntu-latest
+    needs: [create-helm-list, verify-code]
+    if: github.event.pull_request.draft == false
+    name: HELM ${{ matrix.test }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.create-helm-list.outputs.matrix) }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -44,9 +84,6 @@ jobs:
       - name: Ensure Golang runtime dependencies
         run: make go-dependencies
 
-      - name: Verify code formatting
-        run: make verify
-
       - name: Kind setup
         uses: helm/kind-action@v1.9.0
         with:
@@ -55,13 +92,26 @@ jobs:
 
       - name: Prepare environment for e2e
         run: |
-          sudo apt-get update
+          sudo apt-get update && sudo apt-get install -y socat
-          sudo apt-get install socat
           sudo mkdir -p $HOME/.kube
           sudo chown -R $USER $HOME/.kube
 
       - name: Jenkins Operator - Helm Chart tests
+        env:
+          TNAME: ${{ matrix.test }}
+          TFILE: ${{ matrix.file }}
+          TLINE: ${{ matrix.line }}
         run: |
           git reset --hard
           make helm-lint
-          make helm-e2e E2E_TEST_ARGS='-ginkgo.v'
+          printf "\n \n > Running test: %s from file: $s line: %s\n" "${TNAME}" "${TFILE}" "${TLINE}"
+          make helm-e2e E2E_TEST_ARGS='-ginkgo.v -ginkgo.focus="${TNAME}"'
+
+      - name: Debug
+        if: failure()
+        shell: bash
+        continue-on-error: true
+        run: |
+          randomns=$(kubectl get ns| grep -i 'ns[0-9]\+' |cut -d ' ' -f 1)
+          kubectl get pods -n ${randomns}
+          kubectl get events -n ${randomns}

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/sirosen/check-jsonschema
-    rev: 0.22.0
+    rev: 0.28.0
     hooks:
       - id: check-github-workflows
   - repo: https://github.com/pre-commit/pre-commit-hooks

flake.nix

@@ -33,6 +33,11 @@
               pkgs.gnumake
               pkgs.wget
               pkgs.helm-docs
+              (pkgs.writeShellApplication {
+                name = "make_matrix";
+                runtimeInputs = with pkgs; [ bash gnugrep gawk ];
+                text = builtins.readFile ./test/make_matrix_ginkgo.sh;
+              })
               go_15_pkgs.go
               golangci_pkgs.golangci-lint
           ];

test/e2e/jenkins_configuration_test.go

@@ -90,7 +90,7 @@ var _ = Describe("Jenkins controller configuration", func() {
 	})
 
 	Context("when deploying CR to cluster", func() {
-		It("creates Jenkins instance and configures it", func() {
+		It("creates vanilla Jenkins instance and configures it", func() {
 			WaitForJenkinsBaseConfigurationToComplete(jenkins)
 			verifyJenkinsMasterPodAttributes(jenkins)
 			verifyServices(jenkins)
@@ -138,7 +138,7 @@ var _ = Describe("Jenkins controller priority class", func() {
 	})
 
 	Context("when deploying CR with priority class to cluster", func() {
-		It("creates Jenkins instance and configures it", func() {
+		It("creates Jenkins instance with priority class and configures it", func() {
 			WaitForJenkinsBaseConfigurationToComplete(jenkins)
 			verifyJenkinsMasterPodAttributes(jenkins)
 		})

test/e2e/jenkins_pod_restart_test.go

@@ -47,7 +47,7 @@ var _ = Describe("Jenkins controller", func() {
 	})
 
 	Context("when restarting Jenkins master pod", func() {
-		It("new Jenkins Master pod should be created", func() {
+		It("new Jenkins pod should be created after a restart", func() {
 			WaitForJenkinsBaseConfigurationToComplete(jenkins)
 			restartJenkinsMasterPod(jenkins)
 			waitForRecreateJenkinsMasterPod(jenkins)
@@ -96,19 +96,16 @@ var _ = Describe("Jenkins controller", func() {
 	})
 
 	Context("when running Jenkins safe restart", func() {
-		It("authorization strategy is not overwritten", func() {
+		It("authorization strategy is not overwritten after a restart", func() {
 			// TODO: @brokenpip3 temporary disable this flaky test
 			Skip("Temporary skipping this test")
-			WaitForJenkinsBaseConfigurationToComplete(jenkins)
 			WaitForJenkinsUserConfigurationToComplete(jenkins)
 			jenkinsClient, cleanUpFunc := verifyJenkinsAPIConnection(jenkins, namespace.Name)
 			defer cleanUpFunc()
 			checkIfAuthorizationStrategyUnsecuredIsSet(jenkinsClient)
-
 			err := jenkinsClient.SafeRestart()
 			Expect(err).NotTo(HaveOccurred())
 			waitForJenkinsSafeRestart(jenkinsClient)
-
 			checkIfAuthorizationStrategyUnsecuredIsSet(jenkinsClient)
 		})
 	})

test/e2e/jenkins_test.go

@@ -240,13 +240,14 @@ func verifyJenkinsAPIConnection(jenkins *v1alpha2.Jenkins, namespace string) (je
 func restartJenkinsMasterPod(jenkins *v1alpha2.Jenkins) {
 	_, _ = fmt.Fprintf(GinkgoWriter, "Restarting Jenkins master pod\n")
 	jenkinsPod := getJenkinsMasterPod(jenkins)
-	_, _ = fmt.Fprintf(GinkgoWriter, "Jenkins pod: %+v\n", jenkinsPod)
+	initialCreationTimestamp := jenkinsPod.CreationTimestamp.DeepCopy()
+
+	_, _ = fmt.Fprintf(GinkgoWriter, "Jenkins pod: %+v\n", jenkinsPod.Status.Phase)
 	Expect(K8sClient.Delete(context.TODO(), jenkinsPod)).Should(Succeed())
 
 	Eventually(func() (bool, error) {
 		jenkinsPod = getJenkinsMasterPod(jenkins)
-		fmt.Printf("Jenkins pod deletion timestamp: %v\n", jenkinsPod.DeletionTimestamp)
+		return !jenkinsPod.CreationTimestamp.Equal(initialCreationTimestamp), nil
-		return jenkinsPod.DeletionTimestamp != nil, nil
 	}, 45*retryInterval, retryInterval).Should(BeTrue())
 
 	_, _ = fmt.Fprintf(GinkgoWriter, "Jenkins master pod has been restarted\n")

test/helm/helm_test.go

@@ -36,7 +36,7 @@ var _ = Describe("Jenkins Controller", func() {
 	})
 
 	Context("Deploys jenkins operator with helm charts with default values", func() {
-		It("Deploys Jenkins operator and configures default Jenkins instance", func() {
+		It("Deploys Jenkins operator and configures the default Jenkins instance", func() {
 			jenkins := &v1alpha2.Jenkins{
 				TypeMeta: v1alpha2.JenkinsTypeMeta(),
 				ObjectMeta: metav1.ObjectMeta{
@@ -99,7 +99,7 @@ var _ = Describe("Jenkins Controller with security validator", func() {
 	})
 
 	Context("When Jenkins CR contains plugins with security warnings", func() {
-		It("Denies creating a jenkins CR with a warning", func() {
+		It("Denies creating a jenkins CR with a plugin contains security warning", func() {
 			By("Deploying the operator along with webhook and cert-manager")
 			cmd := exec.Command("../../bin/helm", "upgrade", "jenkins", "../../chart/jenkins-operator", "--namespace", namespace.Name, "--debug",
 				"--set-string", fmt.Sprintf("jenkins.namespace=%s", namespace.Name),
@@ -121,7 +121,7 @@ var _ = Describe("Jenkins Controller with security validator", func() {
 		})
 	})
 	Context("When Jenkins CR doesn't contain plugins with security warnings", func() {
-		It("Jenkins instance is successfully created", func() {
+		It("Permit creating a jenkins CR without security warning in plugins", func() {
 			By("Deploying the operator along with webhook and cert-manager")
 			cmd := exec.Command("../../bin/helm", "upgrade", "jenkins", "../../chart/jenkins-operator", "--namespace", namespace.Name, "--debug",
 				"--set-string", fmt.Sprintf("jenkins.namespace=%s", namespace.Name),

test/make_matrix_ginkgo.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+TESTDIR="${TESTDIR:-test}"
+
+json_output(){
+    # Make shellcheck happy,
+    # declare local before assign
+    local lastl
+    local line
+    local grep_info
+    local f
+    local l
+    local t
+
+    lastl=$(echo "${1}" | wc -l)
+    line=0
+    printf '{\"include\":['
+    while read -r test; do
+        line=$((line + 1))
+        grep_info=$(echo "${test}"|awk -F '"' '{print $1}')
+        f=$(echo "${grep_info}"|cut -d ':' -f 1)
+        l=$(echo "${grep_info}"|cut -d ':' -f 2)
+        t=$(echo "${test}"|awk -F '"' '{print $2}')
+        printf '{\"file\":\"%s\",\"line\":\"%s\",\"test\":\"%s\"}' "$f" "$l" "$t"
+        [[ $line -ne $lastl ]] && printf ","
+    done <<< "${1}"
+    printf "]}"
+}
+
+parse(){
+    grep -nrE 'It\([^)]+\)' "$1"
+}
+
+tests_list=$(parse "${TESTDIR}"/"${1}")
+json_output "${tests_list}"