e2e: Concurrent workflow jobs (#667)

Enhances out existing E2E test suite to additionally support triggering two or more concurrent workflow jobs and verifying all the results, so that you can ensure the runners managed by the controller are able to handle jobs reliably when loaded.

test/e2e/e2e_test.go

@@ -194,12 +194,28 @@ func TestE2E(t *testing.T) {
 		}
 	})
 
-	testResultCMName := fmt.Sprintf("test-result-%s", id)
+	testResultCMNamePrefix := "test-result-"
 
 	if t.Failed() {
 		return
 	}
 
+	numJobs := 2
+
+	type job struct {
+		name, testArg, configMapName string
+	}
+
+	var testJobs []job
+
+	for i := 0; i < numJobs; i++ {
+		name := fmt.Sprintf("test%d", i)
+		testArg := fmt.Sprintf("%s%d", id, i)
+		configMapName := testResultCMNamePrefix + testArg
+
+		testJobs = append(testJobs, job{name: name, testArg: testArg, configMapName: configMapName})
+	}
+
 	t.Run("Install workflow", func(t *testing.T) {
 		wfName := "E2E " + testID
 		wf := testing.Workflow{
@@ -209,8 +225,12 @@ func TestE2E(t *testing.T) {
 					Branches: []string{"main"},
 				},
 			},
-			Jobs: map[string]testing.Job{
+			Jobs: map[string]testing.Job{},
-				"test": {
+		}
+
+		for i := 0; i < numJobs; i++ {
+			j := testJobs[i]
+			wf.Jobs[j.name] = testing.Job{
 				RunsOn: runnerLabel,
 				Steps: []testing.Step{
 					{
@@ -223,12 +243,11 @@ func TestE2E(t *testing.T) {
 						},
 					},
 					{
-							Run: "./test.sh",
+						Run: fmt.Sprintf("./test.sh %s %s", t.Name(), j.testArg),
-						},
-					},
 					},
 				},
 			}
+		}
 
 		wfContent, err := yaml.Marshal(wf)
 		if err != nil {
@@ -237,10 +256,12 @@ func TestE2E(t *testing.T) {
 
 		script := []byte(fmt.Sprintf(`#!/usr/bin/env bash
 set -vx
-echo hello from %s
+name=$1
-kubectl delete cm %s || true
+id=$2
-kubectl create cm %s --from-literal=status=ok
+echo hello from $name
-`, testID, testResultCMName, testResultCMName))
+kubectl delete cm %s$id || true
+kubectl create cm %s$id --from-literal=status=ok
+`, testResultCMNamePrefix, testResultCMNamePrefix))
 
 		g := testing.GitRepo{
 			Dir:           filepath.Join(t.TempDir(), "gitrepo"),
@@ -262,16 +283,43 @@ kubectl create cm %s --from-literal=status=ok
 	}
 
 	t.Run("Verify workflow run result", func(t *testing.T) {
-		gomega.NewGomegaWithT(t).Eventually(func() (string, error) {
+		var expected []string
-			m, err := k.GetCMLiterals(ctx, testResultCMName, cmCfg)
+
-			if err != nil {
+		for i := 0; i < numJobs; i++ {
-				return "", err
+			expected = append(expected, "ok")
 		}
 
-			result := m["status"]
+		gomega.NewGomegaWithT(t).Eventually(func() ([]string, error) {
+			var results []string
 
-			return result, nil
+			var errs []error
-		}, 60*time.Second, 10*time.Second).Should(gomega.Equal("ok"))
+
+			for i := 0; i < numJobs; i++ {
+				testResultCMName := testJobs[i].configMapName
+
+				m, err := k.GetCMLiterals(ctx, testResultCMName, cmCfg)
+				if err != nil {
+					errs = append(errs, err)
+				} else {
+					result := m["status"]
+					results = append(results, result)
+				}
+			}
+
+			var err error
+
+			if len(errs) > 0 {
+				var msg string
+
+				for i, e := range errs {
+					msg += fmt.Sprintf("error%d: %v\n", i, e)
+				}
+
+				err = fmt.Errorf("%d errors occurred: %s", len(errs), msg)
+			}
+
+			return results, err
+		}, 60*time.Second, 10*time.Second).Should(gomega.Equal(expected))
 	})
 }