Refactor tests and make them run in parallel. (#499)

integration/integration_test.go

@@ -30,6 +30,8 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/sync/errgroup"
+
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/daemon"
 
@@ -140,67 +142,76 @@ func TestMain(m *testing.M) {
 	RunOnInterrupt(func() { DeleteFromBucket(fileInBucket) })
 	defer DeleteFromBucket(fileInBucket)
 
-	fmt.Println("Building kaniko image")
+	setupCommands := []struct {
-	cmd := exec.Command("docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", "..")
+		name    string
-	if _, err = RunCommandWithoutTest(cmd); err != nil {
+		command []string
-		fmt.Printf("Building kaniko failed: %s", err)
+	}{
-		os.Exit(1)
+		{
+			name:    "Building kaniko image",
+			command: []string{"docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", ".."},
+		},
+		{
+			name:    "Building cache warmer image",
+			command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."},
+		},
+		{
+			name:    "Building onbuild base image",
+			command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", "dockerfiles/Dockerfile_onbuild_base", "."},
+		},
+		{
+			name:    "Pushing onbuild base image",
+			command: []string{"docker", "push", config.onbuildBaseImage},
+		},
+		{
+			name:    "Building hardlink base image",
+			command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", "dockerfiles/Dockerfile_hardlink_base", "."},
+		},
+		{
+			name:    "Pushing hardlink base image",
+			command: []string{"docker", "push", config.hardlinkBaseImage},
+		},
 	}
 
-	fmt.Println("Building cache warmer image")
+	for _, setupCmd := range setupCommands {
-	cmd = exec.Command("docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", "..")
+		fmt.Println(setupCmd.name)
-	if _, err = RunCommandWithoutTest(cmd); err != nil {
+		cmd := exec.Command(setupCmd.command[0], setupCmd.command[1:]...)
-		fmt.Printf("Building kaniko's cache warmer failed: %s", err)
+		if _, err := RunCommandWithoutTest(cmd); err != nil {
-		os.Exit(1)
+			fmt.Printf("%s failed: %s", setupCmd.name, err)
+			os.Exit(1)
+		}
 	}
 
-	fmt.Println("Building onbuild base image")
-	buildOnbuildBase := exec.Command("docker", "build", "-t", config.onbuildBaseImage, "-f", "dockerfiles/Dockerfile_onbuild_base", ".")
-	if err := buildOnbuildBase.Run(); err != nil {
-		fmt.Printf("error building onbuild base: %v", err)
-		os.Exit(1)
-	}
-
-	pushOnbuildBase := exec.Command("docker", "push", config.onbuildBaseImage)
-	if err := pushOnbuildBase.Run(); err != nil {
-		fmt.Printf("error pushing onbuild base %s: %v", config.onbuildBaseImage, err)
-		os.Exit(1)
-	}
-
-	fmt.Println("Building hardlink base image")
-	buildHardlinkBase := exec.Command("docker", "build", "-t", config.hardlinkBaseImage, "-f", "dockerfiles/Dockerfile_hardlink_base", ".")
-	if err := buildHardlinkBase.Run(); err != nil {
-		fmt.Printf("error building hardlink base: %v", err)
-		os.Exit(1)
-	}
-	pushHardlinkBase := exec.Command("docker", "push", config.hardlinkBaseImage)
-	if err := pushHardlinkBase.Run(); err != nil {
-		fmt.Printf("error pushing hardlink base %s: %v", config.hardlinkBaseImage, err)
-		os.Exit(1)
-	}
 	dockerfiles, err := FindDockerFiles(dockerfilesPath)
 	if err != nil {
 		fmt.Printf("Coudn't create map of dockerfiles: %s", err)
 		os.Exit(1)
 	}
 	imageBuilder = NewDockerFileBuilder(dockerfiles)
+
+	g := errgroup.Group{}
+	for dockerfile := range imageBuilder.FilesBuilt {
+		df := dockerfile
+		g.Go(func() error {
+			return imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, df)
+		})
+	}
+	if err := g.Wait(); err != nil {
+		fmt.Printf("Error building images: %s", err)
+		os.Exit(1)
+	}
 	os.Exit(m.Run())
 }
 func TestRun(t *testing.T) {
-	for dockerfile, built := range imageBuilder.FilesBuilt {
+	for dockerfile := range imageBuilder.FilesBuilt {
 		t.Run("test_"+dockerfile, func(t *testing.T) {
+			dockerfile := dockerfile
+			t.Parallel()
 			if _, ok := imageBuilder.DockerfilesToIgnore[dockerfile]; ok {
 				t.SkipNow()
 			}
 			if _, ok := imageBuilder.TestCacheDockerfiles[dockerfile]; ok {
 				t.SkipNow()
 			}
-			if !built {
-				err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile)
-				if err != nil {
-					t.Fatalf("Failed to build kaniko and docker images for %s: %s", dockerfile, err)
-				}
-			}
 			dockerImage := GetDockerImage(config.imageRepo, dockerfile)
 			kanikoImage := GetKanikoImage(config.imageRepo, dockerfile)
 
@@ -229,17 +240,13 @@ func TestLayers(t *testing.T) {
 		"Dockerfile_test_add":     11,
 		"Dockerfile_test_scratch": 3,
 	}
-	for dockerfile, built := range imageBuilder.FilesBuilt {
+	for dockerfile := range imageBuilder.FilesBuilt {
 		t.Run("test_layer_"+dockerfile, func(t *testing.T) {
+			dockerfile := dockerfile
+			t.Parallel()
 			if _, ok := imageBuilder.DockerfilesToIgnore[dockerfile]; ok {
 				t.SkipNow()
 			}
-			if !built {
-				err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile)
-				if err != nil {
-					t.Fatalf("Failed to build kaniko and docker images for %s: %s", dockerfile, err)
-				}
-			}
 			// Pull the kaniko image
 			dockerImage := GetDockerImage(config.imageRepo, dockerfile)
 			kanikoImage := GetKanikoImage(config.imageRepo, dockerfile)
@@ -260,6 +267,8 @@ func TestCache(t *testing.T) {
 	populateVolumeCache()
 	for dockerfile := range imageBuilder.TestCacheDockerfiles {
 		t.Run("test_cache_"+dockerfile, func(t *testing.T) {
+			dockerfile := dockerfile
+			t.Parallel()
 			cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano()))
 			// Build the initial image which will cache layers
 			if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, 0); err != nil {
@@ -286,8 +295,7 @@ func TestCache(t *testing.T) {
 		})
 	}
 
-	err := logBenchmarks("benchmark_cache")
+	if err := logBenchmarks("benchmark_cache"); err != nil {
-	if err != nil {
 		t.Logf("Failed to create benchmark file: %v", err)
 	}
 }