Fix the e2e K8s test (#1842)

* Debugging the integration testing.

It seems to be consistently failing (also fix a typo in the file/leg name).

I'm disabling `-v` for `go test` because it interleaves the test output in ways that make it painful to read a single failing test's output when `t.Parallel` is involved.

* Try swapping registry:2 for the registry add-on

* Drop logf and make Logger private

.github/workflows/integration-layers-tests.yaml

@@ -28,4 +28,5 @@ jobs:
     - name: Run integration-test-layers
       run: |
         make travis-setup
+        make minikube-setup
         make integration-test-layers

.github/workflows/integration-misc.yaml

@@ -37,4 +37,5 @@ jobs:
       - name: Run integration-test-misc
         run : |
           make travis-setup
+          make minikube-setup
           make integration-test-misc

.github/workflows/integration-run-tests.yaml

@@ -27,4 +27,5 @@ jobs:
       - name: Run integration-test-run
         run: |
           make travis-setup
+          make minikube-setup
           make integration-test-run

.github/workflows/integration_k8_tests.yaml

@@ -1,6 +1,6 @@
-name: Intergration tests (K8s)
+name: Integration tests (K8s)
 
-# Triggers the workflow on push or pull request events
+# Triggers the workflow on pull request events
 on: [pull_request]
 
 concurrency:

integration/benchmark_test.go

@@ -59,7 +59,7 @@ func TestSnapshotBenchmark(t *testing.T) {
 				kanikoImage := fmt.Sprintf("%s_%d", GetKanikoImage(config.imageRepo, dockerfile), num)
 				buildArgs := []string{"--build-arg", fmt.Sprintf("NUM=%d", num)}
 				var benchmarkDir string
-				benchmarkDir, *err = buildKanikoImage("", dockerfile,
+				benchmarkDir, *err = buildKanikoImage(t.Logf, "", dockerfile,
 					buildArgs, []string{}, kanikoImage, contextDir, config.gcsBucket,
 					config.serviceAccount, false)
 				if *err != nil {
@@ -77,11 +77,11 @@ func TestSnapshotBenchmark(t *testing.T) {
 	}
 	wg.Wait()
 
-	fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files")
+	t.Log("Number of Files,Total Build Time,Walking Filesystem, Resolving Files")
 	timeMap.Range(func(key interface{}, value interface{}) bool {
 		d, _ := key.(int)
 		v, _ := value.(result)
-		fmt.Println(fmt.Sprintf("%d,%f,%f,%f", d, v.totalBuildTime, v.walkingFiles, v.resolvingFiles))
+		t.Logf("%d,%f,%f,%f", d, v.totalBuildTime, v.walkingFiles, v.resolvingFiles)
 		return true
 	})
 
@@ -111,7 +111,7 @@ func newResult(t *testing.T, f string) result {
 	if c, ok := current["Hashing files"]; ok {
 		r.hashingFiles = c.Seconds()
 	}
-	fmt.Println(r)
+	t.Log(r)
 	return r
 }
 
@@ -128,7 +128,7 @@ func TestSnapshotBenchmarkGcloud(t *testing.T) {
 	nums := []int{10000, 50000, 100000, 200000, 300000, 500000, 700000}
 
 	var wg sync.WaitGroup
-	fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files")
+	t.Log("Number of Files,Total Build Time,Walking Filesystem, Resolving Files")
 	for _, num := range nums {
 		t.Run(fmt.Sprintf("test_benchmark_%d", num), func(t *testing.T) {
 			wg.Add(1)
@@ -139,7 +139,7 @@ func TestSnapshotBenchmarkGcloud(t *testing.T) {
 					return
 				}
 				r := newResult(t, filepath.Join(dir, "results"))
-				fmt.Println(fmt.Sprintf("%d,%f,%f,%f, %f", num, r.totalBuildTime, r.walkingFiles, r.resolvingFiles, r.hashingFiles))
+				t.Log(fmt.Sprintf("%d,%f,%f,%f, %f", num, r.totalBuildTime, r.walkingFiles, r.resolvingFiles, r.hashingFiles))
 				wg.Done()
 				defer os.Remove(dir)
 				defer os.Chdir(cwd)

integration/images.go

@@ -27,6 +27,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"testing"
 	"time"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/timing"
@@ -183,10 +184,14 @@ type DockerFileBuilder struct {
 	TestCacheDockerfiles map[string]struct{}
 }
 
+type logger func(string, ...interface{})
+
 // NewDockerFileBuilder will create a DockerFileBuilder initialized with dockerfiles, which
 // it will assume are all as yet unbuilt.
 func NewDockerFileBuilder() *DockerFileBuilder {
-	d := DockerFileBuilder{filesBuilt: map[string]struct{}{}}
+	d := DockerFileBuilder{
+		filesBuilt: map[string]struct{}{},
+	}
 	d.DockerfilesToIgnore = map[string]struct{}{
 		"Dockerfile_test_add_404": {},
 		// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
@@ -218,8 +223,8 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string {
 	return flags
 }
 
-func (d *DockerFileBuilder) BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir string) error {
+func (d *DockerFileBuilder) BuildDockerImage(t *testing.T, imageRepo, dockerfilesPath, dockerfile, contextDir string) error {
-	fmt.Printf("Building image for Dockerfile %s\n", dockerfile)
+	t.Logf("Building image for Dockerfile %s\n", dockerfile)
 
 	var buildArgs []string
 	buildArgFlag := "--build-arg"
@@ -252,21 +257,21 @@ func (d *DockerFileBuilder) BuildDockerImage(imageRepo, dockerfilesPath, dockerf
 	if err != nil {
 		return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out))
 	}
-	fmt.Printf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out)
+	t.Logf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out)
 	return nil
 }
 
 // BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker.
 // The resulting image will be tagged with imageRepo. If the dockerfile will be built with
 // context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket.
-func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error {
+func (d *DockerFileBuilder) BuildImage(t *testing.T, config *integrationTestConfig, dockerfilesPath, dockerfile string) error {
 	_, ex, _, _ := runtime.Caller(0)
 	cwd := filepath.Dir(ex)
 
-	return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd)
+	return d.BuildImageWithContext(t, config, dockerfilesPath, dockerfile, cwd)
 }
 
-func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error {
+func (d *DockerFileBuilder) BuildImageWithContext(t *testing.T, config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error {
 	if _, present := d.filesBuilt[dockerfile]; present {
 		return nil
 	}
@@ -279,7 +284,7 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
 	}
 
 	timer := timing.Start(dockerfile + "_docker")
-	d.BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir)
+	d.BuildDockerImage(t, imageRepo, dockerfilesPath, dockerfile, contextDir)
 	timing.DefaultRun.Stop(timer)
 
 	contextFlag := "-c"
@@ -302,7 +307,7 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
 
 	kanikoImage := GetKanikoImage(imageRepo, dockerfile)
 	timer = timing.Start(dockerfile + "_kaniko")
-	if _, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage,
+	if _, err := buildKanikoImage(t.Logf, dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage,
 		contextDir, gcsBucket, serviceAccount, true); err != nil {
 		return err
 	}
@@ -420,6 +425,7 @@ func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile, servi
 }
 
 func buildKanikoImage(
+	logf logger,
 	dockerfilesPath string,
 	dockerfile string,
 	buildArgs []string,
@@ -447,7 +453,7 @@ func buildKanikoImage(
 
 	// build kaniko image
 	additionalFlags := append(buildArgs, kanikoArgs...)
-	fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags)
+	logf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags)
 
 	dockerRunFlags := []string{"run", "--net=host",
 		"-e", benchmarkEnv,

integration/integration_test.go

@@ -87,8 +87,8 @@ func getDockerMajorVersion() int {
 	}
 	return ver
 }
-func launchTests(m *testing.M) (int, error) {
 
+func launchTests(m *testing.M) (int, error) {
 	if config.isGcrRepository() {
 		contextFile, err := CreateIntegrationTarball()
 		if err != nil {
@@ -141,32 +141,25 @@ func buildRequiredImages() error {
 	setupCommands := []struct {
 		name    string
 		command []string
-	}{
+	}{{
-		{
+		name:    "Building kaniko image",
-			name:    "Building kaniko image",
+		command: []string{"docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", ".."},
-			command: []string{"docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", ".."},
+	}, {
-		},
+		name:    "Building cache warmer image",
-		{
+		command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."},
-			name:    "Building cache warmer image",
+	}, {
-			command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."},
+		name:    "Building onbuild base image",
-		},
+		command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_onbuild_base", dockerfilesPath), "."},
-		{
+	}, {
-			name:    "Building onbuild base image",
+		name:    "Pushing onbuild base image",
-			command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_onbuild_base", dockerfilesPath), "."},
+		command: []string{"docker", "push", config.onbuildBaseImage},
-		},
+	}, {
-		{
+		name:    "Building hardlink base image",
-			name:    "Pushing onbuild base image",
+		command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_hardlink_base", dockerfilesPath), "."},
-			command: []string{"docker", "push", config.onbuildBaseImage},
+	}, {
-		},
+		name:    "Pushing hardlink base image",
-		{
+		command: []string{"docker", "push", config.hardlinkBaseImage},
-			name:    "Building hardlink base image",
+	}}
-			command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_hardlink_base", dockerfilesPath), "."},
-		},
-		{
-			name:    "Pushing hardlink base image",
-			command: []string{"docker", "push", config.hardlinkBaseImage},
-		},
-	}
 
 	for _, setupCmd := range setupCommands {
 		fmt.Println(setupCmd.name)
@@ -522,7 +515,7 @@ func TestLayers(t *testing.T) {
 }
 
 func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder) {
-	if err := imageBuilder.BuildImage(config, dockerfilesPath, dockerfile); err != nil {
+	if err := imageBuilder.BuildImage(t, config, dockerfilesPath, dockerfile); err != nil {
 		t.Errorf("Error building image: %s", err)
 		t.FailNow()
 	}

integration/integration_with_context_test.go

@@ -47,7 +47,7 @@ func TestWithContext(t *testing.T) {
 			t.Parallel()
 
 			if err := builder.BuildImageWithContext(
-				config, "", name, testDir,
+				t, config, "", name, testDir,
 			); err != nil {
 				t.Fatal(err)
 			}

integration/k8s-job.yaml

@@ -9,11 +9,12 @@ spec:
       containers:
       - name: kaniko
         image: localhost:5000/executor:latest
+        workingDir: /workspace
         args: [ "--context=dir:///workspace",
                 "--destination={{.KanikoImage}}"]
         volumeMounts:
-          - name: context
+        - name: context
-            mountPath: /workspace
+          mountPath: /workspace
       restartPolicy: Never
       volumes:
       - name: context

integration/k8s_test.go

@@ -56,7 +56,7 @@ func TestK8s(t *testing.T) {
 			t.Parallel()
 
 			if err := builder.BuildDockerImage(
-				config.imageRepo, "", name, testDir,
+				t, config.imageRepo, "", name, testDir,
 			); err != nil {
 				t.Fatal(err)
 			}
@@ -75,21 +75,21 @@ func TestK8s(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			fmt.Printf("Testing K8s based Kaniko building of dockerfile %s and push to %s \n",
+			t.Logf("Testing K8s based Kaniko building of dockerfile %s and push to %s \n",
 				testDir, kanikoImage)
 			content, err := ioutil.ReadFile(tmpfile.Name())
 			if err != nil {
 				log.Fatal(err)
 			}
-			fmt.Printf("K8s template %s:\n%s\n", tmpfile.Name(), content)
+			t.Logf("K8s template %s:\n%s\n", tmpfile.Name(), content)
 
 			kubeCmd := exec.Command("kubectl", "apply", "-f", tmpfile.Name())
 			RunCommand(kubeCmd, t)
 
-			fmt.Printf("Waiting for K8s kaniko build job to finish: %s\n",
+			t.Logf("Waiting for K8s kaniko build job to finish: %s\n",
 				"job/kaniko-test-"+job.Name)
 
-			kubeWaitCmd := exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=120s",
+			kubeWaitCmd := exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=1m",
 				"job/kaniko-test-"+job.Name)
 			if out, errR := RunCommandWithoutTest(kubeWaitCmd); errR != nil {
 				t.Log(kubeWaitCmd.Args)
@@ -100,9 +100,25 @@ func TestK8s(t *testing.T) {
 					t.Error(errD)
 				} else {
 					t.Log(string(outD))
-					t.Error(errR)
 				}
-				t.FailNow()
+
+				descCmd = exec.Command("kubectl", "describe", "pods", "--selector", "job-name=kaniko-test-"+job.Name)
+				outD, errD = RunCommandWithoutTest(descCmd)
+				if errD != nil {
+					t.Error(errD)
+				} else {
+					t.Log(string(outD))
+				}
+
+				logsCmd := exec.Command("kubectl", "logs", "--all-containers", "job/kaniko-test-"+job.Name)
+				outL, errL := RunCommandWithoutTest(logsCmd)
+				if errL != nil {
+					t.Error(errL)
+				} else {
+					t.Log(string(outL))
+				}
+
+				t.Fatal(errR)
 			}
 
 			diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache")

scripts/integration-test.sh

@@ -24,4 +24,4 @@ docker version
 echo "Running integration tests..."
 make out/executor
 make out/warmer
-go test ./integration/... -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@"
+go test ./integration/... --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@"

scripts/minikube-setup.sh

@@ -15,6 +15,24 @@
 
 set -ex
 
-curl -Lo minikube https://storage.googleapis.com/minikube-builds/master/minikube-linux-amd64
+# conntrack is required for minikube 1.19 and higher for none driver
-sudo install minikube /usr/local/bin/minikube
+if ! conntrack --version &>/dev/null; then
-minikube start --profile=minikube --driver=docker
+  echo "WARNING: No contrack is not installed"
+  sudo apt-get update -qq
+  sudo apt-get -qq -y install conntrack
+fi
+
+curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
+chmod +x minikube
+sudo mv minikube /usr/local/bin/
+
+sudo apt-get update
+sudo apt-get install -y liblz4-tool
+cat /proc/cpuinfo
+
+minikube start --vm-driver=none --force
+minikube status
+minikube addons enable registry
+kubectl cluster-info
+
+kubectl port-forward --namespace kube-system service/registry 5000:80 &

scripts/travis-setup.sh

@@ -20,7 +20,6 @@ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubun
 sudo apt-get update
 sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
 curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && sudo mv container-diff-linux-amd64 /usr/local/bin/container-diff
-docker run -d -p 5000:5000 --restart always --name registry registry:2
 
 mkdir -p $HOME/.docker/
 echo '{}' > $HOME/.docker/config.json