Add K8s integration tests

- Use minikube for deploying a lightweight K8s on Travis CI
- Build and push Kaniko image to local docker registry
- Build dockerfiles with context on both docker and K8s and check
  the diff between the 2.

.travis.yml

@@ -2,7 +2,8 @@ language: go
 os: linux
 dist: bionic
 env:
-  - IMAGE_REPO=localhost:5000
+  global:
+    - IMAGE_REPO=localhost:5000 REGISTRY=localhost:5000
 go:
   - "1.13.3"
 go_import_path: github.com/GoogleContainerTools/kaniko
@@ -21,11 +22,16 @@ jobs:
         - make travis-setup
       script:
         - make integration-test-layers
+    - name: build-image-and-k8s-integration-test
+      before_install:
+        - make travis-setup
+        - make minikube-setup
+      script:
+        - make images
+        - make push
+        - make integration-test-k8s
     - name: integration-test-misc
       before_install:
         - make travis-setup
       script:
         - make integration-test-misc
-    - stage: build-images
-      script:
-        - make images

Makefile

@@ -57,6 +57,10 @@ out/warmer: $(GO_FILES)
 travis-setup:
 	@ ./scripts/travis-setup.sh
 
+.PHONY: minikube-setup
+minikube-setup:
+	@ ./scripts/minikube-setup.sh
+
 .PHONY: test
 test: out/executor
 	@ ./scripts/test.sh
@@ -73,6 +77,10 @@ integration-test-run:
 integration-test-layers:
 	@ ./scripts/integration-test.sh -run "TestLayers"
 
+.PHONY: integration-test-k8s
+integration-test-k8s:
+	@ ./scripts/integration-test.sh -run "TestK8s"
+
 .PHONY: integration-test-misc
 integration-test-misc:
 	$(eval RUN_ARG=$(shell ./scripts/misc-integration-test.sh))
@@ -86,6 +94,6 @@ images:
 
 .PHONY: push
 push:
-    docker push $(REGISTRY)/executor:latest
-    docker push $(REGISTRY)/executor:debug
-    docker push $(REGISTRY)/warmer:latest
+	docker push $(REGISTRY)/executor:latest
+	docker push $(REGISTRY)/executor:debug
+	docker push $(REGISTRY)/warmer:latest

integration/images.go

@@ -185,23 +185,8 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string {
 	return flags
 }
 
-// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker.
-// The resulting image will be tagged with imageRepo. If the dockerfile will be built with
-// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket.
-func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error {
-	_, ex, _, _ := runtime.Caller(0)
-	cwd := filepath.Dir(ex)
-
-	return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd)
-}
-
-func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error {
-	if _, present := d.filesBuilt[dockerfile]; present {
-		return nil
-	}
-	gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo
-
-	fmt.Printf("Building images for Dockerfile %s\n", dockerfile)
+func (d *DockerFileBuilder) BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir string) error {
+	fmt.Printf("Building image for Dockerfile %s\n", dockerfile)
 
 	var buildArgs []string
 	buildArgFlag := "--build-arg"
@@ -230,13 +215,39 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
 		dockerCmd.Env = append(dockerCmd.Env, env...)
 	}
 
-	timer := timing.Start(dockerfile + "_docker")
 	out, err := RunCommandWithoutTest(dockerCmd)
-	timing.DefaultRun.Stop(timer)
 	if err != nil {
 		return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out))
 	}
 	fmt.Printf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out)
+	return nil
+}
+
+// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker.
+// The resulting image will be tagged with imageRepo. If the dockerfile will be built with
+// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket.
+func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error {
+	_, ex, _, _ := runtime.Caller(0)
+	cwd := filepath.Dir(ex)
+
+	return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd)
+}
+
+func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error {
+	if _, present := d.filesBuilt[dockerfile]; present {
+		return nil
+	}
+	gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo
+
+	var buildArgs []string
+	buildArgFlag := "--build-arg"
+	for _, arg := range argsMap[dockerfile] {
+		buildArgs = append(buildArgs, buildArgFlag, arg)
+	}
+
+	timer := timing.Start(dockerfile + "_docker")
+	d.BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir)
+	timing.DefaultRun.Stop(timer)
 
 	contextFlag := "-c"
 	contextPath := buildContextPath
@@ -269,7 +280,7 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
 	}
 
 	// build kaniko image
-	additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...)
+	additionalFlags := append(buildArgs, additionalKanikoFlagsMap[dockerfile]...)
 	kanikoImage := GetKanikoImage(imageRepo, dockerfile)
 	fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags)
 
@@ -301,16 +312,16 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
 	kanikoCmd := exec.Command("docker", dockerRunFlags...)
 
 	timer = timing.Start(dockerfile + "_kaniko")
-	out, err = RunCommandWithoutTest(kanikoCmd)
+	out, err := RunCommandWithoutTest(kanikoCmd)
 	timing.DefaultRun.Stop(timer)
 
 	if err != nil {
-		return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out))
+		return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out))
 	}
 
 	if outputCheck := outputChecks[dockerfile]; outputCheck != nil {
 		if err := outputCheck(dockerfile, out); err != nil {
-			return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out))
+			return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out))
 		}
 	}
 

integration/k8s-job.yaml

@@ -0,0 +1,22 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kaniko-test-{{.Name}}
+spec:
+  template:
+    spec:
+      hostNetwork: true
+      containers:
+      - name: kaniko
+        image: localhost:5000/executor:debug
+        args: [ "--context=dir:///workspace",
+                "--destination={{.KanikoImage}}"]
+        volumeMounts:
+          - name: context
+            mountPath: /workspace
+      restartPolicy: Never
+      volumes:
+      - name: context
+        hostPath:
+          path: {{.Context}}
+  backoffLimit: 1

integration/k8s_test.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"text/template"
+)
+
+type K8sConfig struct {
+	KanikoImage string
+	Context     string
+	Name        string
+}
+
+func TestK8s(t *testing.T) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dir := filepath.Join(cwd, "dockerfiles-with-context")
+
+	testDirs, err := ioutil.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	builder := NewDockerFileBuilder()
+
+	for _, tdInfo := range testDirs {
+		name := tdInfo.Name()
+		testDir := filepath.Join(dir, name)
+
+		t.Run("test_k8s_with_context_"+name, func(t *testing.T) {
+			t.Parallel()
+
+			if err := builder.BuildDockerImage(
+				config.imageRepo, "", name, testDir,
+			); err != nil {
+				t.Fatal(err)
+			}
+
+			dockerImage := GetDockerImage(config.imageRepo, name)
+			kanikoImage := GetKanikoImage(config.imageRepo, name)
+
+			tmpfile, err := ioutil.TempFile("", "k8s-job-*.yaml")
+			if err != nil {
+				log.Fatal(err)
+			}
+			defer os.Remove(tmpfile.Name()) // clean up
+			tmpl := template.Must(template.ParseFiles("k8s-job.yaml"))
+			job := K8sConfig{KanikoImage: kanikoImage, Context: testDir, Name: name}
+			if err := tmpl.Execute(tmpfile, job); err != nil {
+				t.Fatal(err)
+			}
+
+			fmt.Printf("Testing K8s based Kaniko building of dockerfile %s and push to %s \n",
+				testDir, kanikoImage)
+			content, err := ioutil.ReadFile(tmpfile.Name())
+			if err != nil {
+				log.Fatal(err)
+			}
+			fmt.Printf("K8s template %s:\n%s\n", tmpfile.Name(), content)
+
+			kubeCmd := exec.Command("kubectl", "apply", "-f", tmpfile.Name())
+			RunCommand(kubeCmd, t)
+
+			fmt.Printf("Waiting for K8s kaniko build job to finish: %s\n",
+				"job/kaniko-test-"+job.Name)
+
+			kubeWaitCmd := exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=60s",
+				"job/kaniko-test-"+job.Name)
+			RunCommand(kubeWaitCmd, t)
+
+			diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache")
+
+			expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage)
+			checkContainerDiffOutput(t, diff, expected)
+		})
+	}
+
+	if err := logBenchmarks("benchmark"); err != nil {
+		t.Logf("Failed to create benchmark file: %v", err)
+	}
+}

scripts/minikube-setup.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
+chmod +x kubectl
+sudo mv kubectl /usr/local/bin/
+kubectl version --client
+
+curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
+chmod +x minikube
+sudo mv minikube /usr/local/bin/
+
+sudo apt-get update
+sudo apt-get install -y liblz4-tool
+
+sudo minikube start --vm-driver=none
+sudo minikube status
+sudo chown -R $USER $HOME/.kube $HOME/.minikube
+kubectl cluster-info

scripts/misc-integration-test.sh

@@ -19,7 +19,7 @@ set -e
 
 TESTS=$(./scripts/integration-test.sh -list=Test -mod=vendor)
 
-TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers')
+TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers' | grep -v 'TestK8s')
 
 RUN_ARG=''
 count=0