Integration test refactoring (#126)

* integration test refactoring

* config file cleanup

* more test refactoring

* remove debug file

* moving around more files

* fixing up integration tests

* integration tests work

* some housekeeping

* fixing tests

* addressing comments

* debugging

* debugging

* actual debugging

* skip integration tests for travis

* install container-diff before integration tests

* syntax

* make test failures less noisy

* fixing tests

* hopefully fixing CI?

* fixes

* more fixes

* let's actually fix CI

* more testing

* testing

* proper auth

* typos

* adding support for args in integration tests

* formatting

* formatting

* adding support for testing bucket context

* adding bucket test dockerfile

* addressing comments

* syntax

integration-test.sh

@@ -21,8 +21,14 @@ if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then
     mkdir -p /usr/local/go/src/github.com/GoogleContainerTools/
     cp -r github/kaniko /usr/local/go/src/github.com/GoogleContainerTools/
     pushd /usr/local/go/src/github.com/GoogleContainerTools/kaniko
+    echo "Installing container-diff..."
+    mv $KOKORO_GFILE_DIR/container-diff-linux-amd64 $KOKORO_GFILE_DIR/container-diff
+    chmod +x $KOKORO_GFILE_DIR/container-diff
+    export PATH=$PATH:$KOKORO_GFILE_DIR
+    cp $KOKORO_ROOT/src/keystore/72508_gcr_application_creds $HOME/.config/gcloud/application_default_credentials.json
 fi
 
 echo "Running integration tests..."
 make out/executor
-go run integration_tests/integration_test_yaml.go | gcloud container builds submit --config /dev/fd/0 .
+pushd integration
+go test

integration/dockerfiles/Dockerfile_test_copy_bucket

@@ -0,0 +1,20 @@
+FROM alpine:3.7
+COPY context/foo foo
+COPY context/foo /foodir/
+COPY context/bar/b* bar/
+COPY context/fo? /foo2
+COPY context/bar/doesnotexist* context/foo hello
+COPY ./context/empty /empty
+COPY ./ dir/
+COPY . newdir
+COPY context/bar /baz/
+COPY ["context/foo", "/tmp/foo" ]
+COPY context/b* /baz/
+COPY context/foo context/bar/ba? /test/
+COPY context/arr[[]0].txt /mydir/
+COPY context/bar/bat .
+
+ENV contextenv ./context
+COPY ${contextenv}/foo /tmp/foo2
+COPY $contextenv/foo /tmp/foo3
+COPY $contextenv/* /tmp/${contextenv}/

integration/integration_test.go

@@ -0,0 +1,192 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/GoogleContainerTools/kaniko/testutil"
+)
+
+const (
+	executorImage      = "executor-image"
+	dockerImage        = "gcr.io/cloud-builders/docker"
+	ubuntuImage        = "ubuntu"
+	testRepo           = "gcr.io/kaniko-test/"
+	dockerPrefix       = "docker-"
+	kanikoPrefix       = "kaniko-"
+	daemonPrefix       = "daemon://"
+	kanikoTestBucket   = "kaniko-test-bucket"
+	dockerfilesPath    = "dockerfiles"
+	onbuildBaseImage   = testRepo + "onbuild-base:latest"
+	buildContextPath   = "/workspace"
+	emptyContainerDiff = `[
+     {
+       "Image1": "%s:latest",
+       "Image2": "%s:latest",
+       "DiffType": "File",
+       "Diff": {
+	 	"Adds": null,
+	 	"Dels": null,
+	 	"Mods": null
+       }
+     }
+   ]`
+)
+
+func TestMain(m *testing.M) {
+	buildKaniko := exec.Command("docker", "build", "-t", executorImage, "-f", "../deploy/Dockerfile", "..")
+	err := buildKaniko.Run()
+	if err != nil {
+		fmt.Print(err)
+		fmt.Print("Building kaniko failed.")
+		os.Exit(1)
+	}
+
+	// Make sure container-diff is on user's PATH
+	_, err = exec.LookPath("container-diff")
+	if err != nil {
+		fmt.Print("Make sure you have container-diff installed and on your PATH")
+		os.Exit(1)
+	}
+
+	os.Exit(m.Run())
+}
+
+func TestRun(t *testing.T) {
+	dockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile*"))
+	if err != nil {
+		t.Error(err)
+		t.FailNow()
+	}
+
+	// Map for test Dockerfile to expected ARGs
+	argsMap := map[string][]string{
+		"Dockerfile_test_run":     {"file=/file"},
+		"Dockerfile_test_workdir": {"workdir=/arg/workdir"},
+		"Dockerfile_test_add":     {"file=context/foo"},
+		"Dockerfile_test_onbuild": {"file=/tmp/onbuild"},
+		"Dockerfile_test_scratch": {
+			"hello=hello-value",
+			"file=context/foo",
+			"file3=context/b*",
+		},
+	}
+
+	bucketContextTests := []string{"Dockerfile_test_copy_bucket"}
+
+	_, ex, _, _ := runtime.Caller(0)
+	cwd := filepath.Dir(ex)
+
+	for _, dockerfile := range dockerfiles {
+		t.Run("test_"+dockerfile, func(t *testing.T) {
+			dockerfile = dockerfile[len("dockerfile/")+1:]
+			t.Logf("%s\n", dockerfile)
+
+			var buildArgs []string
+			buildArgFlag := "--build-arg"
+			for _, arg := range argsMap[dockerfile] {
+				buildArgs = append(buildArgs, buildArgFlag)
+				buildArgs = append(buildArgs, arg)
+			}
+			// build docker image
+			dockerImage := strings.ToLower(testRepo + dockerPrefix + dockerfile)
+			dockerCmd := exec.Command("docker",
+				append([]string{"build",
+					"-t", dockerImage,
+					"-f", path.Join(dockerfilesPath, dockerfile),
+					"."},
+					buildArgs...)...,
+			)
+			RunCommand(dockerCmd, t)
+
+			contextFlag := "-c"
+			contextPath := buildContextPath
+			for _, d := range bucketContextTests {
+				if d == dockerfile {
+					contextFlag = "-b"
+					contextPath = kanikoTestBucket
+					break
+				}
+			}
+
+			// build kaniko image
+			kanikoImage := strings.ToLower(testRepo + kanikoPrefix + dockerfile)
+			kanikoCmd := exec.Command("docker",
+				append([]string{"run",
+					"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
+					"-v", cwd + ":/workspace",
+					executorImage,
+					"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
+					"-d", kanikoImage,
+					contextFlag, contextPath},
+					buildArgs...)...,
+			)
+
+			RunCommand(kanikoCmd, t)
+
+			// container-diff
+			daemonDockerImage := daemonPrefix + dockerImage
+			containerdiffCmd := exec.Command("container-diff", "diff",
+				daemonDockerImage, kanikoImage,
+				"-q", "--type=file", "--json")
+			diff := RunCommand(containerdiffCmd, t)
+			t.Logf("diff = %s", string(diff))
+
+			expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage)
+
+			// Let's compare the json objects themselves instead of strings to avoid
+			// issues with spaces and indents
+			var diffInt interface{}
+			var expectedInt interface{}
+
+			err = json.Unmarshal(diff, &diffInt)
+			if err != nil {
+				t.Error(err)
+				t.Fail()
+			}
+
+			err = json.Unmarshal([]byte(expected), &expectedInt)
+			if err != nil {
+				t.Error(err)
+				t.Fail()
+			}
+
+			testutil.CheckErrorAndDeepEqual(t, false, nil, expectedInt, diffInt)
+		})
+	}
+}
+
+func RunCommand(cmd *exec.Cmd, t *testing.T) []byte {
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Log(cmd.Args)
+		t.Log(string(output))
+		t.Error(err)
+		t.FailNow()
+	}
+
+	return output
+}

integration_tests/dockerfiles/config_test_add.json

@@ -1,12 +0,0 @@
-[
-  {
-    "Image1": "gcr.io/kaniko-test/docker-test-add:latest",
-    "Image2": "gcr.io/kaniko-test/kaniko-test-add:latest",
-    "DiffType": "File",
-    "Diff": {
-      "Adds": null,
-      "Dels": null,
-      "Mods": null
-    }
-  }
-]

integration_tests/dockerfiles/config_test_bucket_buildcontext.json

@@ -1,12 +0,0 @@
-[
-  {
-    "Image1": "gcr.io/kaniko-test/docker-test-bucket-buildcontext:latest",
-    "Image2": "gcr.io/kaniko-test/kaniko-test-bucket-buildcontext:latest",
-    "DiffType": "File",
-    "Diff": {
-      "Adds": null,
-      "Dels": null,
-      "Mods": null
-    }
-  }
-]

integration_tests/dockerfiles/config_test_copy.json

@@ -1,12 +0,0 @@
-[
-  {
-    "Image1": "gcr.io/kaniko-test/docker-test-copy:latest",
-    "Image2": "gcr.io/kaniko-test/kaniko-test-copy:latest",
-    "DiffType": "File",
-    "Diff": {
-      "Adds": null,
-      "Dels": null,
-      "Mods": null
-    }
-  }
-]

integration_tests/dockerfiles/config_test_extract_fs.json

@@ -1,12 +0,0 @@
-[
-  {
-    "Image1": "gcr.io/kaniko-test/docker-extract-filesystem:latest",
-    "Image2": "gcr.io/kaniko-test/kaniko-extract-filesystem:latest",
-    "DiffType": "File",
-    "Diff": {
-      "Adds": null,
-      "Dels": null,
-      "Mods": null
-    }
-  }
-]

integration_tests/dockerfiles/config_test_mv_add.json

@@ -1,12 +0,0 @@
-[
-  {
-    "Image1": "gcr.io/kaniko-test/docker-test-mv-add:latest",
-    "Image2": "gcr.io/kaniko-test/kaniko-test-mv-add:latest",
-    "DiffType": "File",
-    "Diff": {
-      "Adds": null,
-      "Dels": null,
-      "Mods": null
-    }
-  }
-]

integration_tests/dockerfiles/config_test_onbuild.json

@@ -1,12 +0,0 @@
-[
-  {
-    "Image1": "gcr.io/kaniko-test/docker-test-onbuild:latest",
-    "Image2": "gcr.io/kaniko-test/kaniko-test-onbuild:latest",
-    "DiffType": "File",
-    "Diff": {
-      "Adds": null,
-      "Dels": null,
-      "Mods": null
-    }
-  }
-]

integration_tests/dockerfiles/config_test_registry.json

@@ -1,12 +0,0 @@
-[
-  {
-    "Image1": "gcr.io/kaniko-test/docker-test-registry:latest",
-    "Image2": "gcr.io/kaniko-test/kaniko-test-registry:latest",
-    "DiffType": "File",
-    "Diff": {
-      "Adds": null,
-      "Dels": null,
-      "Mods": null
-    }
-  }
-]

integration_tests/dockerfiles/config_test_run.json

@@ -1,12 +0,0 @@
-[
-  {
-    "Image1": "gcr.io/kaniko-test/docker-test-run:latest",
-    "Image2": "gcr.io/kaniko-test/kaniko-test-run:latest",
-    "DiffType": "File",
-    "Diff": {
-      "Adds": null,
-      "Dels": null,
-      "Mods": null
-    }
-  }
-]

integration_tests/dockerfiles/config_test_run_2.json

@@ -1,12 +0,0 @@
-[
-  {
-    "Image1": "gcr.io/kaniko-test/docker-test-run-2:latest",
-    "Image2": "gcr.io/kaniko-test/kaniko-test-run-2:latest",
-    "DiffType": "File",
-    "Diff": {
-      "Adds": null,
-      "Dels": null,
-      "Mods": null
-    }
-  }
-]

integration_tests/dockerfiles/config_test_scratch.json

@@ -1,12 +0,0 @@
-[
-  {
-    "Image1": "gcr.io/kaniko-test/docker-test-scratch:latest",
-    "Image2": "gcr.io/kaniko-test/kaniko-test-scratch:latest",
-    "DiffType": "File",
-    "Diff": {
-      "Adds": null,
-      "Dels": null,
-      "Mods": null
-    }
-  }
-]

integration_tests/dockerfiles/config_test_volume.json

@@ -1,12 +0,0 @@
-[
-  {
-    "Image1": "gcr.io/kaniko-test/docker-test-volume:latest",
-    "Image2": "gcr.io/kaniko-test/kaniko-test-volume:latest",
-    "DiffType": "File",
-    "Diff": {
-      "Adds": null,
-      "Dels": null,
-      "Mods": null
-    }
-  }
-]

integration_tests/dockerfiles/config_test_workdir.json

@@ -1,12 +0,0 @@
-[
-  {
-    "Image1": "gcr.io/kaniko-test/docker-test-workdir:latest",
-    "Image2": "gcr.io/kaniko-test/kaniko-test-workdir:latest",
-    "DiffType": "File",
-    "Diff": {
-      "Adds": null,
-      "Dels": null,
-      "Mods": null
-    }
-  }
-]

integration_tests/dockerfiles/test_env.yaml

@@ -1,41 +0,0 @@
-schemaVersion: '2.0.0'
-metadataTest:
-  env:
-    - key: hey
-      value: hello
-    - key: PATH
-      value: something
-    - key: first
-      value: foo
-    - key: second
-      value: foo2
-    - key: third
-      value: foo2:/third
-    - key: myName
-      value: John Doe
-    - key: myDog
-      value: Rex The Dog
-    - key: myCat
-      value: fluffy
-    - key: test
-      value: value value2
-    - key: test1
-      value: a'b'c
-    - key: test2
-      value: a"b"c
-    - key: test3
-      value: a b
-    - key: name2
-      value: b c
-    - key: test4
-      value: a"b
-    - key: test5
-      value: a\"b
-    - key: test6
-      value: a\'b
-    - key: atomic
-      value: two
-    - key: newatomic
-      value: one
-    - key: newenv
-      value: /newenv

integration_tests/dockerfiles/test_metadata.yaml

@@ -1,4 +0,0 @@
-schemaVersion: '2.0.0'
-metadataTest:
-  cmd: ["/bin/sh", "-c", "echo \"hello\""]
-  entrypoint: ["execute", "entrypoint"]

integration_tests/dockerfiles/test_user.yaml

@@ -1,15 +0,0 @@
-schemaVersion: '2.0.0'
-commandTests:
-- name: 'whoami'
-  command: 'whoami'
-  expectedOutput: ['testuser']
-  excludedOutput: ['root']
-- name: 'file owner'
-  command: 'ls'
-  args: ['-l', '/tmp/foo']
-  expectedOutput: ['.*testuser.*', '.*testgroup.*']
-  excludedOutput: ['.*root.*']
-fileContentTests:
-- name: "/tmp/foo"
-  path: "/tmp/foo"
-  expectedContent: ["hey"]

pkg/util/command_util_test.go

@@ -107,7 +107,7 @@ func Test_EnvReplacement(t *testing.T) {
 	}
 }
 
-var buildContextPath = "../../integration_tests/"
+var buildContextPath = "../../integration/"
 
 var destinationFilepathTests = []struct {
 	src              string

pkg/util/fs_util_test.go

@@ -475,7 +475,7 @@ func TestExtractFile(t *testing.T) {
 			},
 		},
 		{
-			name: "symlink parent does not exist",
+			name: "symlink parent does not exist 2",
 			hdrs: []*tar.Header{linkHeader("./foo/bar/baz", "../../bat")},
 			checkers: []checker{
 				linkPointsTo("/foo/bar/baz", "../../bat"),

test.sh

@@ -21,7 +21,7 @@ GREEN='\033[0;32m'
 RESET='\033[0m'
 
 echo "Running go tests..."
-go test -cover -v -timeout 60s `go list ./... | grep -v vendor` | sed ''/PASS/s//$(printf "${GREEN}PASS${RESET}")/'' | sed ''/FAIL/s//$(printf "${RED}FAIL${RESET}")/''
+go test -cover -v -timeout 60s `go list ./... | grep -v vendor | grep -v integration` | sed ''/PASS/s//$(printf "${GREEN}PASS${RESET}")/'' | sed ''/FAIL/s//$(printf "${RED}FAIL${RESET}")/''
 GO_TEST_EXIT_CODE=${PIPESTATUS[0]}
 if [[ $GO_TEST_EXIT_CODE -ne 0 ]]; then
     exit $GO_TEST_EXIT_CODE