Integration test refactoring (#126)

* integration test refactoring

* config file cleanup

* more test refactoring

* remove debug file

* moving around more files

* fixing up integration tests

* integration tests work

* some housekeeping

* fixing tests

* addressing comments

* debugging

* debugging

* actual debugging

* skip integration tests for travis

* install container-diff before integration tests

* syntax

* make test failures less noisy

* fixing tests

* hopefully fixing CI?

* fixes

* more fixes

* let's actually fix CI

* more testing

* testing

* proper auth

* typos

* adding support for args in integration tests

* formatting

* formatting

* adding support for testing bucket context

* adding bucket test dockerfile

* addressing comments

* syntax

integration/context/arr[0].txt

@@ -0,0 +1 @@
+hello

integration/context/bar/bam/bat

@@ -0,0 +1 @@
+bat

integration/context/bar/bat

@@ -0,0 +1 @@
+bat

integration/context/bar/baz

@@ -0,0 +1 @@
+baz

integration/context/foo

@@ -0,0 +1 @@
+foo

integration/context/tars/file

@@ -0,0 +1 @@
+normal file

integration/dockerfiles/Dockerfile_onbuild_base

@@ -0,0 +1,6 @@
+FROM gcr.io/google-appengine/debian9:latest
+ENV dir /tmp/dir/
+ONBUILD RUN echo "onbuild" > /tmp/onbuild
+ONBUILD RUN  mkdir $dir
+ONBUILD RUN echo "onbuild 2" > ${dir}/onbuild2
+ONBUILD WORKDIR /new/workdir

integration/dockerfiles/Dockerfile_test_add

@@ -0,0 +1,22 @@
+FROM gcr.io/google-appengine/debian9:latest
+# First, try adding some regular files
+ADD context/foo foo
+ADD context/foo /foodir/
+ADD context/bar/b* bar/
+ADD . newdir
+ADD ["context/foo", "/tmp/foo" ]
+
+# Next, make sure environment replacement works
+ENV contextenv ./context
+ADD $contextenv/* /tmp/${contextenv}/
+
+# Now, test extracting local tar archives
+ADD context/tars/fil* /tars/
+ADD context/tars/file.tar /tars_again
+
+# Test with ARG
+ARG file
+COPY $file /arg
+
+# Finally, test adding a remote URL, concurrently with a normal file
+ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v1.4.3/docker-credential-gcr_linux_386-1.4.3.tar.gz context/foo /test/all/

integration/dockerfiles/Dockerfile_test_copy

@@ -0,0 +1,20 @@
+FROM alpine:3.7
+COPY context/foo foo
+COPY context/foo /foodir/
+COPY context/bar/b* bar/
+COPY context/fo? /foo2
+COPY context/bar/doesnotexist* context/foo hello
+COPY ./context/empty /empty
+COPY ./ dir/
+COPY . newdir
+COPY context/bar /baz/
+COPY ["context/foo", "/tmp/foo" ]
+COPY context/b* /baz/
+COPY context/foo context/bar/ba? /test/
+COPY context/arr[[]0].txt /mydir/
+COPY context/bar/bat .
+
+ENV contextenv ./context
+COPY ${contextenv}/foo /tmp/foo2
+COPY $contextenv/foo /tmp/foo3
+COPY $contextenv/* /tmp/${contextenv}/

integration/dockerfiles/Dockerfile_test_copy_bucket

@@ -0,0 +1,20 @@
+FROM alpine:3.7
+COPY context/foo foo
+COPY context/foo /foodir/
+COPY context/bar/b* bar/
+COPY context/fo? /foo2
+COPY context/bar/doesnotexist* context/foo hello
+COPY ./context/empty /empty
+COPY ./ dir/
+COPY . newdir
+COPY context/bar /baz/
+COPY ["context/foo", "/tmp/foo" ]
+COPY context/b* /baz/
+COPY context/foo context/bar/ba? /test/
+COPY context/arr[[]0].txt /mydir/
+COPY context/bar/bat .
+
+ENV contextenv ./context
+COPY ${contextenv}/foo /tmp/foo2
+COPY $contextenv/foo /tmp/foo3
+COPY $contextenv/* /tmp/${contextenv}/

integration/dockerfiles/Dockerfile_test_env

@@ -0,0 +1,19 @@
+FROM gcr.io/google-appengine/debian9
+ENV hey hey
+ENV PATH /usr/local
+ENV hey hello
+ENV first=foo second=foo2
+ENV third $second:/third
+ENV myName="John Doe" myDog=Rex\ The\ Dog \
+    myCat=fluffy
+ENV PATH something
+ENV test=value\ value2
+ENV test1="a'b'c"
+ENV test2='a"b"c'
+ENV test3=a\ b name2=b\ c
+ENV test4="a\"b"
+ENV test5='a\"b'
+ENV test6="a\'b"
+ENV atomic=one
+ENV atomic=two newatomic=$atomic
+ENV newenv=$doesntexist/newenv

integration/dockerfiles/Dockerfile_test_extract_fs

@@ -0,0 +1,15 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/google-appengine/debian9:latest

integration/dockerfiles/Dockerfile_test_metadata

@@ -0,0 +1,7 @@
+FROM gcr.io/distroless/base:latest
+CMD ["command", "one"]
+CMD ["command", "two"]
+CMD echo "hello"
+
+ENTRYPOINT ["execute", "something"]
+ENTRYPOINT ["execute", "entrypoint"]

integration/dockerfiles/Dockerfile_test_mv_add

@@ -0,0 +1,3 @@
+FROM busybox
+ADD context/tars /tmp/tars
+RUN mv /tmp/tars /foo

integration/dockerfiles/Dockerfile_test_onbuild

@@ -0,0 +1,7 @@
+FROM gcr.io/kaniko-test/onbuild-base:latest
+COPY context/foo foo
+ENV dir /new/workdir/
+ARG file
+ONBUILD RUN echo "onbuild" > $file
+ONBUILD RUN echo "onbuild 2" > ${dir}
+ONBUILD WORKDIR /new/workdir

integration/dockerfiles/Dockerfile_test_registry

@@ -0,0 +1,2 @@
+FROM busybox
+RUN echo "hey" > /hey

integration/dockerfiles/Dockerfile_test_run

@@ -0,0 +1,23 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/google-appengine/debian9
+RUN echo "hey" > /etc/foo
+RUN echo "baz" > /etc/baz
+RUN cp /etc/baz /etc/bar
+RUN rm /etc/baz
+
+# Test with ARG
+ARG file
+RUN echo "run" > $file

integration/dockerfiles/Dockerfile_test_run_2

@@ -0,0 +1,20 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test to make sure the executor builds an image correctly
+# when no files are changed
+
+FROM gcr.io/google-appengine/debian9
+RUN echo "hey" 
+MAINTAINER kaniko

integration/dockerfiles/Dockerfile_test_scratch

@@ -0,0 +1,14 @@
+FROM scratch
+# First, make sure simple arg replacement works
+ARG file
+COPY $file /foo
+# Check that setting a default value works
+ARG file2=context/bar/bat
+COPY $file2 /bat
+# Check that overriding a default value works
+ARG file3=context/bar/baz
+COPY $file3 /baz
+# Check that setting an ENV will override the ARG
+ENV file context/bar/bam/bat
+COPY $file /env 
+

integration/dockerfiles/Dockerfile_test_user_run

@@ -0,0 +1,19 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/google-appengine/debian9
+RUN useradd testuser
+RUN groupadd testgroup
+USER testuser:testgroup
+RUN echo "hey" > /tmp/foo

integration/dockerfiles/Dockerfile_test_volume

@@ -0,0 +1,7 @@
+FROM gcr.io/google-appengine/debian9
+RUN mkdir /foo
+RUN echo "hello" > /foo/hey
+VOLUME /foo/bar /tmp
+ENV VOL /baz/bat
+VOLUME ["${VOL}"]
+RUN echo "hello again" > /tmp/hey

integration/dockerfiles/Dockerfile_test_workdir

@@ -0,0 +1,17 @@
+FROM gcr.io/google-appengine/debian9@sha256:6b3aa04751aa2ac3b0c7be4ee71148b66d693ad212ce6d3244bd2a2a147f314a
+COPY context/foo foo
+WORKDIR /test
+# Test that this will be appended on to the previous command, to create /test/workdir
+WORKDIR workdir 
+COPY context/foo ./currentfoo
+# Test that the RUN command will happen in the correct directory
+RUN cp currentfoo newfoo
+WORKDIR /new/dir
+ENV dir /another/new/dir
+WORKDIR $dir/newdir
+WORKDIR $dir/$doesntexist
+WORKDIR /
+
+# Test with ARG
+ARG workdir
+WORKDIR $workdir

integration/integration_test.go

@@ -0,0 +1,192 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/GoogleContainerTools/kaniko/testutil"
+)
+
+const (
+	executorImage      = "executor-image"
+	dockerImage        = "gcr.io/cloud-builders/docker"
+	ubuntuImage        = "ubuntu"
+	testRepo           = "gcr.io/kaniko-test/"
+	dockerPrefix       = "docker-"
+	kanikoPrefix       = "kaniko-"
+	daemonPrefix       = "daemon://"
+	kanikoTestBucket   = "kaniko-test-bucket"
+	dockerfilesPath    = "dockerfiles"
+	onbuildBaseImage   = testRepo + "onbuild-base:latest"
+	buildContextPath   = "/workspace"
+	emptyContainerDiff = `[
+     {
+       "Image1": "%s:latest",
+       "Image2": "%s:latest",
+       "DiffType": "File",
+       "Diff": {
+	 	"Adds": null,
+	 	"Dels": null,
+	 	"Mods": null
+       }
+     }
+   ]`
+)
+
+func TestMain(m *testing.M) {
+	buildKaniko := exec.Command("docker", "build", "-t", executorImage, "-f", "../deploy/Dockerfile", "..")
+	err := buildKaniko.Run()
+	if err != nil {
+		fmt.Print(err)
+		fmt.Print("Building kaniko failed.")
+		os.Exit(1)
+	}
+
+	// Make sure container-diff is on user's PATH
+	_, err = exec.LookPath("container-diff")
+	if err != nil {
+		fmt.Print("Make sure you have container-diff installed and on your PATH")
+		os.Exit(1)
+	}
+
+	os.Exit(m.Run())
+}
+
+func TestRun(t *testing.T) {
+	dockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile*"))
+	if err != nil {
+		t.Error(err)
+		t.FailNow()
+	}
+
+	// Map for test Dockerfile to expected ARGs
+	argsMap := map[string][]string{
+		"Dockerfile_test_run":     {"file=/file"},
+		"Dockerfile_test_workdir": {"workdir=/arg/workdir"},
+		"Dockerfile_test_add":     {"file=context/foo"},
+		"Dockerfile_test_onbuild": {"file=/tmp/onbuild"},
+		"Dockerfile_test_scratch": {
+			"hello=hello-value",
+			"file=context/foo",
+			"file3=context/b*",
+		},
+	}
+
+	bucketContextTests := []string{"Dockerfile_test_copy_bucket"}
+
+	_, ex, _, _ := runtime.Caller(0)
+	cwd := filepath.Dir(ex)
+
+	for _, dockerfile := range dockerfiles {
+		t.Run("test_"+dockerfile, func(t *testing.T) {
+			dockerfile = dockerfile[len("dockerfile/")+1:]
+			t.Logf("%s\n", dockerfile)
+
+			var buildArgs []string
+			buildArgFlag := "--build-arg"
+			for _, arg := range argsMap[dockerfile] {
+				buildArgs = append(buildArgs, buildArgFlag)
+				buildArgs = append(buildArgs, arg)
+			}
+			// build docker image
+			dockerImage := strings.ToLower(testRepo + dockerPrefix + dockerfile)
+			dockerCmd := exec.Command("docker",
+				append([]string{"build",
+					"-t", dockerImage,
+					"-f", path.Join(dockerfilesPath, dockerfile),
+					"."},
+					buildArgs...)...,
+			)
+			RunCommand(dockerCmd, t)
+
+			contextFlag := "-c"
+			contextPath := buildContextPath
+			for _, d := range bucketContextTests {
+				if d == dockerfile {
+					contextFlag = "-b"
+					contextPath = kanikoTestBucket
+					break
+				}
+			}
+
+			// build kaniko image
+			kanikoImage := strings.ToLower(testRepo + kanikoPrefix + dockerfile)
+			kanikoCmd := exec.Command("docker",
+				append([]string{"run",
+					"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
+					"-v", cwd + ":/workspace",
+					executorImage,
+					"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
+					"-d", kanikoImage,
+					contextFlag, contextPath},
+					buildArgs...)...,
+			)
+
+			RunCommand(kanikoCmd, t)
+
+			// container-diff
+			daemonDockerImage := daemonPrefix + dockerImage
+			containerdiffCmd := exec.Command("container-diff", "diff",
+				daemonDockerImage, kanikoImage,
+				"-q", "--type=file", "--json")
+			diff := RunCommand(containerdiffCmd, t)
+			t.Logf("diff = %s", string(diff))
+
+			expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage)
+
+			// Let's compare the json objects themselves instead of strings to avoid
+			// issues with spaces and indents
+			var diffInt interface{}
+			var expectedInt interface{}
+
+			err = json.Unmarshal(diff, &diffInt)
+			if err != nil {
+				t.Error(err)
+				t.Fail()
+			}
+
+			err = json.Unmarshal([]byte(expected), &expectedInt)
+			if err != nil {
+				t.Error(err)
+				t.Fail()
+			}
+
+			testutil.CheckErrorAndDeepEqual(t, false, nil, expectedInt, diffInt)
+		})
+	}
+}
+
+func RunCommand(cmd *exec.Cmd, t *testing.T) []byte {
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Log(cmd.Args)
+		t.Log(string(output))
+		t.Error(err)
+		t.FailNow()
+	}
+
+	return output
+}