Merge pull request #486 from priyawadhwa/dockerignore

Add support for .dockerignore file

cmd/executor/cmd/root.go

@@ -162,7 +162,7 @@ func resolveDockerfilePath() error {
 // copy Dockerfile to /kaniko/Dockerfile so that if it's specified in the .dockerignore
 // it won't be copied into the image
 func copyDockerfile() error {
-	if err := util.CopyFile(opts.DockerfilePath, constants.DockerfilePath); err != nil {
+	if _, err := util.CopyFile(opts.DockerfilePath, constants.DockerfilePath, ""); err != nil {
 		return errors.Wrap(err, "copying dockerfile")
 	}
 	opts.DockerfilePath = constants.DockerfilePath

integration/.dockerignore

@@ -0,0 +1,3 @@
+# A .dockerignore file to make sure dockerignore support works
+ignore/**
+!ignore/foo

integration/dockerfiles/Dockerfile_test_dockerignore

@@ -0,0 +1,5 @@
+# This dockerfile makes sure the .dockerignore is working
+# If so then ignore/foo should copy to /foo
+# If not, then this image won't build because it will attempt to copy three files to /foo, which is a file not a directory
+FROM scratch 
+COPY ignore/* /foo

integration/dockerignore.go

@@ -1,112 +0,0 @@
-/*
-Copyright 2018 Google LLC
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package integration
-
-import (
-	"fmt"
-	"os"
-	"os/exec"
-	"path"
-	"path/filepath"
-	"runtime"
-	"strings"
-)
-
-var filesToIgnore = []string{"ignore/fo*", "!ignore/foobar", "ignore/Dockerfile_test_ignore"}
-
-const (
-	ignoreDir                = "ignore"
-	ignoreDockerfile         = "Dockerfile_test_ignore"
-	ignoreDockerfileContents = `FROM scratch
-	COPY . .`
-)
-
-// Set up a test dir to ignore with the structure:
-// ignore
-//  -- Dockerfile_test_ignore
-//  -- foo
-//  -- foobar
-
-func setupIgnoreTestDir() error {
-	if err := os.MkdirAll(ignoreDir, 0750); err != nil {
-		return err
-	}
-	// Create and write contents to dockerfile
-	path := filepath.Join(ignoreDir, ignoreDockerfile)
-	f, err := os.Create(path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	if _, err := f.Write([]byte(ignoreDockerfileContents)); err != nil {
-		return err
-	}
-
-	additionalFiles := []string{"ignore/foo", "ignore/foobar"}
-	for _, add := range additionalFiles {
-		a, err := os.Create(add)
-		if err != nil {
-			return err
-		}
-		defer a.Close()
-	}
-	return generateDockerIgnore()
-}
-
-// generate the .dockerignore file
-func generateDockerIgnore() error {
-	f, err := os.Create(".dockerignore")
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	contents := strings.Join(filesToIgnore, "\n")
-	if _, err := f.Write([]byte(contents)); err != nil {
-		return err
-	}
-	return nil
-}
-
-func generateDockerignoreImages(imageRepo string) error {
-
-	dockerfilePath := filepath.Join(ignoreDir, ignoreDockerfile)
-
-	dockerImage := strings.ToLower(imageRepo + dockerPrefix + ignoreDockerfile)
-	dockerCmd := exec.Command("docker", "build",
-		"-t", dockerImage,
-		"-f", path.Join(dockerfilePath),
-		".")
-	_, err := RunCommandWithoutTest(dockerCmd)
-	if err != nil {
-		return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s", dockerImage, dockerCmd.Args, err)
-	}
-
-	_, ex, _, _ := runtime.Caller(0)
-	cwd := filepath.Dir(ex)
-	kanikoImage := GetKanikoImage(imageRepo, ignoreDockerfile)
-	kanikoCmd := exec.Command("docker",
-		"run",
-		"-v", os.Getenv("HOME")+"/.config/gcloud:/root/.config/gcloud",
-		"-v", cwd+":/workspace",
-		ExecutorImage,
-		"-f", path.Join(buildContextPath, dockerfilePath),
-		"-d", kanikoImage,
-		"-c", buildContextPath)
-
-	_, err = RunCommandWithoutTest(kanikoCmd)
-	return err
-}

integration/integration_test.go

@@ -286,34 +286,6 @@ func TestCache(t *testing.T) {
 	}
 }
 
-func TestDockerignore(t *testing.T) {
-	// TODO (priyawadhwa@): remove this once .dockerignore is implemented correctly
-	t.Skip()
-
-	t.Run(fmt.Sprintf("test_%s", ignoreDockerfile), func(t *testing.T) {
-		if err := setupIgnoreTestDir(); err != nil {
-			t.Fatalf("error setting up ignore test dir: %v", err)
-		}
-		if err := generateDockerignoreImages(config.imageRepo); err != nil {
-			t.Fatalf("error generating dockerignore test images: %v", err)
-		}
-
-		dockerImage := GetDockerImage(config.imageRepo, ignoreDockerfile)
-		kanikoImage := GetKanikoImage(config.imageRepo, ignoreDockerfile)
-
-		// container-diff
-		daemonDockerImage := daemonPrefix + dockerImage
-		containerdiffCmd := exec.Command("container-diff", "diff",
-			daemonDockerImage, kanikoImage,
-			"-q", "--type=file", "--type=metadata", "--json")
-		diff := RunCommand(containerdiffCmd, t)
-		t.Logf("diff = %s", string(diff))
-
-		expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage)
-		checkContainerDiffOutput(t, diff, expected)
-	})
-}
-
 type fileDiff struct {
 	Name string
 	Size int

pkg/commands/copy.go

@@ -70,22 +70,30 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu
 				// we need to add '/' to the end to indicate the destination is a directory
 				dest = filepath.Join(cwd, dest) + "/"
 			}
-			copiedFiles, err := util.CopyDir(fullPath, dest)
+			copiedFiles, err := util.CopyDir(fullPath, dest, c.buildcontext)
 			if err != nil {
 				return err
 			}
 			c.snapshotFiles = append(c.snapshotFiles, copiedFiles...)
 		} else if fi.Mode()&os.ModeSymlink != 0 {
 			// If file is a symlink, we want to create the same relative symlink
-			if err := util.CopySymlink(fullPath, destPath); err != nil {
+			exclude, err := util.CopySymlink(fullPath, destPath, c.buildcontext)
+			if err != nil {
 				return err
 			}
+			if exclude {
+				continue
+			}
 			c.snapshotFiles = append(c.snapshotFiles, destPath)
 		} else {
 			// ... Else, we want to copy over a file
-			if err := util.CopyFile(fullPath, destPath); err != nil {
+			exclude, err := util.CopyFile(fullPath, destPath, c.buildcontext)
+			if err != nil {
 				return err
 			}
+			if exclude {
+				continue
+			}
 			c.snapshotFiles = append(c.snapshotFiles, destPath)
 		}
 	}

pkg/constants/constants.go

@@ -66,6 +66,9 @@ const (
 	// Docker command names
 	Cmd        = "cmd"
 	Entrypoint = "entrypoint"
+
+	// Name of the .dockerignore file
+	Dockerignore = ".dockerignore"
 )
 
 // KanikoBuildFiles is the list of files required to build kaniko

pkg/dockerfile/dockerfile.go

@@ -20,13 +20,11 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
-	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
-	"github.com/docker/docker/builder/dockerignore"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/moby/buildkit/frontend/dockerfile/parser"
 	"github.com/pkg/errors"
@@ -173,20 +171,3 @@ func saveStage(index int, stages []instructions.Stage) bool {
 	}
 	return false
 }
-
-// DockerignoreExists returns true if .dockerignore exists in the source context
-func DockerignoreExists(opts *config.KanikoOptions) bool {
-	path := filepath.Join(opts.SrcContext, ".dockerignore")
-	return util.FilepathExists(path)
-}
-
-// ParseDockerignore returns a list of all paths in .dockerignore
-func ParseDockerignore(opts *config.KanikoOptions) ([]string, error) {
-	path := filepath.Join(opts.SrcContext, ".dockerignore")
-	contents, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, errors.Wrap(err, "parsing .dockerignore")
-	}
-	reader := bytes.NewBuffer(contents)
-	return dockerignore.ReadAll(reader)
-}

pkg/executor/build.go

@@ -345,6 +345,9 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 	if err != nil {
 		return nil, err
 	}
+	if err := util.GetExcludedFiles(opts.SrcContext); err != nil {
+		return nil, err
+	}
 	// Some stages may refer to other random images, not previous stages
 	if err := fetchExtraStages(stages, opts); err != nil {
 		return nil, err

pkg/util/command_util.go

@@ -186,7 +186,14 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []stri
 	dest := srcsAndDest[len(srcsAndDest)-1]
 
 	if !ContainsWildcards(srcs) {
-		if len(srcs) > 1 && !IsDestDir(dest) {
+		totalSrcs := 0
+		for _, src := range srcs {
+			if excludeFile(src, root) {
+				continue
+			}
+			totalSrcs++
+		}
+		if totalSrcs > 1 && !IsDestDir(dest) {
 			return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'")
 		}
 	}
@@ -216,7 +223,12 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []stri
 		if err != nil {
 			return err
 		}
-		totalFiles += len(files)
+		for _, file := range files {
+			if excludeFile(file, root) {
+				continue
+			}
+			totalFiles++
+		}
 	}
 	if totalFiles == 0 {
 		return errors.New("copy failed: no source files specified")

pkg/util/command_util_test.go

@@ -249,11 +249,13 @@ func Test_MatchSources(t *testing.T) {
 }
 
 var isSrcValidTests = []struct {
+	name            string
 	srcsAndDest     []string
 	resolvedSources []string
 	shouldErr       bool
 }{
 	{
+		name: "dest isn't directory",
 		srcsAndDest: []string{
 			"context/foo",
 			"context/bar",
@@ -266,6 +268,7 @@ var isSrcValidTests = []struct {
 		shouldErr: true,
 	},
 	{
+		name: "dest is directory",
 		srcsAndDest: []string{
 			"context/foo",
 			"context/bar",
@@ -278,6 +281,7 @@ var isSrcValidTests = []struct {
 		shouldErr: false,
 	},
 	{
+		name: "copy file to file",
 		srcsAndDest: []string{
 			"context/bar/bam",
 			"dest",
@@ -288,16 +292,7 @@ var isSrcValidTests = []struct {
 		shouldErr: false,
 	},
 	{
-		srcsAndDest: []string{
+		name: "copy files with wildcards to dir",
-			"context/foo",
-			"dest",
-		},
-		resolvedSources: []string{
-			"context/foo",
-		},
-		shouldErr: false,
-	},
-	{
 		srcsAndDest: []string{
 			"context/foo",
 			"context/b*",
@@ -310,6 +305,7 @@ var isSrcValidTests = []struct {
 		shouldErr: false,
 	},
 	{
+		name: "copy multilple files with wildcards to file",
 		srcsAndDest: []string{
 			"context/foo",
 			"context/b*",
@@ -322,6 +318,7 @@ var isSrcValidTests = []struct {
 		shouldErr: true,
 	},
 	{
+		name: "copy two files to file, one of which doesn't exist",
 		srcsAndDest: []string{
 			"context/foo",
 			"context/doesntexist*",
@@ -333,6 +330,7 @@ var isSrcValidTests = []struct {
 		shouldErr: false,
 	},
 	{
+		name: "copy dir to dest not specified as dir",
 		srcsAndDest: []string{
 			"context/",
 			"dest",
@@ -343,6 +341,7 @@ var isSrcValidTests = []struct {
 		shouldErr: false,
 	},
 	{
+		name: "copy url to file",
 		srcsAndDest: []string{
 			testURL,
 			"dest",
@@ -352,12 +351,43 @@ var isSrcValidTests = []struct {
 		},
 		shouldErr: false,
 	},
+	{
+		name: "copy two srcs, one excluded, to file",
+		srcsAndDest: []string{
+			"ignore/foo",
+			"ignore/bar",
+			"dest",
+		},
+		resolvedSources: []string{
+			"ignore/foo",
+			"ignore/bar",
+		},
+		shouldErr: false,
+	},
+	{
+		name: "copy two srcs, both excluded, to file",
+		srcsAndDest: []string{
+			"ignore/baz",
+			"ignore/bar",
+			"dest",
+		},
+		resolvedSources: []string{
+			"ignore/baz",
+			"ignore/bar",
+		},
+		shouldErr: true,
+	},
 }
 
 func Test_IsSrcsValid(t *testing.T) {
 	for _, test := range isSrcValidTests {
-		err := IsSrcsValid(test.srcsAndDest, test.resolvedSources, buildContextPath)
+		t.Run(test.name, func(t *testing.T) {
-		testutil.CheckError(t, test.shouldErr, err)
+			if err := GetExcludedFiles(buildContextPath); err != nil {
+				t.Fatalf("error getting excluded files: %v", err)
+			}
+			err := IsSrcsValid(test.srcsAndDest, test.resolvedSources, buildContextPath)
+			testutil.CheckError(t, test.shouldErr, err)
+		})
 	}
 }
 

pkg/util/fs_util.go

@@ -19,7 +19,9 @@ package util
 import (
 	"archive/tar"
 	"bufio"
+	"bytes"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -27,10 +29,11 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/docker/docker/builder/dockerignore"
+	"github.com/docker/docker/pkg/fileutils"
 	"github.com/google/go-containerregistry/pkg/v1"
 	"github.com/pkg/errors"
-
-	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/sirupsen/logrus"
 )
 
@@ -59,6 +62,8 @@ var whitelist = []WhitelistEntry{
 	},
 }
 
+var excluded []string
+
 // GetFSFromImage extracts the layers of img to root
 // It returns a list of all files extracted
 func GetFSFromImage(root string, img v1.Image) ([]string, error) {
@@ -462,7 +467,7 @@ func DownloadFileToDest(rawurl, dest string) error {
 
 // CopyDir copies the file or directory at src to dest
 // It returns a list of files it copied over
-func CopyDir(src, dest string) ([]string, error) {
+func CopyDir(src, dest, buildcontext string) ([]string, error) {
 	files, err := RelativeFiles("", src)
 	if err != nil {
 		return nil, err
@@ -474,6 +479,10 @@ func CopyDir(src, dest string) ([]string, error) {
 		if err != nil {
 			return nil, err
 		}
+		if excludeFile(fullPath, buildcontext) {
+			logrus.Debugf("%s found in .dockerignore, ignoring", src)
+			continue
+		}
 		destPath := filepath.Join(dest, file)
 		if fi.IsDir() {
 			logrus.Debugf("Creating directory %s", destPath)
@@ -489,12 +498,12 @@ func CopyDir(src, dest string) ([]string, error) {
 			}
 		} else if fi.Mode()&os.ModeSymlink != 0 {
 			// If file is a symlink, we want to create the same relative symlink
-			if err := CopySymlink(fullPath, destPath); err != nil {
+			if _, err := CopySymlink(fullPath, destPath, buildcontext); err != nil {
 				return nil, err
 			}
 		} else {
 			// ... Else, we want to copy over a file
-			if err := CopyFile(fullPath, destPath); err != nil {
+			if _, err := CopyFile(fullPath, destPath, buildcontext); err != nil {
 				return nil, err
 			}
 		}
@@ -504,37 +513,78 @@ func CopyDir(src, dest string) ([]string, error) {
 }
 
 // CopySymlink copies the symlink at src to dest
-func CopySymlink(src, dest string) error {
+func CopySymlink(src, dest, buildcontext string) (bool, error) {
+	if excludeFile(src, buildcontext) {
+		logrus.Debugf("%s found in .dockerignore, ignoring", src)
+		return true, nil
+	}
 	link, err := os.Readlink(src)
 	if err != nil {
-		return err
+		return false, err
 	}
 	if FilepathExists(dest) {
 		if err := os.RemoveAll(dest); err != nil {
-			return err
+			return false, err
 		}
 	}
-	return os.Symlink(link, dest)
+	return false, os.Symlink(link, dest)
 }
 
 // CopyFile copies the file at src to dest
-func CopyFile(src, dest string) error {
+func CopyFile(src, dest, buildcontext string) (bool, error) {
+	if excludeFile(src, buildcontext) {
+		logrus.Debugf("%s found in .dockerignore, ignoring", src)
+		return true, nil
+	}
 	fi, err := os.Stat(src)
 	if err != nil {
-		return err
+		return false, err
 	}
 	logrus.Debugf("Copying file %s to %s", src, dest)
 	srcFile, err := os.Open(src)
 	if err != nil {
-		return err
+		return false, err
 	}
 	defer srcFile.Close()
 	uid := fi.Sys().(*syscall.Stat_t).Uid
 	gid := fi.Sys().(*syscall.Stat_t).Gid
-	return CreateFile(dest, srcFile, fi.Mode(), uid, gid)
+	return false, CreateFile(dest, srcFile, fi.Mode(), uid, gid)
 }
 
-// HasFilepathPrefix checks if the given file path begins with prefix
+// GetExcludedFiles gets a list of files to exclude from the .dockerignore
+func GetExcludedFiles(buildcontext string) error {
+	path := filepath.Join(buildcontext, ".dockerignore")
+	if !FilepathExists(path) {
+		return nil
+	}
+	contents, err := ioutil.ReadFile(path)
+	if err != nil {
+		return errors.Wrap(err, "parsing .dockerignore")
+	}
+	reader := bytes.NewBuffer(contents)
+	excluded, err = dockerignore.ReadAll(reader)
+	return err
+}
+
+// excludeFile returns true if the .dockerignore specified this file should be ignored
+func excludeFile(path, buildcontext string) bool {
+	if HasFilepathPrefix(path, buildcontext, false) {
+		var err error
+		path, err = filepath.Rel(buildcontext, path)
+		if err != nil {
+			logrus.Errorf("unable to get relative path, including %s in build: %v", path, err)
+			return false
+		}
+	}
+	match, err := fileutils.Matches(path, excluded)
+	if err != nil {
+		logrus.Errorf("error matching, including %s in build: %v", path, err)
+		return false
+	}
+	return match
+}
+
+// HasFilepathPrefix checks  if the given file path begins with prefix
 func HasFilepathPrefix(path, prefix string, prefixMatchOnly bool) bool {
 	path = filepath.Clean(path)
 	prefix = filepath.Clean(prefix)

pkg/util/fs_util_test.go

@@ -583,7 +583,7 @@ func TestCopySymlink(t *testing.T) {
 			if err := os.Symlink(tc.linkTarget, link); err != nil {
 				t.Fatal(err)
 			}
-			if err := CopySymlink(link, dest); err != nil {
+			if _, err := CopySymlink(link, dest, ""); err != nil {
 				t.Fatal(err)
 			}
 			got, err := os.Readlink(dest)