Merge pull request #40 from priyawadhwa/copy-cmd

COPY command

executor/cmd/root.go

@@ -98,7 +98,7 @@ func execute() error {
 	// Currently only supports single stage builds
 	for _, stage := range stages {
 		for _, cmd := range stage.Commands {
-			dockerCommand, err := commands.GetCommand(cmd)
+			dockerCommand, err := commands.GetCommand(cmd, srcContext)
 			if err != nil {
 				return err
 			}

integration_tests/context/arr[0].txt

@@ -0,0 +1 @@
+hello

integration_tests/context/bar/bam/bat

@@ -0,0 +1 @@
+bat

integration_tests/context/bar/bat

@@ -0,0 +1 @@
+bat

integration_tests/context/bar/baz

@@ -0,0 +1 @@
+baz

integration_tests/context/foo

@@ -0,0 +1 @@
+foo

integration_tests/dockerfiles/Dockerfile_test_copy

@@ -0,0 +1,14 @@
+FROM gcr.io/distroless/base
+COPY context/foo foo
+COPY context/foo /foodir/
+COPY context/bar/b* bar/
+COPY context/fo? /foo2
+COPY context/bar/doesnotexist* context/foo hello
+COPY ./context/empty /empty
+COPY ./ dir/
+COPY . newdir
+COPY context/bar /baz/
+COPY ["context/foo", "/tmp/foo" ]
+COPY context/b* /baz/
+COPY context/foo context/bar/ba? /test/
+COPY context/arr[[]0].txt /mydir/

integration_tests/dockerfiles/config_test_copy.json

@@ -0,0 +1,12 @@
+[
+  {
+    "Image1": "gcr.io/kbuild-test/docker-test-copy:latest",
+    "Image2": "gcr.io/kbuild-test/kbuild-test-copy:latest",
+    "DiffType": "File",
+    "Diff": {
+      "Adds": null,
+      "Dels": null,
+      "Mods": null
+    }
+  }
+]

integration_tests/integration_test_yaml.go

@@ -49,6 +49,13 @@ var fileTests = []struct {
 		context:        "integration_tests/dockerfiles/",
 		repo:           "test-run-2",
 	},
+	{
+		description:    "test copy",
+		dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_copy",
+		configPath:     "/workspace/integration_tests/dockerfiles/config_test_copy.json",
+		context:        "/workspace/integration_tests/",
+		repo:           "test-copy",
+	},
 }
 
 var structureTests = []struct {
@@ -126,7 +133,7 @@ func main() {
 		kbuildImage := testRepo + kbuildPrefix + test.repo
 		kbuild := step{
 			Name: executorImage,
-			Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath},
+			Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.context},
 		}
 
 		// Pull the kbuild image

pkg/commands/commands.go

@@ -34,10 +34,12 @@ type DockerCommand interface {
 	FilesToSnapshot() []string
 }
 
-func GetCommand(cmd instructions.Command) (DockerCommand, error) {
+func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, error) {
 	switch c := cmd.(type) {
 	case *instructions.RunCommand:
 		return &RunCommand{cmd: c}, nil
+	case *instructions.CopyCommand:
+		return &CopyCommand{cmd: c, buildcontext: buildcontext}, nil
 	case *instructions.ExposeCommand:
 		return &ExposeCommand{cmd: c}, nil
 	case *instructions.EnvCommand:

pkg/commands/copy.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util"
+	"github.com/containers/image/manifest"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/sirupsen/logrus"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type CopyCommand struct {
+	cmd           *instructions.CopyCommand
+	buildcontext  string
+	snapshotFiles []string
+}
+
+func (c *CopyCommand) ExecuteCommand(config *manifest.Schema2Config) error {
+	srcs := c.cmd.SourcesAndDest[:len(c.cmd.SourcesAndDest)-1]
+	dest := c.cmd.SourcesAndDest[len(c.cmd.SourcesAndDest)-1]
+
+	logrus.Infof("cmd: copy %s", srcs)
+	logrus.Infof("dest: %s", dest)
+
+	// Get a map of [src]:[files rooted at src]
+	srcMap, err := util.ResolveSources(c.cmd.SourcesAndDest, c.buildcontext)
+	if err != nil {
+		return err
+	}
+	// For each source, iterate through each file within and copy it over
+	for src, files := range srcMap {
+		for _, file := range files {
+			fi, err := os.Stat(filepath.Join(c.buildcontext, file))
+			if err != nil {
+				return err
+			}
+			destPath, err := util.DestinationFilepath(file, src, dest, config.WorkingDir, c.buildcontext)
+			if err != nil {
+				return err
+			}
+			// If source file is a directory, we want to create a directory ...
+			if fi.IsDir() {
+				logrus.Infof("Creating directory %s", destPath)
+				if err := os.MkdirAll(destPath, fi.Mode()); err != nil {
+					return err
+				}
+			} else {
+				// ... Else, we want to copy over a file
+				logrus.Infof("Copying file %s to %s", file, destPath)
+				srcFile, err := os.Open(filepath.Join(c.buildcontext, file))
+				if err != nil {
+					return err
+				}
+				defer srcFile.Close()
+				if err := util.CreateFile(destPath, srcFile, fi.Mode()); err != nil {
+					return err
+				}
+			}
+			// Append the destination file to the list of files that should be snapshotted later
+			c.snapshotFiles = append(c.snapshotFiles, destPath)
+		}
+	}
+	return nil
+}
+
+// FilesToSnapshot should return an empty array if still nil; no files were changed
+func (c *CopyCommand) FilesToSnapshot() []string {
+	return c.snapshotFiles
+}
+
+// CreatedBy returns some information about the command for the image config
+func (c *CopyCommand) CreatedBy() string {
+	return strings.Join(c.cmd.SourcesAndDest, " ")
+}

pkg/util/command_util.go

@@ -0,0 +1,163 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// ContainsWildcards returns true if any entry in paths contains wildcards
+func ContainsWildcards(paths []string) bool {
+	for _, path := range paths {
+		if strings.ContainsAny(path, "*?[") {
+			return true
+		}
+	}
+	return false
+}
+
+// ResolveSources resolves the given sources if the sources contains wildcards
+// It returns a map of [src]:[files rooted at src]
+func ResolveSources(srcsAndDest instructions.SourcesAndDest, root string) (map[string][]string, error) {
+	srcs := srcsAndDest[:len(srcsAndDest)-1]
+	// If sources contain wildcards, we first need to resolve them to actual paths
+	if ContainsWildcards(srcs) {
+		logrus.Debugf("Resolving srcs %v...", srcs)
+		files, err := RelativeFiles("", root)
+		if err != nil {
+			return nil, err
+		}
+		srcs, err = matchSources(srcs, files)
+		if err != nil {
+			return nil, err
+		}
+		logrus.Debugf("Resolved sources to %v", srcs)
+	}
+	// Now, get a map of [src]:[files rooted at src]
+	srcMap, err := SourcesToFilesMap(srcs, root)
+	if err != nil {
+		return nil, err
+	}
+	// Check to make sure the sources are valid
+	return srcMap, IsSrcsValid(srcsAndDest, srcMap)
+}
+
+// matchSources returns a list of sources that match wildcards
+func matchSources(srcs, files []string) ([]string, error) {
+	var matchedSources []string
+	for _, src := range srcs {
+		src = filepath.Clean(src)
+		for _, file := range files {
+			matched, err := filepath.Match(src, file)
+			if err != nil {
+				return nil, err
+			}
+			if matched {
+				matchedSources = append(matchedSources, file)
+			}
+		}
+	}
+	return matchedSources, nil
+}
+
+func IsDestDir(path string) bool {
+	return strings.HasSuffix(path, "/") || path == "."
+}
+
+// DestinationFilepath returns the destination filepath from the build context to the image filesystem
+// If source is a file:
+//	If dest is a dir, copy it to /dest/relpath
+// 	If dest is a file, copy directly to dest
+// If source is a dir:
+//	Assume dest is also a dir, and copy to dest/relpath
+// If dest is not an absolute filepath, add /cwd to the beginning
+func DestinationFilepath(filename, srcName, dest, cwd, buildcontext string) (string, error) {
+	fi, err := os.Stat(filepath.Join(buildcontext, filename))
+	if err != nil {
+		return "", err
+	}
+	src, err := os.Stat(filepath.Join(buildcontext, srcName))
+	if err != nil {
+		return "", err
+	}
+	if src.IsDir() || IsDestDir(dest) {
+		relPath, err := filepath.Rel(srcName, filename)
+		if err != nil {
+			return "", err
+		}
+		if relPath == "." && !fi.IsDir() {
+			relPath = filepath.Base(filename)
+		}
+		destPath := filepath.Join(dest, relPath)
+		if filepath.IsAbs(dest) {
+			return destPath, nil
+		}
+		return filepath.Join(cwd, destPath), nil
+	}
+	if filepath.IsAbs(dest) {
+		return dest, nil
+	}
+	return filepath.Join(cwd, dest), nil
+}
+
+// SourcesToFilesMap returns a map of [src]:[files rooted at source]
+func SourcesToFilesMap(srcs []string, root string) (map[string][]string, error) {
+	srcMap := make(map[string][]string)
+	for _, src := range srcs {
+		src = filepath.Clean(src)
+		files, err := RelativeFiles(src, root)
+		if err != nil {
+			return nil, err
+		}
+		srcMap[src] = files
+	}
+	return srcMap, nil
+}
+
+// IsSrcsValid returns an error if the sources provided are invalid, or nil otherwise
+func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, srcMap map[string][]string) error {
+	srcs := srcsAndDest[:len(srcsAndDest)-1]
+	dest := srcsAndDest[len(srcsAndDest)-1]
+
+	totalFiles := 0
+	for _, files := range srcMap {
+		totalFiles += len(files)
+	}
+	if totalFiles == 0 {
+		return errors.New("copy failed: no source files specified")
+	}
+
+	if !ContainsWildcards(srcs) {
+		// If multiple sources and destination isn't a directory, return an error
+		if len(srcs) > 1 && !IsDestDir(dest) {
+			return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'")
+		}
+		return nil
+	}
+
+	// If there are wildcards, and the destination is a file, there must be exactly one file to copy over,
+	// Otherwise, return an error
+	if !IsDestDir(dest) && totalFiles > 1 {
+		return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'")
+	}
+	return nil
+}

pkg/util/command_util_test.go

@@ -0,0 +1,323 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/testutil"
+	"sort"
+	"testing"
+)
+
+var buildContextPath = "../../integration_tests/"
+
+var destinationFilepathTests = []struct {
+	srcName          string
+	filename         string
+	dest             string
+	cwd              string
+	buildcontext     string
+	expectedFilepath string
+}{
+	{
+		srcName:          "context/foo",
+		filename:         "context/foo",
+		dest:             "/foo",
+		cwd:              "/",
+		expectedFilepath: "/foo",
+	},
+	{
+		srcName:          "context/foo",
+		filename:         "context/foo",
+		dest:             "/foodir/",
+		cwd:              "/",
+		expectedFilepath: "/foodir/foo",
+	},
+	{
+		srcName:          "context/foo",
+		filename:         "./context/foo",
+		cwd:              "/",
+		dest:             "foo",
+		expectedFilepath: "/foo",
+	},
+	{
+		srcName:          "context/bar/",
+		filename:         "context/bar/bam/bat",
+		cwd:              "/",
+		dest:             "pkg/",
+		expectedFilepath: "/pkg/bam/bat",
+	},
+	{
+		srcName:          "context/bar/",
+		filename:         "context/bar/bam/bat",
+		cwd:              "/newdir",
+		dest:             "pkg/",
+		expectedFilepath: "/newdir/pkg/bam/bat",
+	},
+	{
+		srcName:          "./context/empty",
+		filename:         "context/empty",
+		cwd:              "/",
+		dest:             "/empty",
+		expectedFilepath: "/empty",
+	},
+	{
+		srcName:          "./context/empty",
+		filename:         "context/empty",
+		cwd:              "/dir",
+		dest:             "/empty",
+		expectedFilepath: "/empty",
+	},
+	{
+		srcName:          "./",
+		filename:         "./",
+		cwd:              "/",
+		dest:             "/dir",
+		expectedFilepath: "/dir",
+	},
+	{
+		srcName:          "./",
+		filename:         "context/foo",
+		cwd:              "/",
+		dest:             "/dir",
+		expectedFilepath: "/dir/context/foo",
+	},
+	{
+		srcName:          ".",
+		filename:         "context/bar",
+		cwd:              "/",
+		dest:             "/dir",
+		expectedFilepath: "/dir/context/bar",
+	},
+	{
+		srcName:          ".",
+		filename:         "context/bar",
+		cwd:              "/",
+		dest:             "/dir",
+		expectedFilepath: "/dir/context/bar",
+	},
+	{
+		srcName:          "context/foo",
+		filename:         "context/foo",
+		cwd:              "/test",
+		dest:             ".",
+		expectedFilepath: "/test/foo",
+	},
+}
+
+func Test_DestinationFilepath(t *testing.T) {
+	for _, test := range destinationFilepathTests {
+		actualFilepath, err := DestinationFilepath(test.filename, test.srcName, test.dest, test.cwd, buildContextPath)
+		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFilepath, actualFilepath)
+	}
+}
+
+var matchSourcesTests = []struct {
+	srcs          []string
+	files         []string
+	expectedFiles []string
+}{
+	{
+		srcs: []string{
+			"pkg/*",
+		},
+		files: []string{
+			"pkg/a",
+			"pkg/b",
+			"/pkg/d",
+			"pkg/b/d/",
+			"dir/",
+		},
+		expectedFiles: []string{
+			"pkg/a",
+			"pkg/b",
+		},
+	},
+}
+
+func Test_MatchSources(t *testing.T) {
+	for _, test := range matchSourcesTests {
+		actualFiles, err := matchSources(test.srcs, test.files)
+		sort.Strings(actualFiles)
+		sort.Strings(test.expectedFiles)
+		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFiles, actualFiles)
+	}
+}
+
+var isSrcValidTests = []struct {
+	srcsAndDest []string
+	files       map[string][]string
+	shouldErr   bool
+}{
+	{
+		srcsAndDest: []string{
+			"src1",
+			"src2",
+			"dest",
+		},
+		files: map[string][]string{
+			"src1": {
+				"file1",
+			},
+			"src2:": {
+				"file2",
+			},
+		},
+		shouldErr: true,
+	},
+	{
+		srcsAndDest: []string{
+			"src1",
+			"src2",
+			"dest/",
+		},
+		files: map[string][]string{
+			"src1": {
+				"file1",
+			},
+			"src2:": {
+				"file2",
+			},
+		},
+		shouldErr: false,
+	},
+	{
+		srcsAndDest: []string{
+			"src2/",
+			"dest",
+		},
+		files: map[string][]string{
+			"src1": {
+				"file1",
+			},
+			"src2:": {
+				"file2",
+			},
+		},
+		shouldErr: false,
+	},
+	{
+		srcsAndDest: []string{
+			"src2",
+			"dest",
+		},
+		files: map[string][]string{
+			"src1": {
+				"file1",
+			},
+			"src2:": {
+				"file2",
+			},
+		},
+		shouldErr: false,
+	},
+	{
+		srcsAndDest: []string{
+			"src2",
+			"src*",
+			"dest/",
+		},
+		files: map[string][]string{
+			"src1": {
+				"file1",
+			},
+			"src2:": {
+				"file2",
+			},
+		},
+		shouldErr: false,
+	},
+	{
+		srcsAndDest: []string{
+			"src2",
+			"src*",
+			"dest",
+		},
+		files: map[string][]string{
+			"src2": {
+				"src2/a",
+				"src2/b",
+			},
+			"src*": {},
+		},
+		shouldErr: true,
+	},
+	{
+		srcsAndDest: []string{
+			"src2",
+			"src*",
+			"dest",
+		},
+		files: map[string][]string{
+			"src2": {
+				"src2/a",
+			},
+			"src*": {},
+		},
+		shouldErr: false,
+	},
+	{
+		srcsAndDest: []string{
+			"src2",
+			"src*",
+			"dest",
+		},
+		files: map[string][]string{
+			"src2": {},
+			"src*": {},
+		},
+		shouldErr: true,
+	},
+}
+
+func Test_IsSrcsValid(t *testing.T) {
+	for _, test := range isSrcValidTests {
+		err := IsSrcsValid(test.srcsAndDest, test.files)
+		testutil.CheckError(t, test.shouldErr, err)
+	}
+}
+
+var testResolveSources = []struct {
+	srcsAndDest []string
+	expectedMap map[string][]string
+}{
+	{
+		srcsAndDest: []string{
+			"context/foo",
+			"context/b*",
+			"dest/",
+		},
+		expectedMap: map[string][]string{
+			"context/foo": {
+				"context/foo",
+			},
+			"context/bar": {
+				"context/bar",
+				"context/bar/bam",
+				"context/bar/bam/bat",
+				"context/bar/bat",
+				"context/bar/baz",
+			},
+		},
+	},
+}
+
+func Test_ResolveSources(t *testing.T) {
+	for _, test := range testResolveSources {
+		actualMap, err := ResolveSources(test.srcsAndDest, buildContextPath)
+		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedMap, actualMap)
+	}
+}

pkg/util/fs_util.go

@@ -97,3 +97,48 @@ func fileSystemWhitelist(path string) ([]string, error) {
 	}
 	return whitelist, nil
 }
+
+// RelativeFiles returns a list of all files at the filepath relative to root
+func RelativeFiles(fp string, root string) ([]string, error) {
+	var files []string
+	fullPath := filepath.Join(root, fp)
+	logrus.Debugf("Getting files and contents at root %s", fullPath)
+	err := filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		relPath, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		files = append(files, relPath)
+		return nil
+	})
+	return files, err
+}
+
+// FilepathExists returns true if the path exists
+func FilepathExists(path string) bool {
+	_, err := os.Stat(path)
+	return !os.IsNotExist(err)
+}
+
+// CreateFile creates a file at path and copies over contents from the reader
+func CreateFile(path string, reader io.Reader, perm os.FileMode) error {
+	// Create directory path if it doesn't exist
+	baseDir := filepath.Dir(path)
+	if _, err := os.Stat(baseDir); os.IsNotExist(err) {
+		logrus.Debugf("baseDir %s for file %s does not exist. Creating.", baseDir, path)
+		if err := os.MkdirAll(baseDir, 0755); err != nil {
+			return err
+		}
+	}
+	dest, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(dest, reader); err != nil {
+		return err
+	}
+	return dest.Chmod(perm)
+}

pkg/util/fs_util_test.go

@@ -51,3 +51,82 @@ func Test_fileSystemWhitelist(t *testing.T) {
 	sort.Strings(expectedWhitelist)
 	testutil.CheckErrorAndDeepEqual(t, false, err, expectedWhitelist, actualWhitelist)
 }
+
+var tests = []struct {
+	files         map[string]string
+	directory     string
+	expectedFiles []string
+}{
+	{
+		files: map[string]string{
+			"/workspace/foo/a": "baz1",
+			"/workspace/foo/b": "baz2",
+			"/kbuild/file":     "file",
+		},
+		directory: "/workspace/foo/",
+		expectedFiles: []string{
+			"workspace/foo/a",
+			"workspace/foo/b",
+			"workspace/foo",
+		},
+	},
+	{
+		files: map[string]string{
+			"/workspace/foo/a": "baz1",
+		},
+		directory: "/workspace/foo/a",
+		expectedFiles: []string{
+			"workspace/foo/a",
+		},
+	},
+	{
+		files: map[string]string{
+			"/workspace/foo/a": "baz1",
+			"/workspace/foo/b": "baz2",
+			"/workspace/baz":   "hey",
+			"/kbuild/file":     "file",
+		},
+		directory: "/workspace",
+		expectedFiles: []string{
+			"workspace/foo/a",
+			"workspace/foo/b",
+			"workspace/baz",
+			"workspace",
+			"workspace/foo",
+		},
+	},
+	{
+		files: map[string]string{
+			"/workspace/foo/a": "baz1",
+			"/workspace/foo/b": "baz2",
+			"/kbuild/file":     "file",
+		},
+		directory: "",
+		expectedFiles: []string{
+			"workspace/foo/a",
+			"workspace/foo/b",
+			"kbuild/file",
+			"workspace",
+			"workspace/foo",
+			"kbuild",
+			".",
+		},
+	},
+}
+
+func Test_RelativeFiles(t *testing.T) {
+	for _, test := range tests {
+		testDir, err := ioutil.TempDir("", "")
+		if err != nil {
+			t.Fatalf("err setting up temp dir: %v", err)
+		}
+		defer os.RemoveAll(testDir)
+		if err := testutil.SetupFiles(testDir, test.files); err != nil {
+			t.Fatalf("err setting up files: %v", err)
+		}
+		actualFiles, err := RelativeFiles(test.directory, testDir)
+		sort.Strings(actualFiles)
+		sort.Strings(test.expectedFiles)
+		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFiles, actualFiles)
+	}
+}

testutil/util.go

@@ -50,6 +50,12 @@ func CheckErrorAndDeepEqual(t *testing.T, shouldErr bool, err error, expected, a
 	}
 }
 
+func CheckError(t *testing.T, shouldErr bool, err error) {
+	if err := checkErr(shouldErr, err); err != nil {
+		t.Error(err)
+	}
+}
+
 func checkErr(shouldErr bool, err error) error {
 	if err == nil && shouldErr {
 		return fmt.Errorf("Expected error, but returned none")