Copy command and unit tests

executor/cmd/root.go

@@ -98,7 +98,7 @@ func execute() error {
 	// Currently only supports single stage builds
 	for _, stage := range stages {
 		for _, cmd := range stage.Commands {
-			dockerCommand, err := commands.GetCommand(cmd)
+			dockerCommand, err := commands.GetCommand(cmd, srcContext)
 			if err != nil {
 				return err
 			}

integration_tests/context/bar/bam/bat

@@ -0,0 +1 @@
+bat

integration_tests/context/bar/bat

@@ -0,0 +1 @@
+bat

integration_tests/context/bar/baz

@@ -0,0 +1 @@
+baz

integration_tests/context/foo

@@ -0,0 +1 @@
+foo

pkg/commands/commands.go

@@ -34,10 +34,12 @@ type DockerCommand interface {
 	FilesToSnapshot() []string
 }
 
-func GetCommand(cmd instructions.Command) (DockerCommand, error) {
+func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, error) {
 	switch c := cmd.(type) {
 	case *instructions.RunCommand:
 		return &RunCommand{cmd: c}, nil
+	case *instructions.CopyCommand:
+		return &CopyCommand{cmd: c, buildcontext: buildcontext}, nil
 	}
 	return nil, errors.Errorf("%s is not a supported command", cmd.Name())
 }

pkg/commands/copy.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util"
+	"github.com/containers/image/manifest"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/sirupsen/logrus"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type CopyCommand struct {
+	cmd           *instructions.CopyCommand
+	buildcontext  string
+	snapshotFiles []string
+}
+
+func (c *CopyCommand) ExecuteCommand(config *manifest.Schema2Config) error {
+	srcs := c.cmd.SourcesAndDest[:len(c.cmd.SourcesAndDest)-1]
+	dest := c.cmd.SourcesAndDest[len(c.cmd.SourcesAndDest)-1]
+
+	logrus.Infof("cmd: copy %s", srcs)
+	logrus.Infof("dest: %s", dest)
+
+	// Get a map of [src]:[files rooted at src]
+	srcMap, err := util.ResolveSources(c.cmd.SourcesAndDest, c.buildcontext, config.WorkingDir)
+	if err != nil {
+		return err
+	}
+	// For each source, iterate through each file within and copy it over
+	for src, files := range srcMap {
+		for _, file := range files {
+			fi, err := os.Stat(filepath.Join(c.buildcontext, file))
+			if err != nil {
+				return err
+			}
+			destPath, err := util.RelativeFilepath(file, src, dest, config.WorkingDir, c.buildcontext)
+			if err != nil {
+				return err
+			}
+			// If source file is a directory, we want to create a directory ...
+			if fi.IsDir() {
+				logrus.Infof("Creating directory %s", destPath)
+				if err := os.MkdirAll(destPath, fi.Mode()); err != nil {
+					return err
+				}
+			} else {
+				// ... Else, we want to copy over a file
+				logrus.Infof("Copying file %s to %s", file, destPath)
+				contents, err := ioutil.ReadFile(filepath.Join(c.buildcontext, file))
+				if err != nil {
+					return err
+				}
+				if err := util.CreateFile(destPath, contents, fi.Mode()); err != nil {
+					return err
+				}
+			}
+			// Append the destination file to the list of files that should be snapshotted later
+			c.snapshotFiles = append(c.snapshotFiles, destPath)
+		}
+	}
+	return nil
+}
+
+// FilesToSnapshot should return an empty array if still nil; no files were changed
+func (c *CopyCommand) FilesToSnapshot() []string {
+	if c.snapshotFiles == nil {
+		return []string{}
+	}
+	return c.snapshotFiles
+}
+
+// Author returns some information about the command for the image config
+func (c *CopyCommand) CreatedBy() string {
+	return strings.Join(c.cmd.SourcesAndDest, " ")
+}

pkg/util/command_util.go

@@ -0,0 +1,165 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/pkg/errors"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// ContainsWildcards returns true if any entry in paths contains wildcards
+func ContainsWildcards(paths []string) bool {
+	for _, path := range paths {
+		for i := 0; i < len(path); i++ {
+			ch := path[i]
+			// These are the wildcards that correspond to filepath.Match
+			if ch == '*' || ch == '?' || ch == '[' {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// ResolveSources resolves the given sources if the sources contains wildcard
+// It returns a map of [src]:[files rooted at src]
+func ResolveSources(srcsAndDest instructions.SourcesAndDest, root, cwd string) (map[string][]string, error) {
+	srcs := srcsAndDest[:len(srcsAndDest)-1]
+	// If sources contain wildcards, we first need to resolve them to actual paths
+	wildcard := ContainsWildcards(srcs)
+	if wildcard {
+		files, err := Files("", root)
+		if err != nil {
+			return nil, err
+		}
+		srcs, err = matchSources(srcs, files, cwd)
+		if err != nil {
+			return nil, err
+		}
+	}
+	// Now, get a map of [src]:[files rooted at src]
+	srcMap, err := SourcesToFilesMap(srcs, root)
+	if err != nil {
+		return nil, err
+	}
+	// Check to make sure the sources are valid
+	return srcMap, IsSrcsValid(srcsAndDest, srcMap)
+}
+
+// matchSources returns a map of [src]:[matching filepaths], used to resolve wildcards
+func matchSources(srcs, files []string, cwd string) ([]string, error) {
+	var matchedSources []string
+	for _, src := range srcs {
+		src = filepath.Clean(src)
+		for _, file := range files {
+			matched, err := filepath.Match(src, file)
+			if err != nil {
+				return nil, err
+			}
+			// Check cwd
+			matchedRoot, err := filepath.Match(filepath.Join(cwd, src), file)
+			if err != nil {
+				return nil, err
+			}
+			if !(matched || matchedRoot) {
+				continue
+			}
+			matchedSources = append(matchedSources, file)
+		}
+	}
+	return matchedSources, nil
+}
+
+func IsDestDir(path string) bool {
+	return strings.HasSuffix(path, "/")
+}
+
+// RelativeFilepath returns the relative filepath
+// If source is a file:
+//	If dest is a dir, copy it to /cwd/dest/relpath
+// 	If dest is a file, copy directly to /cwd/dest
+
+// If source is a dir:
+//	Assume dest is also a dir, and copy to /cwd/dest/relpath
+func RelativeFilepath(filename, srcName, dest, cwd, buildcontext string) (string, error) {
+	fi, err := os.Stat(filepath.Join(buildcontext, filename))
+	if err != nil {
+		return "", err
+	}
+	src, err := os.Stat(filepath.Join(buildcontext, srcName))
+	if err != nil {
+		return "", err
+	}
+	if src.IsDir() || IsDestDir(dest) {
+		relPath, err := filepath.Rel(srcName, filename)
+		if err != nil {
+			return "", err
+		}
+		if relPath == "." && !fi.IsDir() {
+			relPath = filepath.Base(filename)
+		}
+		destPath := filepath.Join(cwd, dest, relPath)
+		return destPath, nil
+	}
+	return filepath.Join(cwd, dest), nil
+}
+
+// SourcesToFilesMap returns a map of [src]:[files rooted at source]
+func SourcesToFilesMap(srcs []string, root string) (map[string][]string, error) {
+	srcMap := make(map[string][]string)
+	for _, src := range srcs {
+		src = filepath.Clean(src)
+		files, err := Files(src, root)
+		if err != nil {
+			return nil, err
+		}
+		srcMap[src] = files
+	}
+	return srcMap, nil
+}
+
+// IsSrcsValid returns an error if the sources provided are invalid, or nil otherwise
+func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, srcMap map[string][]string) error {
+	srcs := srcsAndDest[:len(srcsAndDest)-1]
+	dest := srcsAndDest[len(srcsAndDest)-1]
+	// If destination is a directory, return nil
+	if IsDestDir(dest) {
+		return nil
+	}
+	// If no wildcards and multiple sources, return error
+	if !ContainsWildcards(srcs) {
+		if len(srcs) > 1 {
+			return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'")
+		}
+		return nil
+	}
+	// If there are wildcards, and the destination is a file, there must be exactly one file to copy over
+	totalFiles := 0
+	for _, files := range srcMap {
+		totalFiles += len(files)
+	}
+	if totalFiles == 0 {
+		return errors.New("copy failed: no source files specified")
+	}
+	if totalFiles > 1 {
+		return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'")
+	}
+	return nil
+}

pkg/util/command_util_test.go

@@ -0,0 +1,286 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/testutil"
+	"sort"
+	"testing"
+)
+
+var buildContextPath = "../../integration_tests/"
+
+var relativeFilepathTests = []struct {
+	srcName          string
+	filename         string
+	dest             string
+	cwd              string
+	buildcontext     string
+	expectedFilepath string
+}{
+	{
+		srcName:          "context/foo",
+		filename:         "context/foo",
+		dest:             "/foo",
+		cwd:              "/",
+		expectedFilepath: "/foo",
+	},
+	{
+		srcName:          "context/foo",
+		filename:         "context/foo",
+		dest:             "/foodir/",
+		cwd:              "/",
+		expectedFilepath: "/foodir/foo",
+	},
+	{
+		srcName:          "context/foo",
+		filename:         "./context/foo",
+		cwd:              "/",
+		dest:             "foo",
+		expectedFilepath: "/foo",
+	},
+	{
+		srcName:          "context/bar/",
+		filename:         "context/bar/bam/bat",
+		cwd:              "/",
+		dest:             "pkg/",
+		expectedFilepath: "/pkg/bam/bat",
+	},
+	{
+		srcName:          "context/bar/",
+		filename:         "context/bar/bam/bat",
+		cwd:              "/newdir",
+		dest:             "pkg/",
+		expectedFilepath: "/newdir/pkg/bam/bat",
+	},
+	{
+		srcName:          "./context/empty",
+		filename:         "context/empty",
+		cwd:              "/",
+		dest:             "/empty",
+		expectedFilepath: "/empty",
+	},
+	{
+		srcName:          "./context/empty",
+		filename:         "context/empty",
+		cwd:              "/dir",
+		dest:             "/empty",
+		expectedFilepath: "/dir/empty",
+	},
+	{
+		srcName:          "./",
+		filename:         "./",
+		cwd:              "/",
+		dest:             "/dir",
+		expectedFilepath: "/dir",
+	},
+	{
+		srcName:          "./",
+		filename:         "context/foo",
+		cwd:              "/",
+		dest:             "/dir",
+		expectedFilepath: "/dir/context/foo",
+	},
+	{
+		srcName:          ".",
+		filename:         "context/bar",
+		cwd:              "/",
+		dest:             "/dir",
+		expectedFilepath: "/dir/context/bar",
+	},
+	{
+		srcName:          ".",
+		filename:         "context/bar",
+		cwd:              "/",
+		dest:             "/dir",
+		expectedFilepath: "/dir/context/bar",
+	},
+}
+
+func Test_RelativeFilepath(t *testing.T) {
+	for _, test := range relativeFilepathTests {
+		actualFilepath, err := RelativeFilepath(test.filename, test.srcName, test.dest, test.cwd, buildContextPath)
+		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFilepath, actualFilepath)
+	}
+}
+
+var matchSourcesTests = []struct {
+	srcs          []string
+	files         []string
+	cwd           string
+	expectedFiles []string
+}{
+	{
+		srcs: []string{
+			"pkg/*",
+		},
+		files: []string{
+			"pkg/a",
+			"pkg/b",
+			"/pkg/d",
+			"pkg/b/d/",
+			"dir/",
+		},
+		cwd: "/",
+		expectedFiles: []string{
+			"pkg/a",
+			"pkg/b",
+			"/pkg/d",
+		},
+	},
+}
+
+func Test_MatchSources(t *testing.T) {
+	for _, test := range matchSourcesTests {
+		actualFiles, err := matchSources(test.srcs, test.files, test.cwd)
+		sort.Strings(actualFiles)
+		sort.Strings(test.expectedFiles)
+		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFiles, actualFiles)
+	}
+}
+
+var isSrcValidTests = []struct {
+	srcsAndDest []string
+	files       map[string][]string
+	shouldErr   bool
+}{
+	{
+		srcsAndDest: []string{
+			"src1",
+			"src2",
+			"dest",
+		},
+		files:     nil,
+		shouldErr: true,
+	},
+	{
+		srcsAndDest: []string{
+			"src1",
+			"src2",
+			"dest/",
+		},
+		files:     nil,
+		shouldErr: false,
+	},
+	{
+		srcsAndDest: []string{
+			"src2/",
+			"dest",
+		},
+		files:     nil,
+		shouldErr: false,
+	},
+	{
+		srcsAndDest: []string{
+			"src2",
+			"dest",
+		},
+		files:     nil,
+		shouldErr: false,
+	},
+	{
+		srcsAndDest: []string{
+			"src2",
+			"src*",
+			"dest/",
+		},
+		files:     nil,
+		shouldErr: false,
+	},
+	{
+		srcsAndDest: []string{
+			"src2",
+			"src*",
+			"dest",
+		},
+		files: map[string][]string{
+			"src2": {
+				"src2/a",
+				"src2/b",
+			},
+			"src*": {},
+		},
+		shouldErr: true,
+	},
+	{
+		srcsAndDest: []string{
+			"src2",
+			"src*",
+			"dest",
+		},
+		files: map[string][]string{
+			"src2": {
+				"src2/a",
+			},
+			"src*": {},
+		},
+		shouldErr: false,
+	},
+	{
+		srcsAndDest: []string{
+			"src2",
+			"src*",
+			"dest",
+		},
+		files: map[string][]string{
+			"src2": {},
+			"src*": {},
+		},
+		shouldErr: true,
+	},
+}
+
+func Test_IsSrcsValid(t *testing.T) {
+	for _, test := range isSrcValidTests {
+		err := IsSrcsValid(test.srcsAndDest, test.files)
+		testutil.CheckError(t, test.shouldErr, err)
+	}
+}
+
+var testResolveSources = []struct {
+	srcsAndDest []string
+	cwd         string
+	expectedMap map[string][]string
+}{
+	{
+		srcsAndDest: []string{
+			"context/foo",
+			"context/b*",
+			"dest/",
+		},
+		cwd: "/",
+		expectedMap: map[string][]string{
+			"context/foo": {
+				"context/foo",
+			},
+			"context/bar": {
+				"context/bar",
+				"context/bar/bam",
+				"context/bar/bam/bat",
+				"context/bar/bat",
+				"context/bar/baz",
+			},
+		},
+	},
+}
+
+func Test_ResolveSources(t *testing.T) {
+	for _, test := range testResolveSources {
+		actualMap, err := ResolveSources(test.srcsAndDest, buildContextPath, test.cwd)
+		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedMap, actualMap)
+	}
+}

pkg/util/fs_util.go

@@ -97,3 +97,48 @@ func fileSystemWhitelist(path string) ([]string, error) {
 	}
 	return whitelist, nil
 }
+
+// Files returns a list of all files at the filepath relative to root
+func Files(fp string, root string) ([]string, error) {
+	var files []string
+	fullPath := filepath.Join(root, fp)
+	logrus.Debugf("Getting files and contents at root %s", fullPath)
+	err := filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		relPath, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		files = append(files, relPath)
+		return err
+	})
+	return files, err
+}
+
+// FilepathExists returns true if the path exists
+func FilepathExists(path string) bool {
+	_, err := os.Stat(path)
+	return (err == nil)
+}
+
+// CreateFile creates a file at path with contents specified
+func CreateFile(path string, contents []byte, perm os.FileMode) error {
+	// Create directory path if it doesn't exist
+	baseDir := filepath.Dir(path)
+	if _, err := os.Stat(baseDir); os.IsNotExist(err) {
+		logrus.Debugf("baseDir %s for file %s does not exist. Creating.", baseDir, path)
+		if err := os.MkdirAll(baseDir, perm); err != nil {
+			return err
+		}
+	}
+
+	f, err := os.Create(path)
+	defer f.Close()
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(contents)
+	return err
+}

pkg/util/fs_util_test.go

@@ -51,3 +51,82 @@ func Test_fileSystemWhitelist(t *testing.T) {
 	sort.Strings(expectedWhitelist)
 	testutil.CheckErrorAndDeepEqual(t, false, err, expectedWhitelist, actualWhitelist)
 }
+
+var tests = []struct {
+	files         map[string]string
+	directory     string
+	expectedFiles []string
+}{
+	{
+		files: map[string]string{
+			"/workspace/foo/a": "baz1",
+			"/workspace/foo/b": "baz2",
+			"/kbuild/file":     "file",
+		},
+		directory: "/workspace/foo/",
+		expectedFiles: []string{
+			"workspace/foo/a",
+			"workspace/foo/b",
+			"workspace/foo",
+		},
+	},
+	{
+		files: map[string]string{
+			"/workspace/foo/a": "baz1",
+		},
+		directory: "/workspace/foo/a",
+		expectedFiles: []string{
+			"workspace/foo/a",
+		},
+	},
+	{
+		files: map[string]string{
+			"/workspace/foo/a": "baz1",
+			"/workspace/foo/b": "baz2",
+			"/workspace/baz":   "hey",
+			"/kbuild/file":     "file",
+		},
+		directory: "/workspace",
+		expectedFiles: []string{
+			"workspace/foo/a",
+			"workspace/foo/b",
+			"workspace/baz",
+			"workspace",
+			"workspace/foo",
+		},
+	},
+	{
+		files: map[string]string{
+			"/workspace/foo/a": "baz1",
+			"/workspace/foo/b": "baz2",
+			"/kbuild/file":     "file",
+		},
+		directory: "",
+		expectedFiles: []string{
+			"workspace/foo/a",
+			"workspace/foo/b",
+			"kbuild/file",
+			"workspace",
+			"workspace/foo",
+			"kbuild",
+			".",
+		},
+	},
+}
+
+func Test_Files(t *testing.T) {
+	for _, test := range tests {
+		testDir, err := ioutil.TempDir("", "")
+		if err != nil {
+			t.Fatalf("err setting up temp dir: %v", err)
+		}
+		defer os.RemoveAll(testDir)
+		if err := testutil.SetupFiles(testDir, test.files); err != nil {
+			t.Fatalf("err setting up files: %v", err)
+		}
+		actualFiles, err := Files(test.directory, testDir)
+		sort.Strings(actualFiles)
+		sort.Strings(test.expectedFiles)
+		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFiles, actualFiles)
+	}
+}

testutil/util.go

@@ -50,6 +50,12 @@ func CheckErrorAndDeepEqual(t *testing.T, shouldErr bool, err error, expected, a
 	}
 }
 
+func CheckError(t *testing.T, shouldErr bool, err error) {
+	if err := checkErr(shouldErr, err); err != nil {
+		t.Error(err)
+	}
+}
+
 func checkErr(shouldErr bool, err error) error {
 	if err == nil && shouldErr {
 		return fmt.Errorf("Expected error, but returned none")