diff --git a/executor/cmd/root.go b/executor/cmd/root.go index a34a5f3cd..80b2c088f 100644 --- a/executor/cmd/root.go +++ b/executor/cmd/root.go @@ -98,7 +98,7 @@ func execute() error { // Currently only supports single stage builds for _, stage := range stages { for _, cmd := range stage.Commands { - dockerCommand, err := commands.GetCommand(cmd) + dockerCommand, err := commands.GetCommand(cmd, srcContext) if err != nil { return err } diff --git a/integration_tests/context/bar/bam/bat b/integration_tests/context/bar/bam/bat new file mode 100644 index 000000000..1054901d8 --- /dev/null +++ b/integration_tests/context/bar/bam/bat @@ -0,0 +1 @@ +bat diff --git a/integration_tests/context/bar/bat b/integration_tests/context/bar/bat new file mode 100644 index 000000000..1054901d8 --- /dev/null +++ b/integration_tests/context/bar/bat @@ -0,0 +1 @@ +bat diff --git a/integration_tests/context/bar/baz b/integration_tests/context/bar/baz new file mode 100644 index 000000000..76018072e --- /dev/null +++ b/integration_tests/context/bar/baz @@ -0,0 +1 @@ +baz diff --git a/integration_tests/context/foo b/integration_tests/context/foo new file mode 100644 index 000000000..257cc5642 --- /dev/null +++ b/integration_tests/context/foo @@ -0,0 +1 @@ +foo diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index eb2cab5bc..1837f803f 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -34,10 +34,12 @@ type DockerCommand interface { FilesToSnapshot() []string } -func GetCommand(cmd instructions.Command) (DockerCommand, error) { +func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, error) { switch c := cmd.(type) { case *instructions.RunCommand: return &RunCommand{cmd: c}, nil + case *instructions.CopyCommand: + return &CopyCommand{cmd: c, buildcontext: buildcontext}, nil } return nil, errors.Errorf("%s is not a supported command", cmd.Name()) } diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go new file mode 100644 index 000000000..9b0ffaf56 --- /dev/null +++ b/pkg/commands/copy.go @@ -0,0 +1,94 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util" + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/sirupsen/logrus" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +type CopyCommand struct { + cmd *instructions.CopyCommand + buildcontext string + snapshotFiles []string +} + +func (c *CopyCommand) ExecuteCommand(config *manifest.Schema2Config) error { + srcs := c.cmd.SourcesAndDest[:len(c.cmd.SourcesAndDest)-1] + dest := c.cmd.SourcesAndDest[len(c.cmd.SourcesAndDest)-1] + + logrus.Infof("cmd: copy %s", srcs) + logrus.Infof("dest: %s", dest) + + // Get a map of [src]:[files rooted at src] + srcMap, err := util.ResolveSources(c.cmd.SourcesAndDest, c.buildcontext, config.WorkingDir) + if err != nil { + return err + } + // For each source, iterate through each file within and copy it over + for src, files := range srcMap { + for _, file := range files { + fi, err := os.Stat(filepath.Join(c.buildcontext, file)) + if err != nil { + return err + } + destPath, err := util.RelativeFilepath(file, src, dest, config.WorkingDir, c.buildcontext) + if err != nil { + return err + } + // If source file is a directory, we want to create a directory ... + if fi.IsDir() { + logrus.Infof("Creating directory %s", destPath) + if err := os.MkdirAll(destPath, fi.Mode()); err != nil { + return err + } + } else { + // ... Else, we want to copy over a file + logrus.Infof("Copying file %s to %s", file, destPath) + contents, err := ioutil.ReadFile(filepath.Join(c.buildcontext, file)) + if err != nil { + return err + } + if err := util.CreateFile(destPath, contents, fi.Mode()); err != nil { + return err + } + } + // Append the destination file to the list of files that should be snapshotted later + c.snapshotFiles = append(c.snapshotFiles, destPath) + } + } + return nil +} + +// FilesToSnapshot should return an empty array if still nil; no files were changed +func (c *CopyCommand) FilesToSnapshot() []string { + if c.snapshotFiles == nil { + return []string{} + } + return c.snapshotFiles +} + +// Author returns some information about the command for the image config +func (c *CopyCommand) CreatedBy() string { + return strings.Join(c.cmd.SourcesAndDest, " ") +} diff --git a/pkg/util/command_util.go b/pkg/util/command_util.go new file mode 100644 index 000000000..86321c5bd --- /dev/null +++ b/pkg/util/command_util.go @@ -0,0 +1,165 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/pkg/errors" + "os" + "path/filepath" + "strings" +) + +// ContainsWildcards returns true if any entry in paths contains wildcards +func ContainsWildcards(paths []string) bool { + for _, path := range paths { + for i := 0; i < len(path); i++ { + ch := path[i] + // These are the wildcards that correspond to filepath.Match + if ch == '*' || ch == '?' || ch == '[' { + return true + } + } + } + return false +} + +// ResolveSources resolves the given sources if the sources contains wildcard +// It returns a map of [src]:[files rooted at src] +func ResolveSources(srcsAndDest instructions.SourcesAndDest, root, cwd string) (map[string][]string, error) { + srcs := srcsAndDest[:len(srcsAndDest)-1] + // If sources contain wildcards, we first need to resolve them to actual paths + wildcard := ContainsWildcards(srcs) + if wildcard { + files, err := Files("", root) + if err != nil { + return nil, err + } + srcs, err = matchSources(srcs, files, cwd) + if err != nil { + return nil, err + } + } + // Now, get a map of [src]:[files rooted at src] + srcMap, err := SourcesToFilesMap(srcs, root) + if err != nil { + return nil, err + } + // Check to make sure the sources are valid + return srcMap, IsSrcsValid(srcsAndDest, srcMap) +} + +// matchSources returns a map of [src]:[matching filepaths], used to resolve wildcards +func matchSources(srcs, files []string, cwd string) ([]string, error) { + var matchedSources []string + for _, src := range srcs { + src = filepath.Clean(src) + for _, file := range files { + matched, err := filepath.Match(src, file) + if err != nil { + return nil, err + } + // Check cwd + matchedRoot, err := filepath.Match(filepath.Join(cwd, src), file) + if err != nil { + return nil, err + } + if !(matched || matchedRoot) { + continue + } + matchedSources = append(matchedSources, file) + } + } + return matchedSources, nil +} + +func IsDestDir(path string) bool { + return strings.HasSuffix(path, "/") +} + +// RelativeFilepath returns the relative filepath +// If source is a file: +// If dest is a dir, copy it to /cwd/dest/relpath +// If dest is a file, copy directly to /cwd/dest + +// If source is a dir: +// Assume dest is also a dir, and copy to /cwd/dest/relpath +func RelativeFilepath(filename, srcName, dest, cwd, buildcontext string) (string, error) { + fi, err := os.Stat(filepath.Join(buildcontext, filename)) + if err != nil { + return "", err + } + src, err := os.Stat(filepath.Join(buildcontext, srcName)) + if err != nil { + return "", err + } + if src.IsDir() || IsDestDir(dest) { + relPath, err := filepath.Rel(srcName, filename) + if err != nil { + return "", err + } + if relPath == "." && !fi.IsDir() { + relPath = filepath.Base(filename) + } + destPath := filepath.Join(cwd, dest, relPath) + return destPath, nil + } + return filepath.Join(cwd, dest), nil +} + +// SourcesToFilesMap returns a map of [src]:[files rooted at source] +func SourcesToFilesMap(srcs []string, root string) (map[string][]string, error) { + srcMap := make(map[string][]string) + for _, src := range srcs { + src = filepath.Clean(src) + files, err := Files(src, root) + if err != nil { + return nil, err + } + srcMap[src] = files + } + return srcMap, nil +} + +// IsSrcsValid returns an error if the sources provided are invalid, or nil otherwise +func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, srcMap map[string][]string) error { + srcs := srcsAndDest[:len(srcsAndDest)-1] + dest := srcsAndDest[len(srcsAndDest)-1] + // If destination is a directory, return nil + if IsDestDir(dest) { + return nil + } + // If no wildcards and multiple sources, return error + if !ContainsWildcards(srcs) { + if len(srcs) > 1 { + return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'") + } + return nil + } + // If there are wildcards, and the destination is a file, there must be exactly one file to copy over + totalFiles := 0 + for _, files := range srcMap { + totalFiles += len(files) + } + if totalFiles == 0 { + return errors.New("copy failed: no source files specified") + } + if totalFiles > 1 { + return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'") + } + return nil +} diff --git a/pkg/util/command_util_test.go b/pkg/util/command_util_test.go new file mode 100644 index 000000000..ed89d182f --- /dev/null +++ b/pkg/util/command_util_test.go @@ -0,0 +1,286 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/testutil" + "sort" + "testing" +) + +var buildContextPath = "../../integration_tests/" + +var relativeFilepathTests = []struct { + srcName string + filename string + dest string + cwd string + buildcontext string + expectedFilepath string +}{ + { + srcName: "context/foo", + filename: "context/foo", + dest: "/foo", + cwd: "/", + expectedFilepath: "/foo", + }, + { + srcName: "context/foo", + filename: "context/foo", + dest: "/foodir/", + cwd: "/", + expectedFilepath: "/foodir/foo", + }, + { + srcName: "context/foo", + filename: "./context/foo", + cwd: "/", + dest: "foo", + expectedFilepath: "/foo", + }, + { + srcName: "context/bar/", + filename: "context/bar/bam/bat", + cwd: "/", + dest: "pkg/", + expectedFilepath: "/pkg/bam/bat", + }, + { + srcName: "context/bar/", + filename: "context/bar/bam/bat", + cwd: "/newdir", + dest: "pkg/", + expectedFilepath: "/newdir/pkg/bam/bat", + }, + { + srcName: "./context/empty", + filename: "context/empty", + cwd: "/", + dest: "/empty", + expectedFilepath: "/empty", + }, + { + srcName: "./context/empty", + filename: "context/empty", + cwd: "/dir", + dest: "/empty", + expectedFilepath: "/dir/empty", + }, + { + srcName: "./", + filename: "./", + cwd: "/", + dest: "/dir", + expectedFilepath: "/dir", + }, + { + srcName: "./", + filename: "context/foo", + cwd: "/", + dest: "/dir", + expectedFilepath: "/dir/context/foo", + }, + { + srcName: ".", + filename: "context/bar", + cwd: "/", + dest: "/dir", + expectedFilepath: "/dir/context/bar", + }, + { + srcName: ".", + filename: "context/bar", + cwd: "/", + dest: "/dir", + expectedFilepath: "/dir/context/bar", + }, +} + +func Test_RelativeFilepath(t *testing.T) { + for _, test := range relativeFilepathTests { + actualFilepath, err := RelativeFilepath(test.filename, test.srcName, test.dest, test.cwd, buildContextPath) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFilepath, actualFilepath) + } +} + +var matchSourcesTests = []struct { + srcs []string + files []string + cwd string + expectedFiles []string +}{ + { + srcs: []string{ + "pkg/*", + }, + files: []string{ + "pkg/a", + "pkg/b", + "/pkg/d", + "pkg/b/d/", + "dir/", + }, + cwd: "/", + expectedFiles: []string{ + "pkg/a", + "pkg/b", + "/pkg/d", + }, + }, +} + +func Test_MatchSources(t *testing.T) { + for _, test := range matchSourcesTests { + actualFiles, err := matchSources(test.srcs, test.files, test.cwd) + sort.Strings(actualFiles) + sort.Strings(test.expectedFiles) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFiles, actualFiles) + } +} + +var isSrcValidTests = []struct { + srcsAndDest []string + files map[string][]string + shouldErr bool +}{ + { + srcsAndDest: []string{ + "src1", + "src2", + "dest", + }, + files: nil, + shouldErr: true, + }, + { + srcsAndDest: []string{ + "src1", + "src2", + "dest/", + }, + files: nil, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2/", + "dest", + }, + files: nil, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2", + "dest", + }, + files: nil, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2", + "src*", + "dest/", + }, + files: nil, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2", + "src*", + "dest", + }, + files: map[string][]string{ + "src2": { + "src2/a", + "src2/b", + }, + "src*": {}, + }, + shouldErr: true, + }, + { + srcsAndDest: []string{ + "src2", + "src*", + "dest", + }, + files: map[string][]string{ + "src2": { + "src2/a", + }, + "src*": {}, + }, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2", + "src*", + "dest", + }, + files: map[string][]string{ + "src2": {}, + "src*": {}, + }, + shouldErr: true, + }, +} + +func Test_IsSrcsValid(t *testing.T) { + for _, test := range isSrcValidTests { + err := IsSrcsValid(test.srcsAndDest, test.files) + testutil.CheckError(t, test.shouldErr, err) + } +} + +var testResolveSources = []struct { + srcsAndDest []string + cwd string + expectedMap map[string][]string +}{ + { + srcsAndDest: []string{ + "context/foo", + "context/b*", + "dest/", + }, + cwd: "/", + expectedMap: map[string][]string{ + "context/foo": { + "context/foo", + }, + "context/bar": { + "context/bar", + "context/bar/bam", + "context/bar/bam/bat", + "context/bar/bat", + "context/bar/baz", + }, + }, + }, +} + +func Test_ResolveSources(t *testing.T) { + for _, test := range testResolveSources { + actualMap, err := ResolveSources(test.srcsAndDest, buildContextPath, test.cwd) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedMap, actualMap) + } +} diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index eafd18916..3b56c6cce 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -97,3 +97,48 @@ func fileSystemWhitelist(path string) ([]string, error) { } return whitelist, nil } + +// Files returns a list of all files at the filepath relative to root +func Files(fp string, root string) ([]string, error) { + var files []string + fullPath := filepath.Join(root, fp) + logrus.Debugf("Getting files and contents at root %s", fullPath) + err := filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(root, path) + if err != nil { + return err + } + files = append(files, relPath) + return err + }) + return files, err +} + +// FilepathExists returns true if the path exists +func FilepathExists(path string) bool { + _, err := os.Stat(path) + return (err == nil) +} + +// CreateFile creates a file at path with contents specified +func CreateFile(path string, contents []byte, perm os.FileMode) error { + // Create directory path if it doesn't exist + baseDir := filepath.Dir(path) + if _, err := os.Stat(baseDir); os.IsNotExist(err) { + logrus.Debugf("baseDir %s for file %s does not exist. Creating.", baseDir, path) + if err := os.MkdirAll(baseDir, perm); err != nil { + return err + } + } + + f, err := os.Create(path) + defer f.Close() + if err != nil { + return err + } + _, err = f.Write(contents) + return err +} diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index ccb7b3658..f21c6f788 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -51,3 +51,82 @@ func Test_fileSystemWhitelist(t *testing.T) { sort.Strings(expectedWhitelist) testutil.CheckErrorAndDeepEqual(t, false, err, expectedWhitelist, actualWhitelist) } + +var tests = []struct { + files map[string]string + directory string + expectedFiles []string +}{ + { + files: map[string]string{ + "/workspace/foo/a": "baz1", + "/workspace/foo/b": "baz2", + "/kbuild/file": "file", + }, + directory: "/workspace/foo/", + expectedFiles: []string{ + "workspace/foo/a", + "workspace/foo/b", + "workspace/foo", + }, + }, + { + files: map[string]string{ + "/workspace/foo/a": "baz1", + }, + directory: "/workspace/foo/a", + expectedFiles: []string{ + "workspace/foo/a", + }, + }, + { + files: map[string]string{ + "/workspace/foo/a": "baz1", + "/workspace/foo/b": "baz2", + "/workspace/baz": "hey", + "/kbuild/file": "file", + }, + directory: "/workspace", + expectedFiles: []string{ + "workspace/foo/a", + "workspace/foo/b", + "workspace/baz", + "workspace", + "workspace/foo", + }, + }, + { + files: map[string]string{ + "/workspace/foo/a": "baz1", + "/workspace/foo/b": "baz2", + "/kbuild/file": "file", + }, + directory: "", + expectedFiles: []string{ + "workspace/foo/a", + "workspace/foo/b", + "kbuild/file", + "workspace", + "workspace/foo", + "kbuild", + ".", + }, + }, +} + +func Test_Files(t *testing.T) { + for _, test := range tests { + testDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("err setting up temp dir: %v", err) + } + defer os.RemoveAll(testDir) + if err := testutil.SetupFiles(testDir, test.files); err != nil { + t.Fatalf("err setting up files: %v", err) + } + actualFiles, err := Files(test.directory, testDir) + sort.Strings(actualFiles) + sort.Strings(test.expectedFiles) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFiles, actualFiles) + } +} diff --git a/testutil/util.go b/testutil/util.go index 45607a772..64819c3ef 100644 --- a/testutil/util.go +++ b/testutil/util.go @@ -50,6 +50,12 @@ func CheckErrorAndDeepEqual(t *testing.T, shouldErr bool, err error, expected, a } } +func CheckError(t *testing.T, shouldErr bool, err error) { + if err := checkErr(shouldErr, err); err != nil { + t.Error(err) + } +} + func checkErr(shouldErr bool, err error) error { if err == nil && shouldErr { return fmt.Errorf("Expected error, but returned none")