diff --git a/executor/cmd/root.go b/executor/cmd/root.go index a34a5f3cd..80b2c088f 100644 --- a/executor/cmd/root.go +++ b/executor/cmd/root.go @@ -98,7 +98,7 @@ func execute() error { // Currently only supports single stage builds for _, stage := range stages { for _, cmd := range stage.Commands { - dockerCommand, err := commands.GetCommand(cmd) + dockerCommand, err := commands.GetCommand(cmd, srcContext) if err != nil { return err } diff --git a/integration_tests/context/arr[0].txt b/integration_tests/context/arr[0].txt new file mode 100644 index 000000000..b6fc4c620 --- /dev/null +++ b/integration_tests/context/arr[0].txt @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/integration_tests/context/bar/bam/bat b/integration_tests/context/bar/bam/bat new file mode 100644 index 000000000..1054901d8 --- /dev/null +++ b/integration_tests/context/bar/bam/bat @@ -0,0 +1 @@ +bat diff --git a/integration_tests/context/bar/bat b/integration_tests/context/bar/bat new file mode 100644 index 000000000..1054901d8 --- /dev/null +++ b/integration_tests/context/bar/bat @@ -0,0 +1 @@ +bat diff --git a/integration_tests/context/bar/baz b/integration_tests/context/bar/baz new file mode 100644 index 000000000..76018072e --- /dev/null +++ b/integration_tests/context/bar/baz @@ -0,0 +1 @@ +baz diff --git a/integration_tests/context/empty/.gitignore b/integration_tests/context/empty/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/integration_tests/context/foo b/integration_tests/context/foo new file mode 100755 index 000000000..257cc5642 --- /dev/null +++ b/integration_tests/context/foo @@ -0,0 +1 @@ +foo diff --git a/integration_tests/dockerfiles/Dockerfile_test_copy b/integration_tests/dockerfiles/Dockerfile_test_copy new file mode 100644 index 000000000..99c179c11 --- /dev/null +++ b/integration_tests/dockerfiles/Dockerfile_test_copy @@ -0,0 +1,20 @@ +FROM gcr.io/distroless/base +COPY context/foo foo +COPY context/foo /foodir/ +COPY context/bar/b* bar/ +COPY context/fo? /foo2 +COPY context/bar/doesnotexist* context/foo hello +COPY ./context/empty /empty +COPY ./ dir/ +COPY . newdir +COPY context/bar /baz/ +COPY ["context/foo", "/tmp/foo" ] +COPY context/b* /baz/ +COPY context/foo context/bar/ba? /test/ +COPY context/arr[[]0].txt /mydir/ +COPY context/bar/bat . + +ENV contextenv ./context +COPY ${contextenv}/foo /tmp/foo2 +COPY $contextenv/foo /tmp/foo3 +COPY $contextenv/* /tmp/${contextenv}/ diff --git a/integration_tests/dockerfiles/config_test_copy.json b/integration_tests/dockerfiles/config_test_copy.json new file mode 100644 index 000000000..6d222de51 --- /dev/null +++ b/integration_tests/dockerfiles/config_test_copy.json @@ -0,0 +1,12 @@ +[ + { + "Image1": "gcr.io/kbuild-test/docker-test-copy:latest", + "Image2": "gcr.io/kbuild-test/kbuild-test-copy:latest", + "DiffType": "File", + "Diff": { + "Adds": null, + "Dels": null, + "Mods": null + } + } +] \ No newline at end of file diff --git a/integration_tests/integration_test_yaml.go b/integration_tests/integration_test_yaml.go index 5471e207e..545add4d5 100644 --- a/integration_tests/integration_test_yaml.go +++ b/integration_tests/integration_test_yaml.go @@ -49,6 +49,13 @@ var fileTests = []struct { context: "integration_tests/dockerfiles/", repo: "test-run-2", }, + { + description: "test copy", + dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_copy", + configPath: "/workspace/integration_tests/dockerfiles/config_test_copy.json", + context: "/workspace/integration_tests/", + repo: "test-copy", + }, } var structureTests = []struct { @@ -126,7 +133,7 @@ func main() { kbuildImage := testRepo + kbuildPrefix + test.repo kbuild := step{ Name: executorImage, - Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath}, + Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.context}, } // Pull the kbuild image diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index 742fd5ef3..8f6b6a8cc 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -34,10 +34,12 @@ type DockerCommand interface { FilesToSnapshot() []string } -func GetCommand(cmd instructions.Command) (DockerCommand, error) { +func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, error) { switch c := cmd.(type) { case *instructions.RunCommand: return &RunCommand{cmd: c}, nil + case *instructions.CopyCommand: + return &CopyCommand{cmd: c, buildcontext: buildcontext}, nil case *instructions.ExposeCommand: return &ExposeCommand{cmd: c}, nil case *instructions.EnvCommand: diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go new file mode 100644 index 000000000..96db34911 --- /dev/null +++ b/pkg/commands/copy.go @@ -0,0 +1,102 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util" + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/sirupsen/logrus" + "os" + "path/filepath" + "strings" +) + +type CopyCommand struct { + cmd *instructions.CopyCommand + buildcontext string + snapshotFiles []string +} + +func (c *CopyCommand) ExecuteCommand(config *manifest.Schema2Config) error { + srcs := c.cmd.SourcesAndDest[:len(c.cmd.SourcesAndDest)-1] + dest := c.cmd.SourcesAndDest[len(c.cmd.SourcesAndDest)-1] + + logrus.Infof("cmd: copy %s", srcs) + logrus.Infof("dest: %s", dest) + + // First, resolve any environment replacement + resolvedEnvs, err := util.ResolveEnvironmentReplacementList(c.copyToString(), c.cmd.SourcesAndDest, config.Env, true) + if err != nil { + return err + } + dest = resolvedEnvs[len(c.cmd.SourcesAndDest)-1] + // Get a map of [src]:[files rooted at src] + srcMap, err := util.ResolveSources(resolvedEnvs, c.buildcontext) + if err != nil { + return err + } + // For each source, iterate through each file within and copy it over + for src, files := range srcMap { + for _, file := range files { + fi, err := os.Stat(filepath.Join(c.buildcontext, file)) + if err != nil { + return err + } + destPath, err := util.DestinationFilepath(file, src, dest, config.WorkingDir, c.buildcontext) + if err != nil { + return err + } + // If source file is a directory, we want to create a directory ... + if fi.IsDir() { + logrus.Infof("Creating directory %s", destPath) + if err := os.MkdirAll(destPath, fi.Mode()); err != nil { + return err + } + } else { + // ... Else, we want to copy over a file + logrus.Infof("Copying file %s to %s", file, destPath) + srcFile, err := os.Open(filepath.Join(c.buildcontext, file)) + if err != nil { + return err + } + defer srcFile.Close() + if err := util.CreateFile(destPath, srcFile, fi.Mode()); err != nil { + return err + } + } + // Append the destination file to the list of files that should be snapshotted later + c.snapshotFiles = append(c.snapshotFiles, destPath) + } + } + return nil +} + +func (c *CopyCommand) copyToString() string { + copy := []string{"COPY"} + return strings.Join(append(copy, c.cmd.SourcesAndDest...), " ") +} + +// FilesToSnapshot should return an empty array if still nil; no files were changed +func (c *CopyCommand) FilesToSnapshot() []string { + return c.snapshotFiles +} + +// CreatedBy returns some information about the command for the image config +func (c *CopyCommand) CreatedBy() string { + return strings.Join(c.cmd.SourcesAndDest, " ") +} diff --git a/pkg/commands/env.go b/pkg/commands/env.go index 23fbfd658..b88b5f01e 100644 --- a/pkg/commands/env.go +++ b/pkg/commands/env.go @@ -35,11 +35,11 @@ func (e *EnvCommand) ExecuteCommand(config *manifest.Schema2Config) error { envString := envToString(e.cmd) newEnvs := e.cmd.Env for index, pair := range newEnvs { - expandedKey, err := util.ResolveEnvironmentReplacement(envString, pair.Key, config.Env) + expandedKey, err := util.ResolveEnvironmentReplacement(envString, pair.Key, config.Env, false) if err != nil { return err } - expandedValue, err := util.ResolveEnvironmentReplacement(envString, pair.Value, config.Env) + expandedValue, err := util.ResolveEnvironmentReplacement(envString, pair.Value, config.Env, false) if err != nil { return err } diff --git a/pkg/commands/expose.go b/pkg/commands/expose.go index 883a4cd35..7421a8697 100644 --- a/pkg/commands/expose.go +++ b/pkg/commands/expose.go @@ -36,7 +36,7 @@ func (r *ExposeCommand) ExecuteCommand(config *manifest.Schema2Config) error { // Add any new ones in for _, p := range r.cmd.Ports { // Resolve any environment variables - p, err := util.ResolveEnvironmentReplacement(exposeString, p, config.Env) + p, err := util.ResolveEnvironmentReplacement(exposeString, p, config.Env, false) if err != nil { return err } diff --git a/pkg/util/command_util.go b/pkg/util/command_util.go index dde948b84..d9187a7ed 100644 --- a/pkg/util/command_util.go +++ b/pkg/util/command_util.go @@ -18,12 +18,31 @@ package util import ( "bytes" + "github.com/docker/docker/builder/dockerfile/instructions" "github.com/docker/docker/builder/dockerfile/parser" "github.com/docker/docker/builder/dockerfile/shell" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "os" "path/filepath" + "strings" ) -// ResolveEnvironmentReplacement resolves replacing env variables in some text from envs +// ResolveEnvironmentReplacement resolves a list of values by calling resolveEnvironmentReplacement +func ResolveEnvironmentReplacementList(command string, values, envs []string, isFilepath bool) ([]string, error) { + var resolvedValues []string + for _, value := range values { + resolved, err := ResolveEnvironmentReplacement(command, value, envs, isFilepath) + logrus.Infof("Resolved %s to %s", value, resolved) + if err != nil { + return nil, err + } + resolvedValues = append(resolvedValues, resolved) + } + return resolvedValues, nil +} + +// resolveEnvironmentReplacement resolves replacing env variables in some text from envs // It takes in a string representation of the command, the value to be resolved, and a list of envs (config.Env) // Ex: fp = $foo/newdir, envs = [foo=/foodir], then this should return /foodir/newdir // The dockerfile/shell package handles processing env values @@ -32,30 +51,159 @@ import ( // ""a'b'c"" -> "a'b'c" // "Rex\ The\ Dog \" -> "Rex The Dog" // "a\"b" -> "a"b" -func ResolveEnvironmentReplacement(command, value string, envs []string) (string, error) { - p, err := parser.Parse(bytes.NewReader([]byte(command))) - if err != nil { - return "", err - } - shlex := shell.NewLex(p.EscapeToken) - return shlex.ProcessWord(value, envs) -} - -// ResolveFilepathEnvironmentReplacement replaces env variables in filepaths -// and returns a cleaned version of the path -func ResolveFilepathEnvironmentReplacement(command, value string, envs []string) (string, error) { +func ResolveEnvironmentReplacement(command, value string, envs []string, isFilepath bool) (string, error) { p, err := parser.Parse(bytes.NewReader([]byte(command))) if err != nil { return "", err } shlex := shell.NewLex(p.EscapeToken) fp, err := shlex.ProcessWord(value, envs) + if !isFilepath { + return fp, err + } if err != nil { return "", err } fp = filepath.Clean(fp) - if filepath.IsAbs(value) { - fp = filepath.Join(fp, "/") + if IsDestDir(value) { + fp = fp + "/" } return fp, nil } + +// ContainsWildcards returns true if any entry in paths contains wildcards +func ContainsWildcards(paths []string) bool { + for _, path := range paths { + if strings.ContainsAny(path, "*?[") { + return true + } + } + return false +} + +// ResolveSources resolves the given sources if the sources contains wildcards +// It returns a map of [src]:[files rooted at src] +func ResolveSources(srcsAndDest instructions.SourcesAndDest, root string) (map[string][]string, error) { + srcs := srcsAndDest[:len(srcsAndDest)-1] + // If sources contain wildcards, we first need to resolve them to actual paths + if ContainsWildcards(srcs) { + logrus.Debugf("Resolving srcs %v...", srcs) + files, err := RelativeFiles("", root) + if err != nil { + return nil, err + } + srcs, err = matchSources(srcs, files) + if err != nil { + return nil, err + } + logrus.Debugf("Resolved sources to %v", srcs) + } + // Now, get a map of [src]:[files rooted at src] + srcMap, err := SourcesToFilesMap(srcs, root) + if err != nil { + return nil, err + } + // Check to make sure the sources are valid + return srcMap, IsSrcsValid(srcsAndDest, srcMap) +} + +// matchSources returns a list of sources that match wildcards +func matchSources(srcs, files []string) ([]string, error) { + var matchedSources []string + for _, src := range srcs { + src = filepath.Clean(src) + for _, file := range files { + matched, err := filepath.Match(src, file) + if err != nil { + return nil, err + } + if matched { + matchedSources = append(matchedSources, file) + } + } + } + return matchedSources, nil +} + +func IsDestDir(path string) bool { + return strings.HasSuffix(path, "/") || path == "." +} + +// DestinationFilepath returns the destination filepath from the build context to the image filesystem +// If source is a file: +// If dest is a dir, copy it to /dest/relpath +// If dest is a file, copy directly to dest +// If source is a dir: +// Assume dest is also a dir, and copy to dest/relpath +// If dest is not an absolute filepath, add /cwd to the beginning +func DestinationFilepath(filename, srcName, dest, cwd, buildcontext string) (string, error) { + fi, err := os.Stat(filepath.Join(buildcontext, filename)) + if err != nil { + return "", err + } + src, err := os.Stat(filepath.Join(buildcontext, srcName)) + if err != nil { + return "", err + } + if src.IsDir() || IsDestDir(dest) { + relPath, err := filepath.Rel(srcName, filename) + if err != nil { + return "", err + } + if relPath == "." && !fi.IsDir() { + relPath = filepath.Base(filename) + } + destPath := filepath.Join(dest, relPath) + if filepath.IsAbs(dest) { + return destPath, nil + } + return filepath.Join(cwd, destPath), nil + } + if filepath.IsAbs(dest) { + return dest, nil + } + return filepath.Join(cwd, dest), nil +} + +// SourcesToFilesMap returns a map of [src]:[files rooted at source] +func SourcesToFilesMap(srcs []string, root string) (map[string][]string, error) { + srcMap := make(map[string][]string) + for _, src := range srcs { + src = filepath.Clean(src) + files, err := RelativeFiles(src, root) + if err != nil { + return nil, err + } + srcMap[src] = files + } + return srcMap, nil +} + +// IsSrcsValid returns an error if the sources provided are invalid, or nil otherwise +func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, srcMap map[string][]string) error { + srcs := srcsAndDest[:len(srcsAndDest)-1] + dest := srcsAndDest[len(srcsAndDest)-1] + + totalFiles := 0 + for _, files := range srcMap { + totalFiles += len(files) + } + if totalFiles == 0 { + return errors.New("copy failed: no source files specified") + } + + if !ContainsWildcards(srcs) { + // If multiple sources and destination isn't a directory, return an error + if len(srcs) > 1 && !IsDestDir(dest) { + return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'") + } + return nil + } + + // If there are wildcards, and the destination is a file, there must be exactly one file to copy over, + // Otherwise, return an error + if !IsDestDir(dest) && totalFiles > 1 { + return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'") + } + return nil +} diff --git a/pkg/util/command_util_test.go b/pkg/util/command_util_test.go index 59cdc2454..f5ec7ae7e 100644 --- a/pkg/util/command_util_test.go +++ b/pkg/util/command_util_test.go @@ -18,6 +18,7 @@ package util import ( "github.com/GoogleCloudPlatform/k8s-container-builder/testutil" + "sort" "testing" ) @@ -25,6 +26,7 @@ var testEnvReplacement = []struct { path string command string envs []string + isFilepath bool expectedPath string }{ { @@ -33,8 +35,18 @@ var testEnvReplacement = []struct { envs: []string{ "simple=/path/", }, + isFilepath: true, expectedPath: "/simple/path", }, + { + path: "/simple/path/", + command: "WORKDIR /simple/path/", + envs: []string{ + "simple=/path/", + }, + isFilepath: true, + expectedPath: "/simple/path/", + }, { path: "${a}/b", command: "WORKDIR ${a}/b", @@ -42,6 +54,7 @@ var testEnvReplacement = []struct { "a=/path/", "b=/path2/", }, + isFilepath: true, expectedPath: "/path/b", }, { @@ -51,14 +64,26 @@ var testEnvReplacement = []struct { "a=/path/", "b=/path2/", }, + isFilepath: true, expectedPath: "/path/b", }, + { + path: "/$a/b/", + command: "COPY /${a}/b /c/", + envs: []string{ + "a=/path/", + "b=/path2/", + }, + isFilepath: true, + expectedPath: "/path/b/", + }, { path: "\\$foo", command: "COPY \\$foo /quux", envs: []string{ "foo=/path/", }, + isFilepath: true, expectedPath: "$foo", }, { @@ -73,7 +98,308 @@ var testEnvReplacement = []struct { func Test_EnvReplacement(t *testing.T) { for _, test := range testEnvReplacement { - actualPath, err := ResolveEnvironmentReplacement(test.command, test.path, test.envs) + actualPath, err := ResolveEnvironmentReplacement(test.command, test.path, test.envs, test.isFilepath) testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedPath, actualPath) + + } +} + +var buildContextPath = "../../integration_tests/" + +var destinationFilepathTests = []struct { + srcName string + filename string + dest string + cwd string + buildcontext string + expectedFilepath string +}{ + { + srcName: "context/foo", + filename: "context/foo", + dest: "/foo", + cwd: "/", + expectedFilepath: "/foo", + }, + { + srcName: "context/foo", + filename: "context/foo", + dest: "/foodir/", + cwd: "/", + expectedFilepath: "/foodir/foo", + }, + { + srcName: "context/foo", + filename: "./context/foo", + cwd: "/", + dest: "foo", + expectedFilepath: "/foo", + }, + { + srcName: "context/bar/", + filename: "context/bar/bam/bat", + cwd: "/", + dest: "pkg/", + expectedFilepath: "/pkg/bam/bat", + }, + { + srcName: "context/bar/", + filename: "context/bar/bam/bat", + cwd: "/newdir", + dest: "pkg/", + expectedFilepath: "/newdir/pkg/bam/bat", + }, + { + srcName: "./context/empty", + filename: "context/empty", + cwd: "/", + dest: "/empty", + expectedFilepath: "/empty", + }, + { + srcName: "./context/empty", + filename: "context/empty", + cwd: "/dir", + dest: "/empty", + expectedFilepath: "/empty", + }, + { + srcName: "./", + filename: "./", + cwd: "/", + dest: "/dir", + expectedFilepath: "/dir", + }, + { + srcName: "./", + filename: "context/foo", + cwd: "/", + dest: "/dir", + expectedFilepath: "/dir/context/foo", + }, + { + srcName: ".", + filename: "context/bar", + cwd: "/", + dest: "/dir", + expectedFilepath: "/dir/context/bar", + }, + { + srcName: ".", + filename: "context/bar", + cwd: "/", + dest: "/dir", + expectedFilepath: "/dir/context/bar", + }, + { + srcName: "context/foo", + filename: "context/foo", + cwd: "/test", + dest: ".", + expectedFilepath: "/test/foo", + }, +} + +func Test_DestinationFilepath(t *testing.T) { + for _, test := range destinationFilepathTests { + actualFilepath, err := DestinationFilepath(test.filename, test.srcName, test.dest, test.cwd, buildContextPath) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFilepath, actualFilepath) + } +} + +var matchSourcesTests = []struct { + srcs []string + files []string + expectedFiles []string +}{ + { + srcs: []string{ + "pkg/*", + }, + files: []string{ + "pkg/a", + "pkg/b", + "/pkg/d", + "pkg/b/d/", + "dir/", + }, + expectedFiles: []string{ + "pkg/a", + "pkg/b", + }, + }, +} + +func Test_MatchSources(t *testing.T) { + for _, test := range matchSourcesTests { + actualFiles, err := matchSources(test.srcs, test.files) + sort.Strings(actualFiles) + sort.Strings(test.expectedFiles) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFiles, actualFiles) + } +} + +var isSrcValidTests = []struct { + srcsAndDest []string + files map[string][]string + shouldErr bool +}{ + { + srcsAndDest: []string{ + "src1", + "src2", + "dest", + }, + files: map[string][]string{ + "src1": { + "file1", + }, + "src2:": { + "file2", + }, + }, + shouldErr: true, + }, + { + srcsAndDest: []string{ + "src1", + "src2", + "dest/", + }, + files: map[string][]string{ + "src1": { + "file1", + }, + "src2:": { + "file2", + }, + }, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2/", + "dest", + }, + files: map[string][]string{ + "src1": { + "file1", + }, + "src2:": { + "file2", + }, + }, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2", + "dest", + }, + files: map[string][]string{ + "src1": { + "file1", + }, + "src2:": { + "file2", + }, + }, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2", + "src*", + "dest/", + }, + files: map[string][]string{ + "src1": { + "file1", + }, + "src2:": { + "file2", + }, + }, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2", + "src*", + "dest", + }, + files: map[string][]string{ + "src2": { + "src2/a", + "src2/b", + }, + "src*": {}, + }, + shouldErr: true, + }, + { + srcsAndDest: []string{ + "src2", + "src*", + "dest", + }, + files: map[string][]string{ + "src2": { + "src2/a", + }, + "src*": {}, + }, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2", + "src*", + "dest", + }, + files: map[string][]string{ + "src2": {}, + "src*": {}, + }, + shouldErr: true, + }, +} + +func Test_IsSrcsValid(t *testing.T) { + for _, test := range isSrcValidTests { + err := IsSrcsValid(test.srcsAndDest, test.files) + testutil.CheckError(t, test.shouldErr, err) + } +} + +var testResolveSources = []struct { + srcsAndDest []string + expectedMap map[string][]string +}{ + { + srcsAndDest: []string{ + "context/foo", + "context/b*", + "dest/", + }, + expectedMap: map[string][]string{ + "context/foo": { + "context/foo", + }, + "context/bar": { + "context/bar", + "context/bar/bam", + "context/bar/bam/bat", + "context/bar/bat", + "context/bar/baz", + }, + }, + }, +} + +func Test_ResolveSources(t *testing.T) { + for _, test := range testResolveSources { + actualMap, err := ResolveSources(test.srcsAndDest, buildContextPath) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedMap, actualMap) } } diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index eafd18916..c0ba9008e 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -97,3 +97,48 @@ func fileSystemWhitelist(path string) ([]string, error) { } return whitelist, nil } + +// RelativeFiles returns a list of all files at the filepath relative to root +func RelativeFiles(fp string, root string) ([]string, error) { + var files []string + fullPath := filepath.Join(root, fp) + logrus.Debugf("Getting files and contents at root %s", fullPath) + err := filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(root, path) + if err != nil { + return err + } + files = append(files, relPath) + return nil + }) + return files, err +} + +// FilepathExists returns true if the path exists +func FilepathExists(path string) bool { + _, err := os.Stat(path) + return !os.IsNotExist(err) +} + +// CreateFile creates a file at path and copies over contents from the reader +func CreateFile(path string, reader io.Reader, perm os.FileMode) error { + // Create directory path if it doesn't exist + baseDir := filepath.Dir(path) + if _, err := os.Stat(baseDir); os.IsNotExist(err) { + logrus.Debugf("baseDir %s for file %s does not exist. Creating.", baseDir, path) + if err := os.MkdirAll(baseDir, 0755); err != nil { + return err + } + } + dest, err := os.Create(path) + if err != nil { + return err + } + if _, err := io.Copy(dest, reader); err != nil { + return err + } + return dest.Chmod(perm) +} diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index ccb7b3658..c8ebc88cf 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -51,3 +51,82 @@ func Test_fileSystemWhitelist(t *testing.T) { sort.Strings(expectedWhitelist) testutil.CheckErrorAndDeepEqual(t, false, err, expectedWhitelist, actualWhitelist) } + +var tests = []struct { + files map[string]string + directory string + expectedFiles []string +}{ + { + files: map[string]string{ + "/workspace/foo/a": "baz1", + "/workspace/foo/b": "baz2", + "/kbuild/file": "file", + }, + directory: "/workspace/foo/", + expectedFiles: []string{ + "workspace/foo/a", + "workspace/foo/b", + "workspace/foo", + }, + }, + { + files: map[string]string{ + "/workspace/foo/a": "baz1", + }, + directory: "/workspace/foo/a", + expectedFiles: []string{ + "workspace/foo/a", + }, + }, + { + files: map[string]string{ + "/workspace/foo/a": "baz1", + "/workspace/foo/b": "baz2", + "/workspace/baz": "hey", + "/kbuild/file": "file", + }, + directory: "/workspace", + expectedFiles: []string{ + "workspace/foo/a", + "workspace/foo/b", + "workspace/baz", + "workspace", + "workspace/foo", + }, + }, + { + files: map[string]string{ + "/workspace/foo/a": "baz1", + "/workspace/foo/b": "baz2", + "/kbuild/file": "file", + }, + directory: "", + expectedFiles: []string{ + "workspace/foo/a", + "workspace/foo/b", + "kbuild/file", + "workspace", + "workspace/foo", + "kbuild", + ".", + }, + }, +} + +func Test_RelativeFiles(t *testing.T) { + for _, test := range tests { + testDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("err setting up temp dir: %v", err) + } + defer os.RemoveAll(testDir) + if err := testutil.SetupFiles(testDir, test.files); err != nil { + t.Fatalf("err setting up files: %v", err) + } + actualFiles, err := RelativeFiles(test.directory, testDir) + sort.Strings(actualFiles) + sort.Strings(test.expectedFiles) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFiles, actualFiles) + } +} diff --git a/testutil/util.go b/testutil/util.go index 45607a772..64819c3ef 100644 --- a/testutil/util.go +++ b/testutil/util.go @@ -50,6 +50,12 @@ func CheckErrorAndDeepEqual(t *testing.T, shouldErr bool, err error, expected, a } } +func CheckError(t *testing.T, shouldErr bool, err error) { + if err := checkErr(shouldErr, err); err != nil { + t.Error(err) + } +} + func checkErr(shouldErr bool, err error) error { if err == nil && shouldErr { return fmt.Errorf("Expected error, but returned none")