diff --git a/Makefile b/Makefile index 9d1064eb8..1694112bf 100644 --- a/Makefile +++ b/Makefile @@ -48,3 +48,7 @@ test: out/executor out/kbuild .PHONY: integration-test integration-test: out/executor out/kbuild @ ./integration-test.sh + +.PHONY: images +images: out/executor out/kbuild + docker build -t $(REGISTRY)/executor:latest -f deploy/Dockerfile . diff --git a/integration_tests/executor/Dockerfile b/deploy/Dockerfile similarity index 100% rename from integration_tests/executor/Dockerfile rename to deploy/Dockerfile diff --git a/executor/cmd/root.go b/executor/cmd/root.go index a34a5f3cd..80b2c088f 100644 --- a/executor/cmd/root.go +++ b/executor/cmd/root.go @@ -98,7 +98,7 @@ func execute() error { // Currently only supports single stage builds for _, stage := range stages { for _, cmd := range stage.Commands { - dockerCommand, err := commands.GetCommand(cmd) + dockerCommand, err := commands.GetCommand(cmd, srcContext) if err != nil { return err } diff --git a/integration_tests/context/arr[0].txt b/integration_tests/context/arr[0].txt new file mode 100644 index 000000000..b6fc4c620 --- /dev/null +++ b/integration_tests/context/arr[0].txt @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/integration_tests/context/bar/bam/bat b/integration_tests/context/bar/bam/bat new file mode 100644 index 000000000..1054901d8 --- /dev/null +++ b/integration_tests/context/bar/bam/bat @@ -0,0 +1 @@ +bat diff --git a/integration_tests/context/bar/bat b/integration_tests/context/bar/bat new file mode 100644 index 000000000..1054901d8 --- /dev/null +++ b/integration_tests/context/bar/bat @@ -0,0 +1 @@ +bat diff --git a/integration_tests/context/bar/baz b/integration_tests/context/bar/baz new file mode 100644 index 000000000..76018072e --- /dev/null +++ b/integration_tests/context/bar/baz @@ -0,0 +1 @@ +baz diff --git a/integration_tests/context/empty/.gitignore b/integration_tests/context/empty/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/integration_tests/context/foo b/integration_tests/context/foo new file mode 100755 index 000000000..257cc5642 --- /dev/null +++ b/integration_tests/context/foo @@ -0,0 +1 @@ +foo diff --git a/integration_tests/dockerfiles/Dockerfile_test_copy b/integration_tests/dockerfiles/Dockerfile_test_copy new file mode 100644 index 000000000..af4dc77d3 --- /dev/null +++ b/integration_tests/dockerfiles/Dockerfile_test_copy @@ -0,0 +1,14 @@ +FROM gcr.io/distroless/base +COPY context/foo foo +COPY context/foo /foodir/ +COPY context/bar/b* bar/ +COPY context/fo? /foo2 +COPY context/bar/doesnotexist* context/foo hello +COPY ./context/empty /empty +COPY ./ dir/ +COPY . newdir +COPY context/bar /baz/ +COPY ["context/foo", "/tmp/foo" ] +COPY context/b* /baz/ +COPY context/foo context/bar/ba? /test/ +COPY context/arr[[]0].txt /mydir/ diff --git a/integration_tests/dockerfiles/config_test_copy.json b/integration_tests/dockerfiles/config_test_copy.json new file mode 100644 index 000000000..6d222de51 --- /dev/null +++ b/integration_tests/dockerfiles/config_test_copy.json @@ -0,0 +1,12 @@ +[ + { + "Image1": "gcr.io/kbuild-test/docker-test-copy:latest", + "Image2": "gcr.io/kbuild-test/kbuild-test-copy:latest", + "DiffType": "File", + "Diff": { + "Adds": null, + "Dels": null, + "Mods": null + } + } +] \ No newline at end of file diff --git a/integration_tests/integration_test_yaml.go b/integration_tests/integration_test_yaml.go index 19c5a8926..3129456ff 100644 --- a/integration_tests/integration_test_yaml.go +++ b/integration_tests/integration_test_yaml.go @@ -18,6 +18,7 @@ package main import ( "fmt" + "gopkg.in/yaml.v2" ) @@ -49,6 +50,13 @@ var fileTests = []struct { context: "integration_tests/dockerfiles/", repo: "test-run-2", }, + { + description: "test copy", + dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_copy", + configPath: "/workspace/integration_tests/dockerfiles/config_test_copy.json", + context: "/workspace/integration_tests/", + repo: "test-copy", + }, } var structureTests = []struct { @@ -116,7 +124,7 @@ func main() { // Build executor image buildExecutorImage := step{ Name: dockerImage, - Args: []string{"build", "-t", executorImage, "-f", "integration_tests/executor/Dockerfile", "."}, + Args: []string{"build", "-t", executorImage, "-f", "deploy/Dockerfile", "."}, } y := testyaml{ Steps: []step{containerDiffStep, containerDiffPermissions, structureTestsStep, structureTestPermissions, buildExecutorImage}, @@ -133,7 +141,7 @@ func main() { kbuildImage := testRepo + kbuildPrefix + test.repo kbuild := step{ Name: executorImage, - Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath}, + Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.context}, } // Pull the kbuild image diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index 04622035b..08fa3ceb1 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -34,16 +34,22 @@ type DockerCommand interface { FilesToSnapshot() []string } -func GetCommand(cmd instructions.Command) (DockerCommand, error) { +func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, error) { switch c := cmd.(type) { case *instructions.RunCommand: return &RunCommand{cmd: c}, nil + case *instructions.CopyCommand: + return &CopyCommand{cmd: c, buildcontext: buildcontext}, nil + case *instructions.ExposeCommand: + return &ExposeCommand{cmd: c}, nil case *instructions.EnvCommand: return &EnvCommand{cmd: c}, nil case *instructions.CmdCommand: return &CmdCommand{cmd: c}, nil case *instructions.EntrypointCommand: return &EntrypointCommand{cmd: c}, nil + case *instructions.LabelCommand: + return &LabelCommand{cmd: c}, nil } return nil, errors.Errorf("%s is not a supported command", cmd.Name()) } diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go new file mode 100644 index 000000000..96fded65d --- /dev/null +++ b/pkg/commands/copy.go @@ -0,0 +1,91 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util" + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/sirupsen/logrus" + "os" + "path/filepath" + "strings" +) + +type CopyCommand struct { + cmd *instructions.CopyCommand + buildcontext string + snapshotFiles []string +} + +func (c *CopyCommand) ExecuteCommand(config *manifest.Schema2Config) error { + srcs := c.cmd.SourcesAndDest[:len(c.cmd.SourcesAndDest)-1] + dest := c.cmd.SourcesAndDest[len(c.cmd.SourcesAndDest)-1] + + logrus.Infof("cmd: copy %s", srcs) + logrus.Infof("dest: %s", dest) + + // Get a map of [src]:[files rooted at src] + srcMap, err := util.ResolveSources(c.cmd.SourcesAndDest, c.buildcontext) + if err != nil { + return err + } + // For each source, iterate through each file within and copy it over + for src, files := range srcMap { + for _, file := range files { + fi, err := os.Stat(filepath.Join(c.buildcontext, file)) + if err != nil { + return err + } + destPath, err := util.DestinationFilepath(file, src, dest, config.WorkingDir, c.buildcontext) + if err != nil { + return err + } + // If source file is a directory, we want to create a directory ... + if fi.IsDir() { + logrus.Infof("Creating directory %s", destPath) + if err := os.MkdirAll(destPath, fi.Mode()); err != nil { + return err + } + } else { + // ... Else, we want to copy over a file + logrus.Infof("Copying file %s to %s", file, destPath) + srcFile, err := os.Open(filepath.Join(c.buildcontext, file)) + if err != nil { + return err + } + defer srcFile.Close() + if err := util.CreateFile(destPath, srcFile, fi.Mode()); err != nil { + return err + } + } + // Append the destination file to the list of files that should be snapshotted later + c.snapshotFiles = append(c.snapshotFiles, destPath) + } + } + return nil +} + +// FilesToSnapshot should return an empty array if still nil; no files were changed +func (c *CopyCommand) FilesToSnapshot() []string { + return c.snapshotFiles +} + +// CreatedBy returns some information about the command for the image config +func (c *CopyCommand) CreatedBy() string { + return strings.Join(c.cmd.SourcesAndDest, " ") +} diff --git a/pkg/commands/expose.go b/pkg/commands/expose.go new file mode 100644 index 000000000..716b03294 --- /dev/null +++ b/pkg/commands/expose.go @@ -0,0 +1,74 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "fmt" + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/sirupsen/logrus" + "strings" +) + +type ExposeCommand struct { + cmd *instructions.ExposeCommand +} + +func (r *ExposeCommand) ExecuteCommand(config *manifest.Schema2Config) error { + return updateExposedPorts(r.cmd.Ports, config) +} + +func validProtocol(protocol string) bool { + validProtocols := [2]string{"tcp", "udp"} + for _, p := range validProtocols { + if protocol == p { + return true + } + } + return false +} + +func updateExposedPorts(ports []string, config *manifest.Schema2Config) error { + // Grab the currently exposed ports + existingPorts := config.ExposedPorts + + // Add any new ones in + for _, p := range ports { + // Add the default protocol if one isn't specified + if !strings.Contains(p, "/") { + p = p + "/tcp" + } + protocol := strings.Split(p, "/")[1] + if !validProtocol(protocol) { + return fmt.Errorf("Invalid protocol: %s", protocol) + } + logrus.Infof("Adding exposed port: %s", p) + var x struct{} + existingPorts[manifest.Schema2Port(p)] = x + } + config.ExposedPorts = existingPorts + return nil +} + +func (r *ExposeCommand) FilesToSnapshot() []string { + return []string{} +} + +func (r *ExposeCommand) CreatedBy() string { + s := []string{r.cmd.Name()} + return strings.Join(append(s, r.cmd.Ports...), " ") +} diff --git a/pkg/commands/expose_test.go b/pkg/commands/expose_test.go new file mode 100644 index 000000000..fc7fb2c0e --- /dev/null +++ b/pkg/commands/expose_test.go @@ -0,0 +1,61 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/testutil" + "github.com/containers/image/manifest" + "testing" +) + +func TestUpdateExposedPorts(t *testing.T) { + cfg := &manifest.Schema2Config{ + ExposedPorts: manifest.Schema2PortSet{ + "8080/tcp": {}, + }, + } + + ports := []string{ + "8080", + "8081/tcp", + "8082", + "8083/udp", + } + + expectedPorts := manifest.Schema2PortSet{ + "8080/tcp": {}, + "8081/tcp": {}, + "8082/tcp": {}, + "8083/udp": {}, + } + + err := updateExposedPorts(ports, cfg) + testutil.CheckErrorAndDeepEqual(t, false, err, expectedPorts, cfg.ExposedPorts) +} + +func TestInvalidProtocol(t *testing.T) { + cfg := &manifest.Schema2Config{ + ExposedPorts: manifest.Schema2PortSet{}, + } + + ports := []string{ + "80/garbage", + } + + err := updateExposedPorts(ports, cfg) + testutil.CheckErrorAndDeepEqual(t, true, err, nil, nil) +} diff --git a/pkg/commands/label.go b/pkg/commands/label.go new file mode 100644 index 000000000..ec80e494f --- /dev/null +++ b/pkg/commands/label.go @@ -0,0 +1,72 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/docker/docker/builder/dockerfile/shell" + "github.com/sirupsen/logrus" + "strings" +) + +type LabelCommand struct { + cmd *instructions.LabelCommand +} + +func (r *LabelCommand) ExecuteCommand(config *manifest.Schema2Config) error { + return updateLabels(r.cmd.Labels, config) +} + +func updateLabels(labels []instructions.KeyValuePair, config *manifest.Schema2Config) error { + existingLabels := config.Labels + + // Let's unescape values before setting the label + shlex := shell.NewLex('\\') + for index, kvp := range labels { + unescaped, err := shlex.ProcessWord(kvp.Value, []string{}) + if err != nil { + return err + } + labels[index] = instructions.KeyValuePair{ + Key: kvp.Key, + Value: unescaped, + } + } + for _, kvp := range labels { + logrus.Infof("Applying label %s=%s", kvp.Key, kvp.Value) + existingLabels[kvp.Key] = kvp.Value + } + + config.Labels = existingLabels + return nil + +} + +// No files have changed, this command only touches metadata. +func (r *LabelCommand) FilesToSnapshot() []string { + return []string{} +} + +// CreatedBy returns some information about the command for the image config history +func (r *LabelCommand) CreatedBy() string { + l := []string{r.cmd.Name()} + for _, kvp := range r.cmd.Labels { + l = append(l, kvp.String()) + } + return strings.Join(l, " ") +} diff --git a/pkg/commands/label_test.go b/pkg/commands/label_test.go new file mode 100644 index 000000000..f997611c3 --- /dev/null +++ b/pkg/commands/label_test.go @@ -0,0 +1,60 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/testutil" + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "testing" +) + +func TestUpdateLabels(t *testing.T) { + cfg := &manifest.Schema2Config{ + Labels: map[string]string{ + "foo": "bar", + }, + } + + labels := []instructions.KeyValuePair{ + { + Key: "foo", + Value: "override", + }, + { + Key: "bar", + Value: "baz", + }, + { + Key: "multiword", + Value: "lots\\ of\\ words", + }, + { + Key: "backslashes", + Value: "lots\\\\ of\\\\ words", + }, + } + + expectedLabels := map[string]string{ + "foo": "override", + "bar": "baz", + "multiword": "lots of words", + "backslashes": "lots\\ of\\ words", + } + updateLabels(labels, cfg) + testutil.CheckErrorAndDeepEqual(t, false, nil, expectedLabels, cfg.Labels) +} diff --git a/pkg/util/command_util.go b/pkg/util/command_util.go new file mode 100644 index 000000000..38521231a --- /dev/null +++ b/pkg/util/command_util.go @@ -0,0 +1,163 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "os" + "path/filepath" + "strings" +) + +// ContainsWildcards returns true if any entry in paths contains wildcards +func ContainsWildcards(paths []string) bool { + for _, path := range paths { + if strings.ContainsAny(path, "*?[") { + return true + } + } + return false +} + +// ResolveSources resolves the given sources if the sources contains wildcards +// It returns a map of [src]:[files rooted at src] +func ResolveSources(srcsAndDest instructions.SourcesAndDest, root string) (map[string][]string, error) { + srcs := srcsAndDest[:len(srcsAndDest)-1] + // If sources contain wildcards, we first need to resolve them to actual paths + if ContainsWildcards(srcs) { + logrus.Debugf("Resolving srcs %v...", srcs) + files, err := RelativeFiles("", root) + if err != nil { + return nil, err + } + srcs, err = matchSources(srcs, files) + if err != nil { + return nil, err + } + logrus.Debugf("Resolved sources to %v", srcs) + } + // Now, get a map of [src]:[files rooted at src] + srcMap, err := SourcesToFilesMap(srcs, root) + if err != nil { + return nil, err + } + // Check to make sure the sources are valid + return srcMap, IsSrcsValid(srcsAndDest, srcMap) +} + +// matchSources returns a list of sources that match wildcards +func matchSources(srcs, files []string) ([]string, error) { + var matchedSources []string + for _, src := range srcs { + src = filepath.Clean(src) + for _, file := range files { + matched, err := filepath.Match(src, file) + if err != nil { + return nil, err + } + if matched { + matchedSources = append(matchedSources, file) + } + } + } + return matchedSources, nil +} + +func IsDestDir(path string) bool { + return strings.HasSuffix(path, "/") || path == "." +} + +// DestinationFilepath returns the destination filepath from the build context to the image filesystem +// If source is a file: +// If dest is a dir, copy it to /dest/relpath +// If dest is a file, copy directly to dest +// If source is a dir: +// Assume dest is also a dir, and copy to dest/relpath +// If dest is not an absolute filepath, add /cwd to the beginning +func DestinationFilepath(filename, srcName, dest, cwd, buildcontext string) (string, error) { + fi, err := os.Stat(filepath.Join(buildcontext, filename)) + if err != nil { + return "", err + } + src, err := os.Stat(filepath.Join(buildcontext, srcName)) + if err != nil { + return "", err + } + if src.IsDir() || IsDestDir(dest) { + relPath, err := filepath.Rel(srcName, filename) + if err != nil { + return "", err + } + if relPath == "." && !fi.IsDir() { + relPath = filepath.Base(filename) + } + destPath := filepath.Join(dest, relPath) + if filepath.IsAbs(dest) { + return destPath, nil + } + return filepath.Join(cwd, destPath), nil + } + if filepath.IsAbs(dest) { + return dest, nil + } + return filepath.Join(cwd, dest), nil +} + +// SourcesToFilesMap returns a map of [src]:[files rooted at source] +func SourcesToFilesMap(srcs []string, root string) (map[string][]string, error) { + srcMap := make(map[string][]string) + for _, src := range srcs { + src = filepath.Clean(src) + files, err := RelativeFiles(src, root) + if err != nil { + return nil, err + } + srcMap[src] = files + } + return srcMap, nil +} + +// IsSrcsValid returns an error if the sources provided are invalid, or nil otherwise +func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, srcMap map[string][]string) error { + srcs := srcsAndDest[:len(srcsAndDest)-1] + dest := srcsAndDest[len(srcsAndDest)-1] + + totalFiles := 0 + for _, files := range srcMap { + totalFiles += len(files) + } + if totalFiles == 0 { + return errors.New("copy failed: no source files specified") + } + + if !ContainsWildcards(srcs) { + // If multiple sources and destination isn't a directory, return an error + if len(srcs) > 1 && !IsDestDir(dest) { + return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'") + } + return nil + } + + // If there are wildcards, and the destination is a file, there must be exactly one file to copy over, + // Otherwise, return an error + if !IsDestDir(dest) && totalFiles > 1 { + return errors.New("when specifying multiple sources in a COPY command, destination must be a directory and end in '/'") + } + return nil +} diff --git a/pkg/util/command_util_test.go b/pkg/util/command_util_test.go new file mode 100644 index 000000000..2d96f8d2a --- /dev/null +++ b/pkg/util/command_util_test.go @@ -0,0 +1,323 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/testutil" + "sort" + "testing" +) + +var buildContextPath = "../../integration_tests/" + +var destinationFilepathTests = []struct { + srcName string + filename string + dest string + cwd string + buildcontext string + expectedFilepath string +}{ + { + srcName: "context/foo", + filename: "context/foo", + dest: "/foo", + cwd: "/", + expectedFilepath: "/foo", + }, + { + srcName: "context/foo", + filename: "context/foo", + dest: "/foodir/", + cwd: "/", + expectedFilepath: "/foodir/foo", + }, + { + srcName: "context/foo", + filename: "./context/foo", + cwd: "/", + dest: "foo", + expectedFilepath: "/foo", + }, + { + srcName: "context/bar/", + filename: "context/bar/bam/bat", + cwd: "/", + dest: "pkg/", + expectedFilepath: "/pkg/bam/bat", + }, + { + srcName: "context/bar/", + filename: "context/bar/bam/bat", + cwd: "/newdir", + dest: "pkg/", + expectedFilepath: "/newdir/pkg/bam/bat", + }, + { + srcName: "./context/empty", + filename: "context/empty", + cwd: "/", + dest: "/empty", + expectedFilepath: "/empty", + }, + { + srcName: "./context/empty", + filename: "context/empty", + cwd: "/dir", + dest: "/empty", + expectedFilepath: "/empty", + }, + { + srcName: "./", + filename: "./", + cwd: "/", + dest: "/dir", + expectedFilepath: "/dir", + }, + { + srcName: "./", + filename: "context/foo", + cwd: "/", + dest: "/dir", + expectedFilepath: "/dir/context/foo", + }, + { + srcName: ".", + filename: "context/bar", + cwd: "/", + dest: "/dir", + expectedFilepath: "/dir/context/bar", + }, + { + srcName: ".", + filename: "context/bar", + cwd: "/", + dest: "/dir", + expectedFilepath: "/dir/context/bar", + }, + { + srcName: "context/foo", + filename: "context/foo", + cwd: "/test", + dest: ".", + expectedFilepath: "/test/foo", + }, +} + +func Test_DestinationFilepath(t *testing.T) { + for _, test := range destinationFilepathTests { + actualFilepath, err := DestinationFilepath(test.filename, test.srcName, test.dest, test.cwd, buildContextPath) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFilepath, actualFilepath) + } +} + +var matchSourcesTests = []struct { + srcs []string + files []string + expectedFiles []string +}{ + { + srcs: []string{ + "pkg/*", + }, + files: []string{ + "pkg/a", + "pkg/b", + "/pkg/d", + "pkg/b/d/", + "dir/", + }, + expectedFiles: []string{ + "pkg/a", + "pkg/b", + }, + }, +} + +func Test_MatchSources(t *testing.T) { + for _, test := range matchSourcesTests { + actualFiles, err := matchSources(test.srcs, test.files) + sort.Strings(actualFiles) + sort.Strings(test.expectedFiles) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFiles, actualFiles) + } +} + +var isSrcValidTests = []struct { + srcsAndDest []string + files map[string][]string + shouldErr bool +}{ + { + srcsAndDest: []string{ + "src1", + "src2", + "dest", + }, + files: map[string][]string{ + "src1": { + "file1", + }, + "src2:": { + "file2", + }, + }, + shouldErr: true, + }, + { + srcsAndDest: []string{ + "src1", + "src2", + "dest/", + }, + files: map[string][]string{ + "src1": { + "file1", + }, + "src2:": { + "file2", + }, + }, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2/", + "dest", + }, + files: map[string][]string{ + "src1": { + "file1", + }, + "src2:": { + "file2", + }, + }, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2", + "dest", + }, + files: map[string][]string{ + "src1": { + "file1", + }, + "src2:": { + "file2", + }, + }, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2", + "src*", + "dest/", + }, + files: map[string][]string{ + "src1": { + "file1", + }, + "src2:": { + "file2", + }, + }, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2", + "src*", + "dest", + }, + files: map[string][]string{ + "src2": { + "src2/a", + "src2/b", + }, + "src*": {}, + }, + shouldErr: true, + }, + { + srcsAndDest: []string{ + "src2", + "src*", + "dest", + }, + files: map[string][]string{ + "src2": { + "src2/a", + }, + "src*": {}, + }, + shouldErr: false, + }, + { + srcsAndDest: []string{ + "src2", + "src*", + "dest", + }, + files: map[string][]string{ + "src2": {}, + "src*": {}, + }, + shouldErr: true, + }, +} + +func Test_IsSrcsValid(t *testing.T) { + for _, test := range isSrcValidTests { + err := IsSrcsValid(test.srcsAndDest, test.files) + testutil.CheckError(t, test.shouldErr, err) + } +} + +var testResolveSources = []struct { + srcsAndDest []string + expectedMap map[string][]string +}{ + { + srcsAndDest: []string{ + "context/foo", + "context/b*", + "dest/", + }, + expectedMap: map[string][]string{ + "context/foo": { + "context/foo", + }, + "context/bar": { + "context/bar", + "context/bar/bam", + "context/bar/bam/bat", + "context/bar/bat", + "context/bar/baz", + }, + }, + }, +} + +func Test_ResolveSources(t *testing.T) { + for _, test := range testResolveSources { + actualMap, err := ResolveSources(test.srcsAndDest, buildContextPath) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedMap, actualMap) + } +} diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index eafd18916..c0ba9008e 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -97,3 +97,48 @@ func fileSystemWhitelist(path string) ([]string, error) { } return whitelist, nil } + +// RelativeFiles returns a list of all files at the filepath relative to root +func RelativeFiles(fp string, root string) ([]string, error) { + var files []string + fullPath := filepath.Join(root, fp) + logrus.Debugf("Getting files and contents at root %s", fullPath) + err := filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(root, path) + if err != nil { + return err + } + files = append(files, relPath) + return nil + }) + return files, err +} + +// FilepathExists returns true if the path exists +func FilepathExists(path string) bool { + _, err := os.Stat(path) + return !os.IsNotExist(err) +} + +// CreateFile creates a file at path and copies over contents from the reader +func CreateFile(path string, reader io.Reader, perm os.FileMode) error { + // Create directory path if it doesn't exist + baseDir := filepath.Dir(path) + if _, err := os.Stat(baseDir); os.IsNotExist(err) { + logrus.Debugf("baseDir %s for file %s does not exist. Creating.", baseDir, path) + if err := os.MkdirAll(baseDir, 0755); err != nil { + return err + } + } + dest, err := os.Create(path) + if err != nil { + return err + } + if _, err := io.Copy(dest, reader); err != nil { + return err + } + return dest.Chmod(perm) +} diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index ccb7b3658..c8ebc88cf 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -51,3 +51,82 @@ func Test_fileSystemWhitelist(t *testing.T) { sort.Strings(expectedWhitelist) testutil.CheckErrorAndDeepEqual(t, false, err, expectedWhitelist, actualWhitelist) } + +var tests = []struct { + files map[string]string + directory string + expectedFiles []string +}{ + { + files: map[string]string{ + "/workspace/foo/a": "baz1", + "/workspace/foo/b": "baz2", + "/kbuild/file": "file", + }, + directory: "/workspace/foo/", + expectedFiles: []string{ + "workspace/foo/a", + "workspace/foo/b", + "workspace/foo", + }, + }, + { + files: map[string]string{ + "/workspace/foo/a": "baz1", + }, + directory: "/workspace/foo/a", + expectedFiles: []string{ + "workspace/foo/a", + }, + }, + { + files: map[string]string{ + "/workspace/foo/a": "baz1", + "/workspace/foo/b": "baz2", + "/workspace/baz": "hey", + "/kbuild/file": "file", + }, + directory: "/workspace", + expectedFiles: []string{ + "workspace/foo/a", + "workspace/foo/b", + "workspace/baz", + "workspace", + "workspace/foo", + }, + }, + { + files: map[string]string{ + "/workspace/foo/a": "baz1", + "/workspace/foo/b": "baz2", + "/kbuild/file": "file", + }, + directory: "", + expectedFiles: []string{ + "workspace/foo/a", + "workspace/foo/b", + "kbuild/file", + "workspace", + "workspace/foo", + "kbuild", + ".", + }, + }, +} + +func Test_RelativeFiles(t *testing.T) { + for _, test := range tests { + testDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("err setting up temp dir: %v", err) + } + defer os.RemoveAll(testDir) + if err := testutil.SetupFiles(testDir, test.files); err != nil { + t.Fatalf("err setting up files: %v", err) + } + actualFiles, err := RelativeFiles(test.directory, testDir) + sort.Strings(actualFiles) + sort.Strings(test.expectedFiles) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFiles, actualFiles) + } +} diff --git a/testutil/util.go b/testutil/util.go index 45607a772..64819c3ef 100644 --- a/testutil/util.go +++ b/testutil/util.go @@ -50,6 +50,12 @@ func CheckErrorAndDeepEqual(t *testing.T, shouldErr bool, err error, expected, a } } +func CheckError(t *testing.T, shouldErr bool, err error) { + if err := checkErr(shouldErr, err); err != nil { + t.Error(err) + } +} + func checkErr(shouldErr bool, err error) error { if err == nil && shouldErr { return fmt.Errorf("Expected error, but returned none")