From 89400b7410a0ae94ffac032a5461c7a129537afd Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 28 Mar 2018 13:34:00 -0700 Subject: [PATCH] Add command with unit tests --- pkg/commands/add.go | 140 ++++++++++++++++++++++++++++++++++ pkg/commands/commands.go | 2 + pkg/util/command_util.go | 45 ++++++++++- pkg/util/command_util_test.go | 67 ++++++++++++++++ pkg/util/fs_util.go | 37 +++++++++ pkg/util/tar_util.go | 86 +++++++++++++++++++++ pkg/util/tar_util_test.go | 108 ++++++++++++++++++++++++++ 7 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 pkg/commands/add.go create mode 100644 pkg/util/tar_util_test.go diff --git a/pkg/commands/add.go b/pkg/commands/add.go new file mode 100644 index 000000000..524018c72 --- /dev/null +++ b/pkg/commands/add.go @@ -0,0 +1,140 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util" + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/sirupsen/logrus" + "path/filepath" + "strings" +) + +type AddCommand struct { + cmd *instructions.AddCommand + buildcontext string + snapshotFiles []string +} + +// ExecuteCommand executes the ADD command +// Special stuff about ADD: +// 1. If is a remote file URL: +// - destination will have permissions of 0600 +// - If remote file has HTTP Last-Modified header, we set the mtime of the file to that timestamp +// - If dest doesn't end with a slash, the filepath is inferred to be / +// 2. If is a local tar archive: +// -If is a local tar archive, it is unpacked at the dest, as 'tar -x' would +func (a *AddCommand) ExecuteCommand(config *manifest.Schema2Config) error { + srcs := a.cmd.SourcesAndDest[:len(a.cmd.SourcesAndDest)-1] + dest := a.cmd.SourcesAndDest[len(a.cmd.SourcesAndDest)-1] + + logrus.Infof("cmd: Add %s", srcs) + logrus.Infof("dest: %s", dest) + + // First, resolve any environment replacement + resolvedEnvs, err := util.ResolveEnvironmentReplacementList(a.cmd.SourcesAndDest, config.Env, true) + if err != nil { + return err + } + dest = resolvedEnvs[len(resolvedEnvs)-1] + // Get a map of [src]:[files rooted at src] + srcMap, err := util.ResolveSources(resolvedEnvs, a.buildcontext) + if err != nil { + return err + } + // If any of the sources are local tar archives: + // 1. Unpack them to the specified destination + // 2. Remove it as a source that needs to be copied over + // If any of the sources is a remote file URL: + // 1. Download and copy it to the specifed dest + // 2. Remove it as a source that needs to be copied + for src, files := range srcMap { + for _, file := range files { + // If file is a local tar archive, then we unpack it to dest + filePath := filepath.Join(a.buildcontext, file) + isFilenameSource, err := isFilenameSource(srcMap, file) + if err != nil { + return err + } + if util.IsSrcRemoteFileURL(file) { + urlDest := util.URLDestinationFilepath(file, dest, config.WorkingDir) + logrus.Infof("Adding remote URL %s to %s", file, urlDest) + if err := util.DownloadFileToDest(file, urlDest); err != nil { + return err + } + a.snapshotFiles = append(a.snapshotFiles, urlDest) + delete(srcMap, src) + } else if isFilenameSource && util.IsFileLocalTarArchive(filePath) { + logrus.Infof("Unpacking local tar archive %s to %s", file, dest) + if err := util.UnpackLocalTarArchive(filePath, dest); err != nil { + return err + } + // Add the unpacked files to the snapshotter + filesAdded, err := util.Files(dest) + if err != nil { + return err + } + logrus.Debugf("Added %v from local tar archive %s", filesAdded, file) + a.snapshotFiles = append(a.snapshotFiles, filesAdded...) + delete(srcMap, src) + } + } + } + // With the remaining "normal" sources, create and execute a standard copy command + if len(srcMap) == 0 { + return nil + } + var regularSrcs []string + for src := range srcMap { + regularSrcs = append(regularSrcs, src) + } + copyCmd := CopyCommand{ + cmd: &instructions.CopyCommand{ + SourcesAndDest: append(regularSrcs, dest), + }, + buildcontext: a.buildcontext, + } + if err := copyCmd.ExecuteCommand(config); err != nil { + return err + } + a.snapshotFiles = append(a.snapshotFiles, copyCmd.snapshotFiles...) + return nil +} + +func isFilenameSource(srcMap map[string][]string, fileName string) (bool, error) { + for src := range srcMap { + matched, err := filepath.Match(src, fileName) + if err != nil { + return false, err + } + if matched || (src == fileName) { + return true, nil + } + } + return false, nil +} + +// FilesToSnapshot should return an empty array if still nil; no files were changed +func (a *AddCommand) FilesToSnapshot() []string { + return a.snapshotFiles +} + +// CreatedBy returns some information about the command for the image config +func (a *AddCommand) CreatedBy() string { + return strings.Join(a.cmd.SourcesAndDest, " ") +} diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index 08fa3ceb1..5f0b6aedf 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -44,6 +44,8 @@ func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, e return &ExposeCommand{cmd: c}, nil case *instructions.EnvCommand: return &EnvCommand{cmd: c}, nil + case *instructions.AddCommand: + return &AddCommand{cmd: c, buildcontext: buildcontext}, nil case *instructions.CmdCommand: return &CmdCommand{cmd: c}, nil case *instructions.EntrypointCommand: diff --git a/pkg/util/command_util.go b/pkg/util/command_util.go index 0f6472f79..7ecb050e3 100644 --- a/pkg/util/command_util.go +++ b/pkg/util/command_util.go @@ -22,6 +22,8 @@ import ( "github.com/docker/docker/builder/dockerfile/shell" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "net/http" + "net/url" "os" "path/filepath" "strings" @@ -31,6 +33,10 @@ import ( func ResolveEnvironmentReplacementList(values, envs []string, isFilepath bool) ([]string, error) { var resolvedValues []string for _, value := range values { + if IsSrcRemoteFileURL(value) { + resolvedValues = append(resolvedValues, value) + continue + } resolved, err := ResolveEnvironmentReplacement(value, envs, isFilepath) logrus.Debugf("Resolved %s to %s", value, resolved) if err != nil { @@ -106,13 +112,17 @@ func ResolveSources(srcsAndDest instructions.SourcesAndDest, root string) (map[s func matchSources(srcs, files []string) ([]string, error) { var matchedSources []string for _, src := range srcs { + if IsSrcRemoteFileURL(src) { + matchedSources = append(matchedSources, src) + continue + } src = filepath.Clean(src) for _, file := range files { matched, err := filepath.Match(src, file) if err != nil { return nil, err } - if matched { + if matched || src == file { matchedSources = append(matchedSources, file) } } @@ -160,10 +170,31 @@ func DestinationFilepath(filename, srcName, dest, cwd, buildcontext string) (str return filepath.Join(cwd, dest), nil } +// URLDestinationFilepath gives the destintion a file from a remote URL should be saved to +func URLDestinationFilepath(rawurl, dest, cwd string) string { + if !IsDestDir(dest) { + if !filepath.IsAbs(dest) { + return filepath.Join(cwd, dest) + } + return dest + } + urlBase := filepath.Base(rawurl) + destPath := filepath.Join(dest, urlBase) + + if !filepath.IsAbs(dest) { + destPath = filepath.Join(cwd, destPath) + } + return destPath +} + // SourcesToFilesMap returns a map of [src]:[files rooted at source] func SourcesToFilesMap(srcs []string, root string) (map[string][]string, error) { srcMap := make(map[string][]string) for _, src := range srcs { + if IsSrcRemoteFileURL(src) { + srcMap[src] = []string{src} + continue + } src = filepath.Clean(src) files, err := RelativeFiles(src, root) if err != nil { @@ -202,3 +233,15 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, srcMap map[string][]st } return nil } + +func IsSrcRemoteFileURL(rawurl string) bool { + _, err := url.ParseRequestURI(rawurl) + if err != nil { + return false + } + _, err = http.Get(rawurl) + if err != nil { + return false + } + return true +} diff --git a/pkg/util/command_util_test.go b/pkg/util/command_util_test.go index 611ad7b81..afd359d30 100644 --- a/pkg/util/command_util_test.go +++ b/pkg/util/command_util_test.go @@ -22,6 +22,8 @@ import ( "testing" ) +var testUrl = "https://github.com/GoogleCloudPlatform/runtimes-common/blob/master/LICENSE" + var testEnvReplacement = []struct { path string command string @@ -207,6 +209,39 @@ func Test_DestinationFilepath(t *testing.T) { } } +var urlDestFilepathTests = []struct { + url string + cwd string + dest string + expectedDest string +}{ + { + url: "https://something/something", + cwd: "/test", + dest: ".", + expectedDest: "/test/something", + }, + { + url: "https://something/something", + cwd: "/cwd", + dest: "/test", + expectedDest: "/test", + }, + { + url: "https://something/something", + cwd: "/test", + dest: "/dest/", + expectedDest: "/dest/something", + }, +} + +func Test_UrlDestFilepath(t *testing.T) { + for _, test := range urlDestFilepathTests { + actualDest := URLDestinationFilepath(test.url, test.dest, test.cwd) + testutil.CheckErrorAndDeepEqual(t, false, nil, test.expectedDest, actualDest) + } +} + var matchSourcesTests = []struct { srcs []string files []string @@ -215,6 +250,7 @@ var matchSourcesTests = []struct { { srcs: []string{ "pkg/*", + testUrl, }, files: []string{ "pkg/a", @@ -226,6 +262,7 @@ var matchSourcesTests = []struct { expectedFiles: []string{ "pkg/a", "pkg/b", + testUrl, }, }, } @@ -380,6 +417,7 @@ var testResolveSources = []struct { srcsAndDest: []string{ "context/foo", "context/b*", + testUrl, "dest/", }, expectedMap: map[string][]string{ @@ -393,6 +431,9 @@ var testResolveSources = []struct { "context/bar/bat", "context/bar/baz", }, + testUrl: { + testUrl, + }, }, }, } @@ -403,3 +444,29 @@ func Test_ResolveSources(t *testing.T) { testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedMap, actualMap) } } + +var testRemoteUrls = []struct { + url string + valid bool +}{ + { + url: testUrl, + valid: true, + }, + { + url: "not/real/", + valid: false, + }, + { + url: "https://url.com/something/not/real", + valid: false, + }, +} + +func Test_RemoteUrls(t *testing.T) { + for _, test := range testRemoteUrls { + valid := IsSrcRemoteFileURL(test.url) + testutil.CheckErrorAndDeepEqual(t, false, nil, test.valid, valid) + } + +} diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index c0ba9008e..b3b020c15 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -23,9 +23,11 @@ import ( "github.com/containers/image/docker" "github.com/sirupsen/logrus" "io" + "net/http" "os" "path/filepath" "strings" + "time" ) var whitelist = []string{"/kbuild"} @@ -117,6 +119,17 @@ func RelativeFiles(fp string, root string) ([]string, error) { return files, err } +// Files returns a list of all files rooted at root +func Files(root string) ([]string, error) { + var files []string + logrus.Debugf("Getting files and contents at root %s", root) + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + files = append(files, path) + return err + }) + return files, err +} + // FilepathExists returns true if the path exists func FilepathExists(path string) bool { _, err := os.Stat(path) @@ -142,3 +155,27 @@ func CreateFile(path string, reader io.Reader, perm os.FileMode) error { } return dest.Chmod(perm) } + +// DownloadFileToDest downloads the file at rawurl to the given dest for the ADD command +// From add command docs: +// 1. If is a remote file URL: +// - destination will have permissions of 0600 +// - If remote file has HTTP Last-Modified header, we set the mtime of the file to that timestamp +func DownloadFileToDest(rawurl, dest string) error { + resp, err := http.Get(rawurl) + if err != nil { + return err + } + defer resp.Body.Close() + if err := CreateFile(dest, resp.Body, 0600); err != nil { + return err + } + mTime := time.Time{} + lastMod := resp.Header.Get("Last-Modified") + if lastMod != "" { + if parsedMTime, err := http.ParseTime(lastMod); err == nil { + mTime = parsedMTime + } + } + return os.Chtimes(dest, mTime, mTime) +} diff --git a/pkg/util/tar_util.go b/pkg/util/tar_util.go index 417f16f04..8c0ffdfb9 100644 --- a/pkg/util/tar_util.go +++ b/pkg/util/tar_util.go @@ -18,8 +18,14 @@ package util import ( "archive/tar" + "compress/bzip2" + "compress/gzip" + pkgutil "github.com/GoogleCloudPlatform/container-diff/pkg/util" + "github.com/docker/docker/pkg/archive" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "io" + "io/ioutil" "os" "syscall" ) @@ -86,3 +92,83 @@ func checkHardlink(p string, i os.FileInfo) (bool, string) { } return hardlink, linkDst } + +//UnpackLocalTarArchive unpacks the tar archive at path to the directory dest +// Returns true if the path was acutally unpacked +func UnpackLocalTarArchive(path, dest string) error { + // First, we need to check if the path is a local tar archive + if compressed, compressionLevel := fileIsCompressedTar(path); compressed { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + if compressionLevel == archive.Gzip { + gzr, err := gzip.NewReader(file) + if err != nil { + return err + } + defer gzr.Close() + return pkgutil.UnTar(gzr, dest, nil) + } else if compressionLevel == archive.Bzip2 { + bzr := bzip2.NewReader(file) + return pkgutil.UnTar(bzr, dest, nil) + } + } + if fileIsUncompressedTar(path) { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + return pkgutil.UnTar(file, dest, nil) + } + return errors.New("path does not lead to local tar archive") +} + +//IsFileLocalTarArchive returns true if the file is a local tar archive +func IsFileLocalTarArchive(src string) bool { + compressed, _ := fileIsCompressedTar(src) + uncompressed := fileIsUncompressedTar(src) + return compressed || uncompressed +} + +func fileIsCompressedTar(src string) (bool, archive.Compression) { + r, err := os.Open(src) + if err != nil { + return false, -1 + } + defer r.Close() + buf, err := ioutil.ReadAll(r) + if err != nil { + return false, -1 + } + compressionLevel := archive.DetectCompression(buf) + return (compressionLevel > 0), compressionLevel +} + +func fileIsUncompressedTar(src string) bool { + r, err := os.Open(src) + defer r.Close() + if err != nil { + return false + } + fi, err := os.Stat(src) + if err != nil { + return false + } + if fi.Size() == 0 { + return false + } + tr := tar.NewReader(r) + if tr == nil { + return false + } + for { + _, err := tr.Next() + if err != nil { + return false + } + return true + } +} diff --git a/pkg/util/tar_util_test.go b/pkg/util/tar_util_test.go new file mode 100644 index 000000000..9974f6c21 --- /dev/null +++ b/pkg/util/tar_util_test.go @@ -0,0 +1,108 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "archive/tar" + "compress/gzip" + "github.com/GoogleCloudPlatform/k8s-container-builder/testutil" + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +var regularFiles = []string{"file", "file.tar", "file.tar.gz"} +var uncompressedTars = []string{"uncompressed", "uncompressed.tar"} +var compressedTars = []string{"compressed", "compressed.tar.gz"} + +func Test_IsLocalTarArchive(t *testing.T) { + testDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("err setting up temp dir: %v", err) + } + defer os.RemoveAll(testDir) + if err := setUpFilesAndTars(testDir); err != nil { + t.Fatal(err) + } + // Test we get the correct result for regular files + for _, regularFile := range regularFiles { + isTarArchive := IsFileLocalTarArchive(filepath.Join(testDir, regularFile)) + testutil.CheckErrorAndDeepEqual(t, false, nil, false, isTarArchive) + } + // Test we get the correct result for uncompressed tars + for _, uncompressedTar := range uncompressedTars { + isTarArchive := IsFileLocalTarArchive(filepath.Join(testDir, uncompressedTar)) + testutil.CheckErrorAndDeepEqual(t, false, nil, true, isTarArchive) + } + // Test we get the correct result for compressed tars + for _, compressedTar := range compressedTars { + isTarArchive := IsFileLocalTarArchive(filepath.Join(testDir, compressedTar)) + testutil.CheckErrorAndDeepEqual(t, false, nil, true, isTarArchive) + } +} + +func setUpFilesAndTars(testDir string) error { + regularFilesAndContents := map[string]string{ + regularFiles[0]: "", + regularFiles[1]: "something", + regularFiles[2]: "here", + } + if err := testutil.SetupFiles(testDir, regularFilesAndContents); err != nil { + return err + } + + for _, uncompressedTar := range uncompressedTars { + tarFile, err := os.Create(filepath.Join(testDir, uncompressedTar)) + if err != nil { + return err + } + if err := createTar(testDir, tarFile); err != nil { + return err + } + } + + for _, compressedTar := range compressedTars { + tarFile, err := os.Create(filepath.Join(testDir, compressedTar)) + if err != nil { + return err + } + gzr := gzip.NewWriter(tarFile) + if err := createTar(testDir, gzr); err != nil { + return err + } + } + return nil +} + +func createTar(testdir string, writer io.Writer) error { + + w := tar.NewWriter(writer) + defer w.Close() + for _, regFile := range regularFiles { + filePath := filepath.Join(testdir, regFile) + fi, err := os.Stat(filePath) + if err != nil { + return err + } + if err := AddToTar(filePath, fi, w); err != nil { + return err + } + } + return nil +}