Add command with unit tests

pkg/commands/add.go

@@ -0,0 +1,140 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util"
+	"github.com/containers/image/manifest"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/sirupsen/logrus"
+	"path/filepath"
+	"strings"
+)
+
+type AddCommand struct {
+	cmd           *instructions.AddCommand
+	buildcontext  string
+	snapshotFiles []string
+}
+
+// ExecuteCommand executes the ADD command
+// Special stuff about ADD:
+// 	1. If <src> is a remote file URL:
+// 		- destination will have permissions of 0600
+// 		- If remote file has HTTP Last-Modified header, we set the mtime of the file to that timestamp
+// 		- If dest doesn't end with a slash, the filepath is inferred to be <dest>/<filename>
+// 	2. If <src> is a local tar archive:
+// 		-If <src> is a local tar archive, it is unpacked at the dest, as 'tar -x' would
+func (a *AddCommand) ExecuteCommand(config *manifest.Schema2Config) error {
+	srcs := a.cmd.SourcesAndDest[:len(a.cmd.SourcesAndDest)-1]
+	dest := a.cmd.SourcesAndDest[len(a.cmd.SourcesAndDest)-1]
+
+	logrus.Infof("cmd: Add %s", srcs)
+	logrus.Infof("dest: %s", dest)
+
+	// First, resolve any environment replacement
+	resolvedEnvs, err := util.ResolveEnvironmentReplacementList(a.cmd.SourcesAndDest, config.Env, true)
+	if err != nil {
+		return err
+	}
+	dest = resolvedEnvs[len(resolvedEnvs)-1]
+	// Get a map of [src]:[files rooted at src]
+	srcMap, err := util.ResolveSources(resolvedEnvs, a.buildcontext)
+	if err != nil {
+		return err
+	}
+	// If any of the sources are local tar archives:
+	// 	1. Unpack them to the specified destination
+	// 	2. Remove it as a source that needs to be copied over
+	// If any of the sources is a remote file URL:
+	//	1. Download and copy it to the specifed dest
+	//  2. Remove it as a source that needs to be copied
+	for src, files := range srcMap {
+		for _, file := range files {
+			// If file is a local tar archive, then we unpack it to dest
+			filePath := filepath.Join(a.buildcontext, file)
+			isFilenameSource, err := isFilenameSource(srcMap, file)
+			if err != nil {
+				return err
+			}
+			if util.IsSrcRemoteFileURL(file) {
+				urlDest := util.URLDestinationFilepath(file, dest, config.WorkingDir)
+				logrus.Infof("Adding remote URL %s to %s", file, urlDest)
+				if err := util.DownloadFileToDest(file, urlDest); err != nil {
+					return err
+				}
+				a.snapshotFiles = append(a.snapshotFiles, urlDest)
+				delete(srcMap, src)
+			} else if isFilenameSource && util.IsFileLocalTarArchive(filePath) {
+				logrus.Infof("Unpacking local tar archive %s to %s", file, dest)
+				if err := util.UnpackLocalTarArchive(filePath, dest); err != nil {
+					return err
+				}
+				// Add the unpacked files to the snapshotter
+				filesAdded, err := util.Files(dest)
+				if err != nil {
+					return err
+				}
+				logrus.Debugf("Added %v from local tar archive %s", filesAdded, file)
+				a.snapshotFiles = append(a.snapshotFiles, filesAdded...)
+				delete(srcMap, src)
+			}
+		}
+	}
+	// With the remaining "normal" sources, create and execute a standard copy command
+	if len(srcMap) == 0 {
+		return nil
+	}
+	var regularSrcs []string
+	for src := range srcMap {
+		regularSrcs = append(regularSrcs, src)
+	}
+	copyCmd := CopyCommand{
+		cmd: &instructions.CopyCommand{
+			SourcesAndDest: append(regularSrcs, dest),
+		},
+		buildcontext: a.buildcontext,
+	}
+	if err := copyCmd.ExecuteCommand(config); err != nil {
+		return err
+	}
+	a.snapshotFiles = append(a.snapshotFiles, copyCmd.snapshotFiles...)
+	return nil
+}
+
+func isFilenameSource(srcMap map[string][]string, fileName string) (bool, error) {
+	for src := range srcMap {
+		matched, err := filepath.Match(src, fileName)
+		if err != nil {
+			return false, err
+		}
+		if matched || (src == fileName) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// FilesToSnapshot should return an empty array if still nil; no files were changed
+func (a *AddCommand) FilesToSnapshot() []string {
+	return a.snapshotFiles
+}
+
+// CreatedBy returns some information about the command for the image config
+func (a *AddCommand) CreatedBy() string {
+	return strings.Join(a.cmd.SourcesAndDest, " ")
+}

pkg/commands/commands.go

@@ -44,6 +44,8 @@ func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, e
 		return &ExposeCommand{cmd: c}, nil
 	case *instructions.EnvCommand:
 		return &EnvCommand{cmd: c}, nil
+	case *instructions.AddCommand:
+		return &AddCommand{cmd: c, buildcontext: buildcontext}, nil
 	case *instructions.CmdCommand:
 		return &CmdCommand{cmd: c}, nil
 	case *instructions.EntrypointCommand:

pkg/util/command_util.go

@@ -22,6 +22,8 @@ import (
 	"github.com/docker/docker/builder/dockerfile/shell"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -31,6 +33,10 @@ import (
 func ResolveEnvironmentReplacementList(values, envs []string, isFilepath bool) ([]string, error) {
 	var resolvedValues []string
 	for _, value := range values {
+		if IsSrcRemoteFileURL(value) {
+			resolvedValues = append(resolvedValues, value)
+			continue
+		}
 		resolved, err := ResolveEnvironmentReplacement(value, envs, isFilepath)
 		logrus.Debugf("Resolved %s to %s", value, resolved)
 		if err != nil {
@@ -106,13 +112,17 @@ func ResolveSources(srcsAndDest instructions.SourcesAndDest, root string) (map[s
 func matchSources(srcs, files []string) ([]string, error) {
 	var matchedSources []string
 	for _, src := range srcs {
+		if IsSrcRemoteFileURL(src) {
+			matchedSources = append(matchedSources, src)
+			continue
+		}
 		src = filepath.Clean(src)
 		for _, file := range files {
 			matched, err := filepath.Match(src, file)
 			if err != nil {
 				return nil, err
 			}
-			if matched {
+			if matched || src == file {
 				matchedSources = append(matchedSources, file)
 			}
 		}
@@ -160,10 +170,31 @@ func DestinationFilepath(filename, srcName, dest, cwd, buildcontext string) (str
 	return filepath.Join(cwd, dest), nil
 }
 
+// URLDestinationFilepath gives the destintion a file from a remote URL should be saved to
+func URLDestinationFilepath(rawurl, dest, cwd string) string {
+	if !IsDestDir(dest) {
+		if !filepath.IsAbs(dest) {
+			return filepath.Join(cwd, dest)
+		}
+		return dest
+	}
+	urlBase := filepath.Base(rawurl)
+	destPath := filepath.Join(dest, urlBase)
+
+	if !filepath.IsAbs(dest) {
+		destPath = filepath.Join(cwd, destPath)
+	}
+	return destPath
+}
+
 // SourcesToFilesMap returns a map of [src]:[files rooted at source]
 func SourcesToFilesMap(srcs []string, root string) (map[string][]string, error) {
 	srcMap := make(map[string][]string)
 	for _, src := range srcs {
+		if IsSrcRemoteFileURL(src) {
+			srcMap[src] = []string{src}
+			continue
+		}
 		src = filepath.Clean(src)
 		files, err := RelativeFiles(src, root)
 		if err != nil {
@@ -202,3 +233,15 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, srcMap map[string][]st
 	}
 	return nil
 }
+
+func IsSrcRemoteFileURL(rawurl string) bool {
+	_, err := url.ParseRequestURI(rawurl)
+	if err != nil {
+		return false
+	}
+	_, err = http.Get(rawurl)
+	if err != nil {
+		return false
+	}
+	return true
+}

pkg/util/command_util_test.go

@@ -22,6 +22,8 @@ import (
 	"testing"
 )
 
+var testUrl = "https://github.com/GoogleCloudPlatform/runtimes-common/blob/master/LICENSE"
+
 var testEnvReplacement = []struct {
 	path         string
 	command      string
@@ -207,6 +209,39 @@ func Test_DestinationFilepath(t *testing.T) {
 	}
 }
 
+var urlDestFilepathTests = []struct {
+	url          string
+	cwd          string
+	dest         string
+	expectedDest string
+}{
+	{
+		url:          "https://something/something",
+		cwd:          "/test",
+		dest:         ".",
+		expectedDest: "/test/something",
+	},
+	{
+		url:          "https://something/something",
+		cwd:          "/cwd",
+		dest:         "/test",
+		expectedDest: "/test",
+	},
+	{
+		url:          "https://something/something",
+		cwd:          "/test",
+		dest:         "/dest/",
+		expectedDest: "/dest/something",
+	},
+}
+
+func Test_UrlDestFilepath(t *testing.T) {
+	for _, test := range urlDestFilepathTests {
+		actualDest := URLDestinationFilepath(test.url, test.dest, test.cwd)
+		testutil.CheckErrorAndDeepEqual(t, false, nil, test.expectedDest, actualDest)
+	}
+}
+
 var matchSourcesTests = []struct {
 	srcs          []string
 	files         []string
@@ -215,6 +250,7 @@ var matchSourcesTests = []struct {
 	{
 		srcs: []string{
 			"pkg/*",
+			testUrl,
 		},
 		files: []string{
 			"pkg/a",
@@ -226,6 +262,7 @@ var matchSourcesTests = []struct {
 		expectedFiles: []string{
 			"pkg/a",
 			"pkg/b",
+			testUrl,
 		},
 	},
 }
@@ -380,6 +417,7 @@ var testResolveSources = []struct {
 		srcsAndDest: []string{
 			"context/foo",
 			"context/b*",
+			testUrl,
 			"dest/",
 		},
 		expectedMap: map[string][]string{
@@ -393,6 +431,9 @@ var testResolveSources = []struct {
 				"context/bar/bat",
 				"context/bar/baz",
 			},
+			testUrl: {
+				testUrl,
+			},
 		},
 	},
 }
@@ -403,3 +444,29 @@ func Test_ResolveSources(t *testing.T) {
 		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedMap, actualMap)
 	}
 }
+
+var testRemoteUrls = []struct {
+	url   string
+	valid bool
+}{
+	{
+		url:   testUrl,
+		valid: true,
+	},
+	{
+		url:   "not/real/",
+		valid: false,
+	},
+	{
+		url:   "https://url.com/something/not/real",
+		valid: false,
+	},
+}
+
+func Test_RemoteUrls(t *testing.T) {
+	for _, test := range testRemoteUrls {
+		valid := IsSrcRemoteFileURL(test.url)
+		testutil.CheckErrorAndDeepEqual(t, false, nil, test.valid, valid)
+	}
+
+}

pkg/util/fs_util.go

@@ -23,9 +23,11 @@ import (
 	"github.com/containers/image/docker"
 	"github.com/sirupsen/logrus"
 	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 )
 
 var whitelist = []string{"/kbuild"}
@@ -117,6 +119,17 @@ func RelativeFiles(fp string, root string) ([]string, error) {
 	return files, err
 }
 
+// Files returns a list of all files rooted at root
+func Files(root string) ([]string, error) {
+	var files []string
+	logrus.Debugf("Getting files and contents at root %s", root)
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		files = append(files, path)
+		return err
+	})
+	return files, err
+}
+
 // FilepathExists returns true if the path exists
 func FilepathExists(path string) bool {
 	_, err := os.Stat(path)
@@ -142,3 +155,27 @@ func CreateFile(path string, reader io.Reader, perm os.FileMode) error {
 	}
 	return dest.Chmod(perm)
 }
+
+// DownloadFileToDest downloads the file at rawurl to the given dest for the ADD command
+// From add command docs:
+// 	1. If <src> is a remote file URL:
+// 		- destination will have permissions of 0600
+// 		- If remote file has HTTP Last-Modified header, we set the mtime of the file to that timestamp
+func DownloadFileToDest(rawurl, dest string) error {
+	resp, err := http.Get(rawurl)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if err := CreateFile(dest, resp.Body, 0600); err != nil {
+		return err
+	}
+	mTime := time.Time{}
+	lastMod := resp.Header.Get("Last-Modified")
+	if lastMod != "" {
+		if parsedMTime, err := http.ParseTime(lastMod); err == nil {
+			mTime = parsedMTime
+		}
+	}
+	return os.Chtimes(dest, mTime, mTime)
+}

pkg/util/tar_util.go

@@ -18,8 +18,14 @@ package util
 
 import (
 	"archive/tar"
+	"compress/bzip2"
+	"compress/gzip"
+	pkgutil "github.com/GoogleCloudPlatform/container-diff/pkg/util"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"io"
+	"io/ioutil"
 	"os"
 	"syscall"
 )
@@ -86,3 +92,83 @@ func checkHardlink(p string, i os.FileInfo) (bool, string) {
 	}
 	return hardlink, linkDst
 }
+
+//UnpackLocalTarArchive unpacks the tar archive at path to the directory dest
+// Returns true if the path was acutally unpacked
+func UnpackLocalTarArchive(path, dest string) error {
+	// First, we need to check if the path is a local tar archive
+	if compressed, compressionLevel := fileIsCompressedTar(path); compressed {
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+		if compressionLevel == archive.Gzip {
+			gzr, err := gzip.NewReader(file)
+			if err != nil {
+				return err
+			}
+			defer gzr.Close()
+			return pkgutil.UnTar(gzr, dest, nil)
+		} else if compressionLevel == archive.Bzip2 {
+			bzr := bzip2.NewReader(file)
+			return pkgutil.UnTar(bzr, dest, nil)
+		}
+	}
+	if fileIsUncompressedTar(path) {
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+		return pkgutil.UnTar(file, dest, nil)
+	}
+	return errors.New("path does not lead to local tar archive")
+}
+
+//IsFileLocalTarArchive returns true if the file is a local tar archive
+func IsFileLocalTarArchive(src string) bool {
+	compressed, _ := fileIsCompressedTar(src)
+	uncompressed := fileIsUncompressedTar(src)
+	return compressed || uncompressed
+}
+
+func fileIsCompressedTar(src string) (bool, archive.Compression) {
+	r, err := os.Open(src)
+	if err != nil {
+		return false, -1
+	}
+	defer r.Close()
+	buf, err := ioutil.ReadAll(r)
+	if err != nil {
+		return false, -1
+	}
+	compressionLevel := archive.DetectCompression(buf)
+	return (compressionLevel > 0), compressionLevel
+}
+
+func fileIsUncompressedTar(src string) bool {
+	r, err := os.Open(src)
+	defer r.Close()
+	if err != nil {
+		return false
+	}
+	fi, err := os.Stat(src)
+	if err != nil {
+		return false
+	}
+	if fi.Size() == 0 {
+		return false
+	}
+	tr := tar.NewReader(r)
+	if tr == nil {
+		return false
+	}
+	for {
+		_, err := tr.Next()
+		if err != nil {
+			return false
+		}
+		return true
+	}
+}

pkg/util/tar_util_test.go

@@ -0,0 +1,108 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"github.com/GoogleCloudPlatform/k8s-container-builder/testutil"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+var regularFiles = []string{"file", "file.tar", "file.tar.gz"}
+var uncompressedTars = []string{"uncompressed", "uncompressed.tar"}
+var compressedTars = []string{"compressed", "compressed.tar.gz"}
+
+func Test_IsLocalTarArchive(t *testing.T) {
+	testDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatalf("err setting up temp dir: %v", err)
+	}
+	defer os.RemoveAll(testDir)
+	if err := setUpFilesAndTars(testDir); err != nil {
+		t.Fatal(err)
+	}
+	// Test we get the correct result for regular files
+	for _, regularFile := range regularFiles {
+		isTarArchive := IsFileLocalTarArchive(filepath.Join(testDir, regularFile))
+		testutil.CheckErrorAndDeepEqual(t, false, nil, false, isTarArchive)
+	}
+	// Test we get the correct result for uncompressed tars
+	for _, uncompressedTar := range uncompressedTars {
+		isTarArchive := IsFileLocalTarArchive(filepath.Join(testDir, uncompressedTar))
+		testutil.CheckErrorAndDeepEqual(t, false, nil, true, isTarArchive)
+	}
+	// Test we get the correct result for compressed tars
+	for _, compressedTar := range compressedTars {
+		isTarArchive := IsFileLocalTarArchive(filepath.Join(testDir, compressedTar))
+		testutil.CheckErrorAndDeepEqual(t, false, nil, true, isTarArchive)
+	}
+}
+
+func setUpFilesAndTars(testDir string) error {
+	regularFilesAndContents := map[string]string{
+		regularFiles[0]: "",
+		regularFiles[1]: "something",
+		regularFiles[2]: "here",
+	}
+	if err := testutil.SetupFiles(testDir, regularFilesAndContents); err != nil {
+		return err
+	}
+
+	for _, uncompressedTar := range uncompressedTars {
+		tarFile, err := os.Create(filepath.Join(testDir, uncompressedTar))
+		if err != nil {
+			return err
+		}
+		if err := createTar(testDir, tarFile); err != nil {
+			return err
+		}
+	}
+
+	for _, compressedTar := range compressedTars {
+		tarFile, err := os.Create(filepath.Join(testDir, compressedTar))
+		if err != nil {
+			return err
+		}
+		gzr := gzip.NewWriter(tarFile)
+		if err := createTar(testDir, gzr); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func createTar(testdir string, writer io.Writer) error {
+
+	w := tar.NewWriter(writer)
+	defer w.Close()
+	for _, regFile := range regularFiles {
+		filePath := filepath.Join(testdir, regFile)
+		fi, err := os.Stat(filePath)
+		if err != nil {
+			return err
+		}
+		if err := AddToTar(filePath, fi, w); err != nil {
+			return err
+		}
+	}
+	return nil
+}