public_git
/
kaniko
mirror of https://github.com/GoogleContainerTools/kaniko
1
0
Fork 0
Code Issues Releases Wiki Activity

Add command with unit tests

This commit is contained in:
Priya Wadhwa 2018-03-28 13:34:00 -07:00
parent 976afd1992
commit 89400b7410
No known key found for this signature in database
GPG Key ID: 0D0DAFD8F7AA73AE
7 changed files with 484 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
pkg/commands/add.go Normal file
View File

@ -0,0 +1,140 @@
/*
Copyright 2018 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package commands
import (
"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/sirupsen/logrus"
"path/filepath"
"strings"
)
type AddCommand struct {
cmd *instructions.AddCommand
buildcontext string
snapshotFiles []string
}
// ExecuteCommand executes the ADD command
// Special stuff about ADD:
// 1. If <src> is a remote file URL:
// - destination will have permissions of 0600
// - If remote file has HTTP Last-Modified header, we set the mtime of the file to that timestamp
// - If dest doesn't end with a slash, the filepath is inferred to be <dest>/<filename>
// 2. If <src> is a local tar archive:
// -If <src> is a local tar archive, it is unpacked at the dest, as 'tar -x' would
func (a *AddCommand) ExecuteCommand(config *manifest.Schema2Config) error {
srcs := a.cmd.SourcesAndDest[:len(a.cmd.SourcesAndDest)-1]
dest := a.cmd.SourcesAndDest[len(a.cmd.SourcesAndDest)-1]
logrus.Infof("cmd: Add %s", srcs)
logrus.Infof("dest: %s", dest)
// First, resolve any environment replacement
resolvedEnvs, err := util.ResolveEnvironmentReplacementList(a.cmd.SourcesAndDest, config.Env, true)
if err != nil {
return err
}
dest = resolvedEnvs[len(resolvedEnvs)-1]
// Get a map of [src]:[files rooted at src]
srcMap, err := util.ResolveSources(resolvedEnvs, a.buildcontext)
if err != nil {
return err
}
// If any of the sources are local tar archives:
// 1. Unpack them to the specified destination
// 2. Remove it as a source that needs to be copied over
// If any of the sources is a remote file URL:
// 1. Download and copy it to the specifed dest
// 2. Remove it as a source that needs to be copied
for src, files := range srcMap {
for _, file := range files {
// If file is a local tar archive, then we unpack it to dest
filePath := filepath.Join(a.buildcontext, file)
isFilenameSource, err := isFilenameSource(srcMap, file)
if err != nil {
return err
}
if util.IsSrcRemoteFileURL(file) {
urlDest := util.URLDestinationFilepath(file, dest, config.WorkingDir)
logrus.Infof("Adding remote URL %s to %s", file, urlDest)
if err := util.DownloadFileToDest(file, urlDest); err != nil {
return err
}
a.snapshotFiles = append(a.snapshotFiles, urlDest)
delete(srcMap, src)
} else if isFilenameSource && util.IsFileLocalTarArchive(filePath) {
logrus.Infof("Unpacking local tar archive %s to %s", file, dest)
if err := util.UnpackLocalTarArchive(filePath, dest); err != nil {
return err
}
// Add the unpacked files to the snapshotter
filesAdded, err := util.Files(dest)
if err != nil {
return err
}
logrus.Debugf("Added %v from local tar archive %s", filesAdded, file)
a.snapshotFiles = append(a.snapshotFiles, filesAdded...)
delete(srcMap, src)
}
}
}
// With the remaining "normal" sources, create and execute a standard copy command
if len(srcMap) == 0 {
return nil
}
var regularSrcs []string
for src := range srcMap {
regularSrcs = append(regularSrcs, src)
}
copyCmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: append(regularSrcs, dest),
},
buildcontext: a.buildcontext,
}
if err := copyCmd.ExecuteCommand(config); err != nil {
return err
}
a.snapshotFiles = append(a.snapshotFiles, copyCmd.snapshotFiles...)
return nil
}
func isFilenameSource(srcMap map[string][]string, fileName string) (bool, error) {
for src := range srcMap {
matched, err := filepath.Match(src, fileName)
if err != nil {
return false, err
}
if matched || (src == fileName) {
return true, nil
}
}
return false, nil
}
// FilesToSnapshot should return an empty array if still nil; no files were changed
func (a *AddCommand) FilesToSnapshot() []string {
return a.snapshotFiles
}
// CreatedBy returns some information about the command for the image config
func (a *AddCommand) CreatedBy() string {
return strings.Join(a.cmd.SourcesAndDest, " ")
}

2
pkg/commands/commands.go
View File

@ -44,6 +44,8 @@ func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, e
return &ExposeCommand{cmd: c}, nil
case *instructions.EnvCommand:
return &EnvCommand{cmd: c}, nil
case *instructions.AddCommand:
return &AddCommand{cmd: c, buildcontext: buildcontext}, nil
case *instructions.CmdCommand:
return &CmdCommand{cmd: c}, nil
case *instructions.EntrypointCommand:

45
pkg/util/command_util.go
View File

@ -22,6 +22,8 @@ import (
"github.com/docker/docker/builder/dockerfile/shell"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
@ -31,6 +33,10 @@ import (
func ResolveEnvironmentReplacementList(values, envs []string, isFilepath bool) ([]string, error) {
var resolvedValues []string
for _, value := range values {
if IsSrcRemoteFileURL(value) {
resolvedValues = append(resolvedValues, value)
continue
}
resolved, err := ResolveEnvironmentReplacement(value, envs, isFilepath)
logrus.Debugf("Resolved %s to %s", value, resolved)
if err != nil {
@ -106,13 +112,17 @@ func ResolveSources(srcsAndDest instructions.SourcesAndDest, root string) (map[s
func matchSources(srcs, files []string) ([]string, error) {
var matchedSources []string
for _, src := range srcs {
if IsSrcRemoteFileURL(src) {
matchedSources = append(matchedSources, src)
continue
}
src = filepath.Clean(src)
for _, file := range files {
matched, err := filepath.Match(src, file)
if err != nil {
return nil, err
}
if matched {
if matched || src == file {
matchedSources = append(matchedSources, file)
}
}
@ -160,10 +170,31 @@ func DestinationFilepath(filename, srcName, dest, cwd, buildcontext string) (str
return filepath.Join(cwd, dest), nil
}
// URLDestinationFilepath gives the destintion a file from a remote URL should be saved to
func URLDestinationFilepath(rawurl, dest, cwd string) string {
if !IsDestDir(dest) {
if !filepath.IsAbs(dest) {
return filepath.Join(cwd, dest)
}
return dest
}
urlBase := filepath.Base(rawurl)
destPath := filepath.Join(dest, urlBase)
if !filepath.IsAbs(dest) {
destPath = filepath.Join(cwd, destPath)
}
return destPath
}
// SourcesToFilesMap returns a map of [src]:[files rooted at source]
func SourcesToFilesMap(srcs []string, root string) (map[string][]string, error) {
srcMap := make(map[string][]string)
for _, src := range srcs {
if IsSrcRemoteFileURL(src) {
srcMap[src] = []string{src}
continue
}
src = filepath.Clean(src)
files, err := RelativeFiles(src, root)
if err != nil {
@ -202,3 +233,15 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, srcMap map[string][]st
}
return nil
}
func IsSrcRemoteFileURL(rawurl string) bool {
_, err := url.ParseRequestURI(rawurl)
if err != nil {
return false
}
_, err = http.Get(rawurl)
if err != nil {
return false
}
return true
}

67
pkg/util/command_util_test.go
View File

@ -22,6 +22,8 @@ import (
"testing"
)
var testUrl = "https://github.com/GoogleCloudPlatform/runtimes-common/blob/master/LICENSE"
var testEnvReplacement = []struct {
path string
command string
@ -207,6 +209,39 @@ func Test_DestinationFilepath(t *testing.T) {
}
}
var urlDestFilepathTests = []struct {
url string
cwd string
dest string
expectedDest string
}{
{
url: "https://something/something",
cwd: "/test",
dest: ".",
expectedDest: "/test/something",
},
{
url: "https://something/something",
cwd: "/cwd",
dest: "/test",
expectedDest: "/test",
},
{
url: "https://something/something",
cwd: "/test",
dest: "/dest/",
expectedDest: "/dest/something",
},
}
func Test_UrlDestFilepath(t *testing.T) {
for _, test := range urlDestFilepathTests {
actualDest := URLDestinationFilepath(test.url, test.dest, test.cwd)
testutil.CheckErrorAndDeepEqual(t, false, nil, test.expectedDest, actualDest)
}
}
var matchSourcesTests = []struct {
srcs []string
files []string
@ -215,6 +250,7 @@ var matchSourcesTests = []struct {
{
srcs: []string{
"pkg/*",
testUrl,
},
files: []string{
"pkg/a",
@ -226,6 +262,7 @@ var matchSourcesTests = []struct {
expectedFiles: []string{
"pkg/a",
"pkg/b",
testUrl,
},
},
}
@ -380,6 +417,7 @@ var testResolveSources = []struct {
srcsAndDest: []string{
"context/foo",
"context/b*",
testUrl,
"dest/",
},
expectedMap: map[string][]string{
@ -393,6 +431,9 @@ var testResolveSources = []struct {
"context/bar/bat",
"context/bar/baz",
},
testUrl: {
testUrl,
},
},
},
}
@ -403,3 +444,29 @@ func Test_ResolveSources(t *testing.T) {
testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedMap, actualMap)
}
}
var testRemoteUrls = []struct {
url string
valid bool
}{
{
url: testUrl,
valid: true,
},
{
url: "not/real/",
valid: false,
},
{
url: "https://url.com/something/not/real",
valid: false,
},
}
func Test_RemoteUrls(t *testing.T) {
for _, test := range testRemoteUrls {
valid := IsSrcRemoteFileURL(test.url)
testutil.CheckErrorAndDeepEqual(t, false, nil, test.valid, valid)
}
}

37
pkg/util/fs_util.go
View File

@ -23,9 +23,11 @@ import (
"github.com/containers/image/docker"
"github.com/sirupsen/logrus"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
var whitelist = []string{"/kbuild"}
@ -117,6 +119,17 @@ func RelativeFiles(fp string, root string) ([]string, error) {
return files, err
}
// Files returns a list of all files rooted at root
func Files(root string) ([]string, error) {
var files []string
logrus.Debugf("Getting files and contents at root %s", root)
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
files = append(files, path)
return err
})
return files, err
}
// FilepathExists returns true if the path exists
func FilepathExists(path string) bool {
_, err := os.Stat(path)
@ -142,3 +155,27 @@ func CreateFile(path string, reader io.Reader, perm os.FileMode) error {
}
return dest.Chmod(perm)
}
// DownloadFileToDest downloads the file at rawurl to the given dest for the ADD command
// From add command docs:
// 1. If <src> is a remote file URL:
// - destination will have permissions of 0600
// - If remote file has HTTP Last-Modified header, we set the mtime of the file to that timestamp
func DownloadFileToDest(rawurl, dest string) error {
resp, err := http.Get(rawurl)
if err != nil {
return err
}
defer resp.Body.Close()
if err := CreateFile(dest, resp.Body, 0600); err != nil {
return err
}
mTime := time.Time{}
lastMod := resp.Header.Get("Last-Modified")
if lastMod != "" {
if parsedMTime, err := http.ParseTime(lastMod); err == nil {
mTime = parsedMTime
}
}
return os.Chtimes(dest, mTime, mTime)
}

86
pkg/util/tar_util.go
View File

@ -18,8 +18,14 @@ package util
import (
"archive/tar"
"compress/bzip2"
"compress/gzip"
pkgutil "github.com/GoogleCloudPlatform/container-diff/pkg/util"
"github.com/docker/docker/pkg/archive"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"io"
"io/ioutil"
"os"
"syscall"
)
@ -86,3 +92,83 @@ func checkHardlink(p string, i os.FileInfo) (bool, string) {
}
return hardlink, linkDst
}
//UnpackLocalTarArchive unpacks the tar archive at path to the directory dest
// Returns true if the path was acutally unpacked
func UnpackLocalTarArchive(path, dest string) error {
// First, we need to check if the path is a local tar archive
if compressed, compressionLevel := fileIsCompressedTar(path); compressed {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
if compressionLevel == archive.Gzip {
gzr, err := gzip.NewReader(file)
if err != nil {
return err
}
defer gzr.Close()
return pkgutil.UnTar(gzr, dest, nil)
} else if compressionLevel == archive.Bzip2 {
bzr := bzip2.NewReader(file)
return pkgutil.UnTar(bzr, dest, nil)
}
}
if fileIsUncompressedTar(path) {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
return pkgutil.UnTar(file, dest, nil)
}
return errors.New("path does not lead to local tar archive")
}
//IsFileLocalTarArchive returns true if the file is a local tar archive
func IsFileLocalTarArchive(src string) bool {
compressed, _ := fileIsCompressedTar(src)
uncompressed := fileIsUncompressedTar(src)
return compressed || uncompressed
}
func fileIsCompressedTar(src string) (bool, archive.Compression) {
r, err := os.Open(src)
if err != nil {
return false, -1
}
defer r.Close()
buf, err := ioutil.ReadAll(r)
if err != nil {
return false, -1
}
compressionLevel := archive.DetectCompression(buf)
return (compressionLevel > 0), compressionLevel
}
func fileIsUncompressedTar(src string) bool {
r, err := os.Open(src)
defer r.Close()
if err != nil {
return false
}
fi, err := os.Stat(src)
if err != nil {
return false
}
if fi.Size() == 0 {
return false
}
tr := tar.NewReader(r)
if tr == nil {
return false
}
for {
_, err := tr.Next()
if err != nil {
return false
}
return true
}
}

108
pkg/util/tar_util_test.go Normal file
View File

@ -0,0 +1,108 @@
/*
Copyright 2018 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"archive/tar"
"compress/gzip"
"github.com/GoogleCloudPlatform/k8s-container-builder/testutil"
"io"
"io/ioutil"
"os"
"path/filepath"
"testing"
)
var regularFiles = []string{"file", "file.tar", "file.tar.gz"}
var uncompressedTars = []string{"uncompressed", "uncompressed.tar"}
var compressedTars = []string{"compressed", "compressed.tar.gz"}
func Test_IsLocalTarArchive(t *testing.T) {
testDir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("err setting up temp dir: %v", err)
}
defer os.RemoveAll(testDir)
if err := setUpFilesAndTars(testDir); err != nil {
t.Fatal(err)
}
// Test we get the correct result for regular files
for _, regularFile := range regularFiles {
isTarArchive := IsFileLocalTarArchive(filepath.Join(testDir, regularFile))
testutil.CheckErrorAndDeepEqual(t, false, nil, false, isTarArchive)
}
// Test we get the correct result for uncompressed tars
for _, uncompressedTar := range uncompressedTars {
isTarArchive := IsFileLocalTarArchive(filepath.Join(testDir, uncompressedTar))
testutil.CheckErrorAndDeepEqual(t, false, nil, true, isTarArchive)
}
// Test we get the correct result for compressed tars
for _, compressedTar := range compressedTars {
isTarArchive := IsFileLocalTarArchive(filepath.Join(testDir, compressedTar))
testutil.CheckErrorAndDeepEqual(t, false, nil, true, isTarArchive)
}
}
func setUpFilesAndTars(testDir string) error {
regularFilesAndContents := map[string]string{
regularFiles[0]: "",
regularFiles[1]: "something",
regularFiles[2]: "here",
}
if err := testutil.SetupFiles(testDir, regularFilesAndContents); err != nil {
return err
}
for _, uncompressedTar := range uncompressedTars {
tarFile, err := os.Create(filepath.Join(testDir, uncompressedTar))
if err != nil {
return err
}
if err := createTar(testDir, tarFile); err != nil {
return err
}
}
for _, compressedTar := range compressedTars {
tarFile, err := os.Create(filepath.Join(testDir, compressedTar))
if err != nil {
return err
}
gzr := gzip.NewWriter(tarFile)
if err := createTar(testDir, gzr); err != nil {
return err
}
}
return nil
}
func createTar(testdir string, writer io.Writer) error {
w := tar.NewWriter(writer)
defer w.Close()
for _, regFile := range regularFiles {
filePath := filepath.Join(testdir, regFile)
fi, err := os.Stat(filePath)
if err != nil {
return err
}
if err := AddToTar(filePath, fi, w); err != nil {
return err
}
}
return nil
}