diff --git a/integration/dockerfiles/Dockerfile_test_cache_copy b/integration/dockerfiles/Dockerfile_test_cache_copy new file mode 100644 index 000000000..661397eae --- /dev/null +++ b/integration/dockerfiles/Dockerfile_test_cache_copy @@ -0,0 +1,3 @@ +FROM google/cloud-sdk:256.0.0-alpine + +COPY context/foo /usr/bin diff --git a/integration/images.go b/integration/images.go index a9847aef8..8da15d6f7 100644 --- a/integration/images.go +++ b/integration/images.go @@ -135,6 +135,7 @@ func NewDockerFileBuilder(dockerfiles []string) *DockerFileBuilder { "Dockerfile_test_cache": {}, "Dockerfile_test_cache_install": {}, "Dockerfile_test_cache_perm": {}, + "Dockerfile_test_cache_copy": {}, } return &d } diff --git a/pkg/commands/add.go b/pkg/commands/add.go index 769d050dd..7a3d6164b 100644 --- a/pkg/commands/add.go +++ b/pkg/commands/add.go @@ -19,12 +19,11 @@ package commands import ( "path/filepath" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" - "github.com/google/go-containerregistry/pkg/v1" - "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/sirupsen/logrus" ) diff --git a/pkg/commands/arg.go b/pkg/commands/arg.go index e17767338..598e59a50 100644 --- a/pkg/commands/arg.go +++ b/pkg/commands/arg.go @@ -19,7 +19,7 @@ package commands import ( "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" "github.com/GoogleContainerTools/kaniko/pkg/util" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) diff --git a/pkg/commands/base_command.go b/pkg/commands/base_command.go index 3d2cd8fad..ddbfd650b 100644 --- a/pkg/commands/base_command.go +++ b/pkg/commands/base_command.go @@ -18,7 +18,7 @@ package commands import ( "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" ) type BaseCommand struct { diff --git a/pkg/commands/cmd.go b/pkg/commands/cmd.go index 87714430b..ff94de509 100644 --- a/pkg/commands/cmd.go +++ b/pkg/commands/cmd.go @@ -20,8 +20,8 @@ import ( "strings" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) diff --git a/pkg/commands/cmd_test.go b/pkg/commands/cmd_test.go index b56786ad1..182e9707c 100644 --- a/pkg/commands/cmd_test.go +++ b/pkg/commands/cmd_test.go @@ -19,7 +19,7 @@ import ( "testing" "github.com/GoogleContainerTools/kaniko/testutil" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index 2615138be..a2ea7788d 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -18,7 +18,7 @@ package commands import ( "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index 9af5ce3fd..15f778e8d 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -21,13 +21,14 @@ import ( "path/filepath" "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" "github.com/GoogleContainerTools/kaniko/pkg/util" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" ) type CopyCommand struct { @@ -134,3 +135,46 @@ func (c *CopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerf func (c *CopyCommand) MetadataOnly() bool { return false } + +func (c *CopyCommand) RequiresUnpackedFS() bool { + return true +} + +func (c *CopyCommand) ShouldCacheOutput() bool { + return true +} + +// CacheCommand returns true since this command should be cached +func (c *CopyCommand) CacheCommand(img v1.Image) DockerCommand { + + return &CachingCopyCommand{ + img: img, + cmd: c.cmd, + } +} + +type CachingCopyCommand struct { + BaseCommand + img v1.Image + extractedFiles []string + cmd *instructions.CopyCommand +} + +func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { + logrus.Infof("Found cached layer, extracting to filesystem") + var err error + cr.extractedFiles, err = util.GetFSFromImage(constants.RootDir, cr.img) + logrus.Infof("extractedFiles: %s", cr.extractedFiles) + if err != nil { + return errors.Wrap(err, "extracting fs from image") + } + return nil +} + +func (cr *CachingCopyCommand) FilesToSnapshot() []string { + return cr.extractedFiles +} + +func (cr *CachingCopyCommand) String() string { + return cr.cmd.String() +} diff --git a/pkg/commands/copy_test.go b/pkg/commands/copy_test.go new file mode 100644 index 000000000..9b16e9291 --- /dev/null +++ b/pkg/commands/copy_test.go @@ -0,0 +1,167 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package commands + +import ( + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + "github.com/GoogleContainerTools/kaniko/testutil" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/sirupsen/logrus" +) + +var copyTests = []struct { + name string + sourcesAndDest []string + expectedDest []string +}{ + { + name: "copy foo into tempCopyExecuteTest/", + sourcesAndDest: []string{"foo", "tempCopyExecuteTest/"}, + expectedDest: []string{"foo"}, + }, + { + name: "copy foo into tempCopyExecuteTest", + sourcesAndDest: []string{"foo", "tempCopyExecuteTest"}, + expectedDest: []string{"tempCopyExecuteTest"}, + }, +} + +func setupTestTemp() string { + tempDir, err := ioutil.TempDir("", "") + if err != nil { + logrus.Fatalf("error creating temp dir %s", err) + } + logrus.Debugf("Tempdir: %s", tempDir) + + srcPath, err := filepath.Abs("../../integration/context") + if err != nil { + logrus.Fatalf("error getting abs path %s", srcPath) + } + cperr := filepath.Walk(srcPath, + func(path string, info os.FileInfo, err error) error { + if path != srcPath { + if err != nil { + return err + } + tempPath := strings.TrimPrefix(path, srcPath) + fileInfo, err := os.Stat(path) + if err != nil { + return err + } + if fileInfo.IsDir() { + os.MkdirAll(tempDir+"/"+tempPath, 0777) + } else { + out, err := os.Create(tempDir + "/" + tempPath) + if err != nil { + return err + } + defer out.Close() + + in, err := os.Open(path) + if err != nil { + return err + } + defer in.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + } + } + return nil + }) + if cperr != nil { + logrus.Fatalf("error populating temp dir %s", cperr) + } + + return tempDir +} +func TestCopyExecuteCmd(t *testing.T) { + tempDir := setupTestTemp() + defer os.RemoveAll(tempDir) + + cfg := &v1.Config{ + Cmd: nil, + Env: []string{}, + WorkingDir: tempDir, + } + + for _, test := range copyTests { + t.Run(test.name, func(t *testing.T) { + dirList := []string{} + + cmd := CopyCommand{ + cmd: &instructions.CopyCommand{ + SourcesAndDest: test.sourcesAndDest, + }, + buildcontext: tempDir, + } + + buildArgs := copySetUpBuildArgs() + dest := cfg.WorkingDir + "/" + test.sourcesAndDest[len(test.sourcesAndDest)-1] + + err := cmd.ExecuteCommand(cfg, buildArgs) + if err != nil { + t.Error() + } + + fi, err := os.Open(dest) + if err != nil { + t.Error() + } + defer fi.Close() + fstat, err := fi.Stat() + if err != nil { + t.Error() + } + if fstat.IsDir() { + files, err := ioutil.ReadDir(dest) + if err != nil { + t.Error() + } + for _, file := range files { + logrus.Debugf("file: %v", file.Name()) + dirList = append(dirList, file.Name()) + } + } else { + dirList = append(dirList, filepath.Base(dest)) + } + + testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedDest, dirList) + os.RemoveAll(dest) + }) + } +} + +func copySetUpBuildArgs() *dockerfile.BuildArgs { + buildArgs := dockerfile.NewBuildArgs([]string{ + "buildArg1=foo", + "buildArg2=foo2", + }) + buildArgs.AddArg("buildArg1", nil) + d := "default" + buildArgs.AddArg("buildArg2", &d) + return buildArgs +} diff --git a/pkg/commands/entrypoint.go b/pkg/commands/entrypoint.go index 23f349347..83964d4fa 100644 --- a/pkg/commands/entrypoint.go +++ b/pkg/commands/entrypoint.go @@ -20,8 +20,8 @@ import ( "strings" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) diff --git a/pkg/commands/entrypoint_test.go b/pkg/commands/entrypoint_test.go index 5343d67fe..190c7055c 100644 --- a/pkg/commands/entrypoint_test.go +++ b/pkg/commands/entrypoint_test.go @@ -19,7 +19,7 @@ import ( "testing" "github.com/GoogleContainerTools/kaniko/testutil" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) diff --git a/pkg/commands/env.go b/pkg/commands/env.go index 5286f7260..2be271490 100644 --- a/pkg/commands/env.go +++ b/pkg/commands/env.go @@ -18,9 +18,9 @@ package commands import ( "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/GoogleContainerTools/kaniko/pkg/util" - "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) diff --git a/pkg/commands/env_test.go b/pkg/commands/env_test.go index d4870bb1e..ccada538c 100644 --- a/pkg/commands/env_test.go +++ b/pkg/commands/env_test.go @@ -20,7 +20,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" "github.com/GoogleContainerTools/kaniko/testutil" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) diff --git a/pkg/commands/expose.go b/pkg/commands/expose.go index 9d56ee3ea..0aacbb6cf 100644 --- a/pkg/commands/expose.go +++ b/pkg/commands/expose.go @@ -21,9 +21,9 @@ import ( "strings" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/GoogleContainerTools/kaniko/pkg/util" - "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/sirupsen/logrus" ) diff --git a/pkg/commands/expose_test.go b/pkg/commands/expose_test.go index 02ad17576..fa1f9520a 100644 --- a/pkg/commands/expose_test.go +++ b/pkg/commands/expose_test.go @@ -20,9 +20,9 @@ import ( "testing" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/GoogleContainerTools/kaniko/testutil" - "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) diff --git a/pkg/commands/healthcheck.go b/pkg/commands/healthcheck.go index 610d9c935..af40ba652 100644 --- a/pkg/commands/healthcheck.go +++ b/pkg/commands/healthcheck.go @@ -18,7 +18,7 @@ package commands import ( "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) diff --git a/pkg/commands/label.go b/pkg/commands/label.go index c8790824b..7568886c0 100644 --- a/pkg/commands/label.go +++ b/pkg/commands/label.go @@ -18,9 +18,9 @@ package commands import ( "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/GoogleContainerTools/kaniko/pkg/util" - "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/sirupsen/logrus" ) diff --git a/pkg/commands/label_test.go b/pkg/commands/label_test.go index da2cfbaea..644746311 100644 --- a/pkg/commands/label_test.go +++ b/pkg/commands/label_test.go @@ -21,7 +21,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" "github.com/GoogleContainerTools/kaniko/testutil" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) diff --git a/pkg/commands/onbuild.go b/pkg/commands/onbuild.go index 0d36afe4b..74c6ad01e 100644 --- a/pkg/commands/onbuild.go +++ b/pkg/commands/onbuild.go @@ -18,7 +18,7 @@ package commands import ( "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/sirupsen/logrus" ) diff --git a/pkg/commands/onbuild_test.go b/pkg/commands/onbuild_test.go index 10ee9ba26..e2d5d940d 100644 --- a/pkg/commands/onbuild_test.go +++ b/pkg/commands/onbuild_test.go @@ -20,9 +20,9 @@ import ( "testing" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/GoogleContainerTools/kaniko/testutil" - "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) diff --git a/pkg/commands/run.go b/pkg/commands/run.go index 7a9bf03dc..0122f37d5 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -28,7 +28,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" "github.com/GoogleContainerTools/kaniko/pkg/util" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index b10f95338..ab327c860 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -33,7 +33,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/docker/docker/builder/dockerignore" "github.com/docker/docker/pkg/fileutils" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -206,10 +206,13 @@ func extractFile(dest string, hdr *tar.Header, tr io.Reader) error { } switch hdr.Typeflag { case tar.TypeReg: - logrus.Tracef("creating file %s", path) - // It's possible a file is in the tar before its directory. - if _, err := os.Stat(dir); os.IsNotExist(err) { - logrus.Tracef("base %s for file %s does not exist. Creating.", base, path) + logrus.Debugf("creating file %s", path) + // It's possible a file is in the tar before its directory, + // or a file was copied over a directory prior to now + fi, err := os.Stat(dir) + if os.IsNotExist(err) || !fi.IsDir() { + logrus.Debugf("base %s for file %s does not exist. Creating.", base, path) + if err := os.MkdirAll(dir, 0755); err != nil { return err } diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index 0978a6fef..01dc12fae 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -28,7 +28,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/creds" "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball"