Merge pull request #773 from donmccasland/bug-742-2

Fixes caching with COPY command

integration/dockerfiles/Dockerfile_test_cache_copy

@@ -0,0 +1,3 @@
+FROM google/cloud-sdk:256.0.0-alpine
+
+COPY context/foo /usr/bin

integration/images.go

@@ -135,6 +135,7 @@ func NewDockerFileBuilder(dockerfiles []string) *DockerFileBuilder {
 		"Dockerfile_test_cache":         {},
 		"Dockerfile_test_cache_install": {},
 		"Dockerfile_test_cache_perm":    {},
+		"Dockerfile_test_cache_copy":    {},
 	}
 	return &d
 }

pkg/commands/add.go

@@ -19,12 +19,11 @@ package commands
 import (
 	"path/filepath"
 
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
 
-	"github.com/google/go-containerregistry/pkg/v1"
-
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/sirupsen/logrus"
 )

pkg/commands/arg.go

@@ -19,7 +19,7 @@ package commands
 import (
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 

pkg/commands/base_command.go

@@ -18,7 +18,7 @@ package commands
 
 import (
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 )
 
 type BaseCommand struct {

pkg/commands/cmd.go

@@ -20,8 +20,8 @@ import (
 	"strings"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 
-	"github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 

pkg/commands/cmd_test.go

@@ -19,7 +19,7 @@ import (
 	"testing"
 
 	"github.com/GoogleContainerTools/kaniko/testutil"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 

pkg/commands/commands.go

@@ -18,7 +18,7 @@ package commands
 
 import (
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"

pkg/commands/copy.go

@@ -21,13 +21,14 @@ import (
 	"path/filepath"
 
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 )
 
 type CopyCommand struct {
@@ -134,3 +135,46 @@ func (c *CopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerf
 func (c *CopyCommand) MetadataOnly() bool {
 	return false
 }
+
+func (c *CopyCommand) RequiresUnpackedFS() bool {
+	return true
+}
+
+func (c *CopyCommand) ShouldCacheOutput() bool {
+	return true
+}
+
+// CacheCommand returns true since this command should be cached
+func (c *CopyCommand) CacheCommand(img v1.Image) DockerCommand {
+
+	return &CachingCopyCommand{
+		img: img,
+		cmd: c.cmd,
+	}
+}
+
+type CachingCopyCommand struct {
+	BaseCommand
+	img            v1.Image
+	extractedFiles []string
+	cmd            *instructions.CopyCommand
+}
+
+func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
+	logrus.Infof("Found cached layer, extracting to filesystem")
+	var err error
+	cr.extractedFiles, err = util.GetFSFromImage(constants.RootDir, cr.img)
+	logrus.Infof("extractedFiles: %s", cr.extractedFiles)
+	if err != nil {
+		return errors.Wrap(err, "extracting fs from image")
+	}
+	return nil
+}
+
+func (cr *CachingCopyCommand) FilesToSnapshot() []string {
+	return cr.extractedFiles
+}
+
+func (cr *CachingCopyCommand) String() string {
+	return cr.cmd.String()
+}

pkg/commands/copy_test.go

@@ -0,0 +1,167 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package commands
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
+	"github.com/GoogleContainerTools/kaniko/testutil"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/moby/buildkit/frontend/dockerfile/instructions"
+	"github.com/sirupsen/logrus"
+)
+
+var copyTests = []struct {
+	name           string
+	sourcesAndDest []string
+	expectedDest   []string
+}{
+	{
+		name:           "copy foo into tempCopyExecuteTest/",
+		sourcesAndDest: []string{"foo", "tempCopyExecuteTest/"},
+		expectedDest:   []string{"foo"},
+	},
+	{
+		name:           "copy foo into tempCopyExecuteTest",
+		sourcesAndDest: []string{"foo", "tempCopyExecuteTest"},
+		expectedDest:   []string{"tempCopyExecuteTest"},
+	},
+}
+
+func setupTestTemp() string {
+	tempDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		logrus.Fatalf("error creating temp dir %s", err)
+	}
+	logrus.Debugf("Tempdir: %s", tempDir)
+
+	srcPath, err := filepath.Abs("../../integration/context")
+	if err != nil {
+		logrus.Fatalf("error getting abs path %s", srcPath)
+	}
+	cperr := filepath.Walk(srcPath,
+		func(path string, info os.FileInfo, err error) error {
+			if path != srcPath {
+				if err != nil {
+					return err
+				}
+				tempPath := strings.TrimPrefix(path, srcPath)
+				fileInfo, err := os.Stat(path)
+				if err != nil {
+					return err
+				}
+				if fileInfo.IsDir() {
+					os.MkdirAll(tempDir+"/"+tempPath, 0777)
+				} else {
+					out, err := os.Create(tempDir + "/" + tempPath)
+					if err != nil {
+						return err
+					}
+					defer out.Close()
+
+					in, err := os.Open(path)
+					if err != nil {
+						return err
+					}
+					defer in.Close()
+
+					_, err = io.Copy(out, in)
+					if err != nil {
+						return err
+					}
+				}
+			}
+			return nil
+		})
+	if cperr != nil {
+		logrus.Fatalf("error populating temp dir %s", cperr)
+	}
+
+	return tempDir
+}
+func TestCopyExecuteCmd(t *testing.T) {
+	tempDir := setupTestTemp()
+	defer os.RemoveAll(tempDir)
+
+	cfg := &v1.Config{
+		Cmd:        nil,
+		Env:        []string{},
+		WorkingDir: tempDir,
+	}
+
+	for _, test := range copyTests {
+		t.Run(test.name, func(t *testing.T) {
+			dirList := []string{}
+
+			cmd := CopyCommand{
+				cmd: &instructions.CopyCommand{
+					SourcesAndDest: test.sourcesAndDest,
+				},
+				buildcontext: tempDir,
+			}
+
+			buildArgs := copySetUpBuildArgs()
+			dest := cfg.WorkingDir + "/" + test.sourcesAndDest[len(test.sourcesAndDest)-1]
+
+			err := cmd.ExecuteCommand(cfg, buildArgs)
+			if err != nil {
+				t.Error()
+			}
+
+			fi, err := os.Open(dest)
+			if err != nil {
+				t.Error()
+			}
+			defer fi.Close()
+			fstat, err := fi.Stat()
+			if err != nil {
+				t.Error()
+			}
+			if fstat.IsDir() {
+				files, err := ioutil.ReadDir(dest)
+				if err != nil {
+					t.Error()
+				}
+				for _, file := range files {
+					logrus.Debugf("file: %v", file.Name())
+					dirList = append(dirList, file.Name())
+				}
+			} else {
+				dirList = append(dirList, filepath.Base(dest))
+			}
+
+			testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedDest, dirList)
+			os.RemoveAll(dest)
+		})
+	}
+}
+
+func copySetUpBuildArgs() *dockerfile.BuildArgs {
+	buildArgs := dockerfile.NewBuildArgs([]string{
+		"buildArg1=foo",
+		"buildArg2=foo2",
+	})
+	buildArgs.AddArg("buildArg1", nil)
+	d := "default"
+	buildArgs.AddArg("buildArg2", &d)
+	return buildArgs
+}

pkg/commands/entrypoint.go

@@ -20,8 +20,8 @@ import (
 	"strings"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 
-	"github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 

pkg/commands/entrypoint_test.go

@@ -19,7 +19,7 @@ import (
 	"testing"
 
 	"github.com/GoogleContainerTools/kaniko/testutil"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 

pkg/commands/env.go

@@ -18,9 +18,9 @@ package commands
 
 import (
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
-	"github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 

pkg/commands/env_test.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
 	"github.com/GoogleContainerTools/kaniko/testutil"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 

pkg/commands/expose.go

@@ -21,9 +21,9 @@ import (
 	"strings"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
-	"github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/sirupsen/logrus"
 )

pkg/commands/expose_test.go

@@ -20,9 +20,9 @@ import (
 	"testing"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 
 	"github.com/GoogleContainerTools/kaniko/testutil"
-	"github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 

pkg/commands/healthcheck.go

@@ -18,7 +18,7 @@ package commands
 
 import (
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 

pkg/commands/label.go

@@ -18,9 +18,9 @@ package commands
 
 import (
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
-	"github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/sirupsen/logrus"
 )

pkg/commands/label_test.go

@@ -21,7 +21,7 @@ import (
 
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
 	"github.com/GoogleContainerTools/kaniko/testutil"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 

pkg/commands/onbuild.go

@@ -18,7 +18,7 @@ package commands
 
 import (
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/sirupsen/logrus"
 )

pkg/commands/onbuild_test.go

@@ -20,9 +20,9 @@ import (
 	"testing"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 
 	"github.com/GoogleContainerTools/kaniko/testutil"
-	"github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 

pkg/commands/run.go

@@ -28,7 +28,7 @@ import (
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"

pkg/util/fs_util.go

@@ -33,7 +33,7 @@ import (
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/docker/docker/builder/dockerignore"
 	"github.com/docker/docker/pkg/fileutils"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -206,10 +206,13 @@ func extractFile(dest string, hdr *tar.Header, tr io.Reader) error {
 	}
 	switch hdr.Typeflag {
 	case tar.TypeReg:
-		logrus.Tracef("creating file %s", path)
-		// It's possible a file is in the tar before its directory.
-		if _, err := os.Stat(dir); os.IsNotExist(err) {
-			logrus.Tracef("base %s for file %s does not exist. Creating.", base, path)
+		logrus.Debugf("creating file %s", path)
+		// It's possible a file is in the tar before its directory,
+		// or a file was copied over a directory prior to now
+		fi, err := os.Stat(dir)
+		if os.IsNotExist(err) || !fi.IsDir() {
+			logrus.Debugf("base %s for file %s does not exist. Creating.", base, path)
+
 			if err := os.MkdirAll(dir, 0755); err != nil {
 				return err
 			}

pkg/util/image_util.go

@@ -28,7 +28,7 @@ import (
 	"github.com/GoogleContainerTools/kaniko/pkg/creds"
 
 	"github.com/google/go-containerregistry/pkg/name"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"