Merge branch 'master' into output

pkg/buildcontext/buildcontext.go

@@ -42,6 +42,8 @@ func GetBuildContext(srcContext string) (BuildContext, error) {
 		return &S3{context: context}, nil
 	case constants.LocalDirBuildContextPrefix:
 		return &Dir{context: context}, nil
+	case constants.GitBuildContextPrefix:
+		return &Git{context: context}, nil
 	}
-	return nil, errors.New("unknown build context prefix provided, please use one of the following: gs://, dir://, s3://")
+	return nil, errors.New("unknown build context prefix provided, please use one of the following: gs://, dir://, s3://, git://")
 }

pkg/buildcontext/git.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildcontext
+
+import (
+	"os"
+	"strings"
+
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	git "gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/plumbing"
+)
+
+// Git unifies calls to download and unpack the build context.
+type Git struct {
+	context string
+}
+
+// UnpackTarFromBuildContext will provide the directory where Git Repository is Cloned
+func (g *Git) UnpackTarFromBuildContext() (string, error) {
+	directory := constants.BuildContextDir
+	parts := strings.Split(g.context, "#")
+	options := git.CloneOptions{
+		URL:      "https://" + parts[0],
+		Progress: os.Stdout,
+	}
+	if len(parts) > 1 {
+		options.ReferenceName = plumbing.ReferenceName(parts[1])
+	}
+	_, err := git.PlainClone(directory, false, &options)
+	return directory, err
+}

pkg/buildcontext/s3.go

@@ -19,6 +19,7 @@ package buildcontext
 import (
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
@@ -36,9 +37,21 @@ type S3 struct {
 // UnpackTarFromBuildContext download and untar a file from s3
 func (s *S3) UnpackTarFromBuildContext() (string, error) {
 	bucket, item := util.GetBucketAndItem(s.context)
-	sess, err := session.NewSessionWithOptions(session.Options{
+	option := session.Options{
 		SharedConfigState: session.SharedConfigEnable,
-	})
+	}
+	endpoint := os.Getenv(constants.S3EndpointEnv)
+	forcePath := false
+	if strings.ToLower(os.Getenv(constants.S3ForcePathStyle)) == "true" {
+		forcePath = true
+	}
+	if endpoint != "" {
+		option.Config = aws.Config{
+			Endpoint:         aws.String(endpoint),
+			S3ForcePathStyle: aws.Bool(forcePath),
+		}
+	}
+	sess, err := session.NewSessionWithOptions(option)
 	if err != nil {
 		return bucket, err
 	}

pkg/cache/cache.go

@@ -59,8 +59,8 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) {
 	}
 
 	registryName := cacheRef.Repository.Registry.Name()
-	if rc.Opts.InsecureRegistries.Contains(registryName) {
-		newReg, err := name.NewInsecureRegistry(registryName, name.WeakValidation)
+	if rc.Opts.Insecure || rc.Opts.InsecureRegistries.Contains(registryName) {
+		newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
 		if err != nil {
 			return nil, err
 		}
@@ -88,7 +88,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) {
 	// Layer is stale, rebuild it.
 	if expiry.Before(time.Now()) {
 		logrus.Infof("Cache entry expired: %s", cache)
-		return nil, errors.New(fmt.Sprintf("Cache entry expired: %s", cache))
+		return nil, fmt.Errorf("Cache entry expired: %s", cache)
 	}
 
 	// Force the manifest to be populated
@@ -114,7 +114,7 @@ func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) {
 }
 
 // LocalSource retieves a source image from a local cache given cacheKey
-func LocalSource(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) {
+func LocalSource(opts *config.CacheOptions, cacheKey string) (v1.Image, error) {
 	cache := opts.CacheDir
 	if cache == "" {
 		return nil, nil

pkg/cache/warm.go

@@ -29,6 +29,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// WarmCache populates the cache
 func WarmCache(opts *config.WarmerOptions) error {
 	cacheDir := opts.CacheDir
 	images := opts.Images
@@ -41,7 +42,7 @@ func WarmCache(opts *config.WarmerOptions) error {
 			return errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image))
 		}
 		img, err := remote.Image(cacheRef)
-		if err != nil {
+		if err != nil || img == nil {
 			return errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image))
 		}
 
@@ -50,6 +51,14 @@ func WarmCache(opts *config.WarmerOptions) error {
 			return errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image))
 		}
 		cachePath := path.Join(cacheDir, digest.String())
+
+		if !opts.Force {
+			_, err := LocalSource(&opts.CacheOptions, digest.String())
+			if err == nil {
+				continue
+			}
+		}
+
 		err = tarball.WriteToFile(cachePath, cacheRef, img)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image))

pkg/commands/add.go

@@ -43,11 +43,11 @@ type AddCommand struct {
 // 		- If remote file has HTTP Last-Modified header, we set the mtime of the file to that timestamp
 // 		- If dest doesn't end with a slash, the filepath is inferred to be <dest>/<filename>
 // 	2. If <src> is a local tar archive:
-// 		-If <src> is a local tar archive, it is unpacked at the dest, as 'tar -x' would
+// 		- it is unpacked at the dest, as 'tar -x' would
 func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
 	replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
 
-	srcs, dest, err := resolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs)
+	srcs, dest, err := util.ResolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs)
 	if err != nil {
 		return err
 	}
@@ -61,15 +61,22 @@ func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui
 	for _, src := range srcs {
 		fullPath := filepath.Join(a.buildcontext, src)
 		if util.IsSrcRemoteFileURL(src) {
-			urlDest := util.URLDestinationFilepath(src, dest, config.WorkingDir)
+			urlDest, err := util.URLDestinationFilepath(src, dest, config.WorkingDir, replacementEnvs)
+			if err != nil {
+				return err
+			}
 			logrus.Infof("Adding remote URL %s to %s", src, urlDest)
 			if err := util.DownloadFileToDest(src, urlDest); err != nil {
 				return err
 			}
 			a.snapshotFiles = append(a.snapshotFiles, urlDest)
 		} else if util.IsFileLocalTarArchive(fullPath) {
-			logrus.Infof("Unpacking local tar archive %s to %s", src, dest)
-			extractedFiles, err := util.UnpackLocalTarArchive(fullPath, dest)
+			tarDest, err := util.DestinationFilepath("", dest, config.WorkingDir)
+			if err != nil {
+				return err
+			}
+			logrus.Infof("Unpacking local tar archive %s to %s", src, tarDest)
+			extractedFiles, err := util.UnpackLocalTarArchive(fullPath, tarDest)
 			if err != nil {
 				return err
 			}
@@ -111,7 +118,7 @@ func (a *AddCommand) String() string {
 func (a *AddCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) {
 	replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
 
-	srcs, _, err := resolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs)
+	srcs, _, err := util.ResolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs)
 	if err != nil {
 		return nil, err
 	}

pkg/commands/arg.go

@@ -30,27 +30,35 @@ type ArgCommand struct {
 
 // ExecuteCommand only needs to add this ARG key/value as seen
 func (r *ArgCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
-	replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
-	resolvedKey, err := util.ResolveEnvironmentReplacement(r.cmd.Key, replacementEnvs, false)
+	key, val, err := ParseArg(r.cmd.Key, r.cmd.Value, config.Env, buildArgs)
 	if err != nil {
 		return err
 	}
+
+	buildArgs.AddArg(key, val)
+	return nil
+}
+
+func ParseArg(key string, val *string, env []string, ba *dockerfile.BuildArgs) (string, *string, error) {
+	replacementEnvs := ba.ReplacementEnvs(env)
+	resolvedKey, err := util.ResolveEnvironmentReplacement(key, replacementEnvs, false)
+	if err != nil {
+		return "", nil, err
+	}
 	var resolvedValue *string
-	if r.cmd.Value != nil {
-		value, err := util.ResolveEnvironmentReplacement(*r.cmd.Value, replacementEnvs, false)
+	if val != nil {
+		value, err := util.ResolveEnvironmentReplacement(*val, replacementEnvs, false)
 		if err != nil {
-			return err
+			return "", nil, err
 		}
 		resolvedValue = &value
 	} else {
-		meta := buildArgs.GetAllMeta()
+		meta := ba.GetAllMeta()
 		if value, ok := meta[resolvedKey]; ok {
 			resolvedValue = &value
 		}
 	}
-
-	buildArgs.AddArg(resolvedKey, resolvedValue)
-	return nil
+	return resolvedKey, resolvedValue, nil
 }
 
 // String returns some information about the command for the image config history

pkg/commands/copy.go

@@ -45,7 +45,7 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu
 
 	replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
 
-	srcs, dest, err := resolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs)
+	srcs, dest, err := util.ResolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs)
 	if err != nil {
 		return err
 	}
@@ -100,18 +100,6 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu
 	return nil
 }
 
-func resolveEnvAndWildcards(sd instructions.SourcesAndDest, buildcontext string, envs []string) ([]string, string, error) {
-	// First, resolve any environment replacement
-	resolvedEnvs, err := util.ResolveEnvironmentReplacementList(sd, envs, true)
-	if err != nil {
-		return nil, "", err
-	}
-	dest := resolvedEnvs[len(resolvedEnvs)-1]
-	// Resolve wildcards and get a list of resolved sources
-	srcs, err := util.ResolveSources(resolvedEnvs, buildcontext)
-	return srcs, dest, err
-}
-
 // FilesToSnapshot should return an empty array if still nil; no files were changed
 func (c *CopyCommand) FilesToSnapshot() []string {
 	return c.snapshotFiles
@@ -129,7 +117,7 @@ func (c *CopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerf
 	}
 
 	replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
-	srcs, _, err := resolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs)
+	srcs, _, err := util.ResolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs)
 	if err != nil {
 		return nil, err
 	}

pkg/commands/expose.go

@@ -54,7 +54,7 @@ func (r *ExposeCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.
 		}
 		protocol := strings.Split(p, "/")[1]
 		if !validProtocol(protocol) {
-			return fmt.Errorf("Invalid protocol: %s", protocol)
+			return fmt.Errorf("invalid protocol: %s", protocol)
 		}
 		logrus.Infof("Adding exposed port: %s", p)
 		existingPorts[p] = struct{}{}

pkg/commands/user.go

@@ -31,10 +31,6 @@ type UserCommand struct {
 	cmd *instructions.UserCommand
 }
 
-func (r *UserCommand) RequiresUnpackedFS() bool {
-	return true
-}
-
 func (r *UserCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
 	logrus.Info("cmd: USER")
 	u := r.cmd.User
@@ -52,11 +48,6 @@ func (r *UserCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu
 		}
 	}
 
-	_, _, err = util.GetUserFromUsername(userStr, groupStr)
-	if err != nil {
-		return err
-	}
-
 	if groupStr != "" {
 		userStr = userStr + ":" + groupStr
 	}

pkg/commands/user_test.go

@@ -28,57 +28,42 @@ import (
 var userTests = []struct {
 	user        string
 	expectedUID string
-	shouldError bool
 }{
 	{
 		user:        "root",
 		expectedUID: "root",
-		shouldError: false,
 	},
 	{
 		user:        "0",
 		expectedUID: "0",
-		shouldError: false,
 	},
 	{
 		user:        "fakeUser",
-		expectedUID: "",
-		shouldError: true,
+		expectedUID: "fakeUser",
 	},
 	{
 		user:        "root:root",
 		expectedUID: "root:root",
-		shouldError: false,
 	},
 	{
 		user:        "0:root",
 		expectedUID: "0:root",
-		shouldError: false,
 	},
 	{
 		user:        "root:0",
 		expectedUID: "root:0",
-		shouldError: false,
 	},
 	{
 		user:        "0:0",
 		expectedUID: "0:0",
-		shouldError: false,
-	},
-	{
-		user:        "root:fakeGroup",
-		expectedUID: "",
-		shouldError: true,
 	},
 	{
 		user:        "$envuser",
 		expectedUID: "root",
-		shouldError: false,
 	},
 	{
 		user:        "root:$envgroup",
 		expectedUID: "root:root",
-		shouldError: false,
 	},
 }
 
@@ -97,6 +82,6 @@ func TestUpdateUser(t *testing.T) {
 		}
 		buildArgs := dockerfile.NewBuildArgs([]string{})
 		err := cmd.ExecuteCommand(cfg, buildArgs)
-		testutil.CheckErrorAndDeepEqual(t, test.shouldError, err, test.expectedUID, cfg.User)
+		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedUID, cfg.User)
 	}
 }

pkg/commands/volume.go

@@ -54,7 +54,7 @@ func (v *VolumeCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.
 		if _, err := os.Stat(volume); os.IsNotExist(err) {
 			logrus.Infof("Creating directory %s", volume)
 			if err := os.MkdirAll(volume, 0755); err != nil {
-				return fmt.Errorf("Could not create directory for volume %s: %s", volume, err)
+				return fmt.Errorf("could not create directory for volume %s: %s", volume, err)
 			}
 		}
 	}

pkg/config/options.go

@@ -20,8 +20,15 @@ import (
 	"time"
 )
 
+// CacheOptions are base image cache options that are set by command line arguments
+type CacheOptions struct {
+	CacheDir string
+	CacheTTL time.Duration
+}
+
 // KanikoOptions are options that are set by command line arguments
 type KanikoOptions struct {
+	CacheOptions
 	DockerfilePath          string
 	SrcContext              string
 	SnapshotMode            string
@@ -29,7 +36,8 @@ type KanikoOptions struct {
 	TarPath                 string
 	Target                  string
 	CacheRepo               string
-	CacheDir                string
+	DigestFile              string
+	OCILayoutPath           string
 	Destinations            multiArg
 	BuildArgs               multiArg
 	Insecure                bool
@@ -41,13 +49,13 @@ type KanikoOptions struct {
 	NoPush                  bool
 	Cache                   bool
 	Cleanup                 bool
-	CacheTTL                time.Duration
 	InsecureRegistries      multiArg
 	SkipTLSVerifyRegistries multiArg
 }
 
 // WarmerOptions are options that are set by command line arguments to the cache warmer.
 type WarmerOptions struct {
-	Images   multiArg
-	CacheDir string
+	CacheOptions
+	Images multiArg
+	Force  bool
 }

pkg/config/stage.go

@@ -26,4 +26,5 @@ type KanikoStage struct {
 	BaseImageStoredLocally bool
 	SaveStage              bool
 	MetaArgs               []instructions.ArgCommand
+	Index                  int
 }

pkg/constants/constants.go

@@ -57,6 +57,7 @@ const (
 	GCSBuildContextPrefix      = "gs://"
 	S3BuildContextPrefix       = "s3://"
 	LocalDirBuildContextPrefix = "dir://"
+	GitBuildContextPrefix      = "git://"
 
 	HOME = "HOME"
 	// DefaultHOMEValue is the default value Docker sets for $HOME
@@ -69,6 +70,10 @@ const (
 
 	// Name of the .dockerignore file
 	Dockerignore = ".dockerignore"
+
+	// S3 Custom endpoint ENV name
+	S3EndpointEnv    = "S3_ENDPOINT"
+	S3ForcePathStyle = "S3_FORCE_PATH_STYLE"
 )
 
 // ScratchEnvVars are the default environment variables needed for a scratch image.

pkg/dockerfile/dockerfile.go

@@ -25,6 +25,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
@@ -67,6 +69,7 @@ func Stages(opts *config.KanikoOptions) ([]config.KanikoStage, error) {
 			return nil, errors.Wrap(err, "resolving base name")
 		}
 		stage.Name = resolvedBaseName
+		logrus.Infof("Resolved base name %s to %s", stage.BaseName, stage.Name)
 		kanikoStages = append(kanikoStages, config.KanikoStage{
 			Stage:                  stage,
 			BaseImageIndex:         baseImageIndex(index, stages),
@@ -74,6 +77,7 @@ func Stages(opts *config.KanikoOptions) ([]config.KanikoStage, error) {
 			SaveStage:              saveStage(index, stages),
 			Final:                  index == targetStage,
 			MetaArgs:               metaArgs,
+			Index:                  index,
 		})
 		if index == targetStage {
 			break
@@ -107,7 +111,7 @@ func Parse(b []byte) ([]instructions.Stage, []instructions.ArgCommand, error) {
 	if err != nil {
 		return nil, nil, err
 	}
-	return stages, metaArgs, err
+	return stages, metaArgs, nil
 }
 
 // targetStage returns the index of the target stage kaniko is trying to build
@@ -175,14 +179,6 @@ func saveStage(index int, stages []instructions.Stage) bool {
 				return true
 			}
 		}
-		for _, cmd := range stage.Commands {
-			switch c := cmd.(type) {
-			case *instructions.CopyCommand:
-				if c.From == strconv.Itoa(index) {
-					return true
-				}
-			}
-		}
 	}
 	return false
 }

pkg/dockerfile/dockerfile_test.go

@@ -114,7 +114,7 @@ func Test_SaveStage(t *testing.T) {
 		{
 			name:     "reference stage in later copy command",
 			index:    0,
-			expected: true,
+			expected: false,
 		},
 		{
 			name:     "reference stage in later from command",

pkg/executor/build.go

@@ -23,6 +23,8 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/otiai10/copy"
+
 	"github.com/google/go-containerregistry/pkg/v1/partial"
 
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
@@ -60,10 +62,11 @@ type stageBuilder struct {
 	opts            *config.KanikoOptions
 	cmds            []commands.DockerCommand
 	args            *dockerfile.BuildArgs
+	crossStageDeps  map[int][]string
 }
 
 // newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage
-func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*stageBuilder, error) {
+func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string) (*stageBuilder, error) {
 	sourceImage, err := util.RetrieveSourceImage(stage, opts)
 	if err != nil {
 		return nil, err
@@ -96,6 +99,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*sta
 		snapshotter:     snapshotter,
 		baseImageDigest: digest.String(),
 		opts:            opts,
+		crossStageDeps:  crossStageDeps,
 	}
 
 	for _, cmd := range s.stage.Commands {
@@ -120,7 +124,7 @@ func initializeConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) {
 		return nil, err
 	}
 
-	if img == empty.Image {
+	if imageConfig.Config.Env == nil {
 		imageConfig.Config.Env = constants.ScratchEnvVars
 	}
 	return imageConfig, nil
@@ -186,11 +190,7 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro
 
 func (s *stageBuilder) build() error {
 	// Set the initial cache key to be the base image digest, the build args and the SrcContext.
-	dgst, err := util.ReproducibleDigest(s.image)
-	if err != nil {
-		return err
-	}
-	compositeKey := NewCompositeCache(dgst)
+	compositeKey := NewCompositeCache(s.baseImageDigest)
 	compositeKey.AddKey(s.opts.BuildArgs...)
 
 	// Apply optimizations to the instructions.
@@ -207,6 +207,10 @@ func (s *stageBuilder) build() error {
 			break
 		}
 	}
+	if len(s.crossStageDeps[s.stage.Index]) > 0 {
+		shouldUnpack = true
+	}
+
 	if shouldUnpack {
 		t := timing.Start("FS Unpacking")
 		if _, err := util.GetFSFromImage(constants.RootDir, s.image); err != nil {
@@ -353,6 +357,68 @@ func (s *stageBuilder) saveSnapshotToImage(createdBy string, tarPath string) err
 
 }
 
+func CalculateDependencies(opts *config.KanikoOptions) (map[int][]string, error) {
+	stages, err := dockerfile.Stages(opts)
+	if err != nil {
+		return nil, err
+	}
+	images := []v1.Image{}
+	depGraph := map[int][]string{}
+	for _, s := range stages {
+		ba := dockerfile.NewBuildArgs(opts.BuildArgs)
+		ba.AddMetaArgs(s.MetaArgs)
+		var image v1.Image
+		var err error
+		if s.BaseImageStoredLocally {
+			image = images[s.BaseImageIndex]
+		} else if s.Name == constants.NoBaseImage {
+			image = empty.Image
+		} else {
+			image, err = util.RetrieveSourceImage(s, opts)
+			if err != nil {
+				return nil, err
+			}
+		}
+		cfg, err := initializeConfig(image)
+		if err != nil {
+			return nil, err
+		}
+		for _, c := range s.Commands {
+			switch cmd := c.(type) {
+			case *instructions.CopyCommand:
+				if cmd.From != "" {
+					i, err := strconv.Atoi(cmd.From)
+					if err != nil {
+						continue
+					}
+					resolved, err := util.ResolveEnvironmentReplacementList(cmd.SourcesAndDest, ba.ReplacementEnvs(cfg.Config.Env), true)
+					if err != nil {
+						return nil, err
+					}
+
+					depGraph[i] = append(depGraph[i], resolved[0:len(resolved)-1]...)
+				}
+			case *instructions.EnvCommand:
+				if err := util.UpdateConfigEnv(cmd.Env, &cfg.Config, ba.ReplacementEnvs(cfg.Config.Env)); err != nil {
+					return nil, err
+				}
+				image, err = mutate.Config(image, cfg.Config)
+				if err != nil {
+					return nil, err
+				}
+			case *instructions.ArgCommand:
+				k, v, err := commands.ParseArg(cmd.Key, cmd.Value, cfg.Config.Env, ba)
+				if err != nil {
+					return nil, err
+				}
+				ba.AddArg(k, v)
+			}
+		}
+		images = append(images, image)
+	}
+	return depGraph, nil
+}
+
 // DoBuild executes building the Dockerfile
 func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 	t := timing.Start("Total Build Time")
@@ -361,7 +427,7 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 	if err != nil {
 		return nil, err
 	}
-	if err := util.GetExcludedFiles(opts.SrcContext); err != nil {
+	if err := util.GetExcludedFiles(opts.DockerfilePath, opts.SrcContext); err != nil {
 		return nil, err
 	}
 	// Some stages may refer to other random images, not previous stages
@@ -369,8 +435,14 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 		return nil, err
 	}
 
+	crossStageDependencies, err := CalculateDependencies(opts)
+	if err != nil {
+		return nil, err
+	}
+	logrus.Infof("Built cross stage deps: %v", crossStageDependencies)
+
 	for index, stage := range stages {
-		sb, err := newStageBuilder(opts, stage)
+		sb, err := newStageBuilder(opts, stage, crossStageDependencies)
 		if err != nil {
 			return nil, err
 		}
@@ -405,10 +477,21 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 			if err := saveStageAsTarball(strconv.Itoa(index), sourceImage); err != nil {
 				return nil, err
 			}
-			if err := extractImageToDependecyDir(strconv.Itoa(index), sourceImage); err != nil {
-				return nil, err
-			}
 		}
+
+		filesToSave, err := filesToSave(crossStageDependencies[index])
+		if err != nil {
+			return nil, err
+		}
+		dstDir := filepath.Join(constants.KanikoDir, strconv.Itoa(index))
+		if err := os.MkdirAll(dstDir, 0644); err != nil {
+			return nil, err
+		}
+		for _, p := range filesToSave {
+			logrus.Infof("Saving file %s for later use.", p)
+			copy.Copy(p, filepath.Join(dstDir, p))
+		}
+
 		// Delete the filesystem
 		if err := util.DeleteFilesystem(); err != nil {
 			return nil, err
@@ -418,6 +501,18 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 	return nil, err
 }
 
+func filesToSave(deps []string) ([]string, error) {
+	allFiles := []string{}
+	for _, src := range deps {
+		srcs, err := filepath.Glob(src)
+		if err != nil {
+			return nil, err
+		}
+		allFiles = append(allFiles, srcs...)
+	}
+	return allFiles, nil
+}
+
 func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) error {
 	t := timing.Start("Fetching Extra Stages")
 	defer timing.DefaultRun.Stop(t)
@@ -453,7 +548,7 @@ func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) e
 			if err := saveStageAsTarball(c.From, sourceImage); err != nil {
 				return err
 			}
-			if err := extractImageToDependecyDir(c.From, sourceImage); err != nil {
+			if err := extractImageToDependencyDir(c.From, sourceImage); err != nil {
 				return err
 			}
 		}
@@ -464,7 +559,7 @@ func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) e
 	}
 	return nil
 }
-func extractImageToDependecyDir(name string, image v1.Image) error {
+func extractImageToDependencyDir(name string, image v1.Image) error {
 	t := timing.Start("Extracting Image to Dependency Dir")
 	defer timing.DefaultRun.Stop(t)
 	dependencyDir := filepath.Join(constants.KanikoDir, name)

pkg/executor/build_test.go

@@ -17,14 +17,21 @@ limitations under the License.
 package executor
 
 import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
 	"testing"
 
-	"github.com/moby/buildkit/frontend/dockerfile/instructions"
-
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
 	"github.com/GoogleContainerTools/kaniko/testutil"
-	"github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-cmp/cmp"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 
 func Test_reviewConfig(t *testing.T) {
@@ -180,3 +187,278 @@ func Test_stageBuilder_shouldTakeSnapshot(t *testing.T) {
 		})
 	}
 }
+
+func TestCalculateDependencies(t *testing.T) {
+	type args struct {
+		dockerfile string
+	}
+	tests := []struct {
+		name string
+		args args
+		want map[int][]string
+	}{
+		{
+			name: "no deps",
+			args: args{
+				dockerfile: `
+FROM debian as stage1
+RUN foo
+FROM stage1
+RUN bar
+`,
+			},
+			want: map[int][]string{},
+		},
+		{
+			name: "args",
+			args: args{
+				dockerfile: `
+ARG myFile=foo
+FROM debian as stage1
+RUN foo
+FROM stage1
+ARG myFile
+COPY --from=stage1 /tmp/$myFile.txt .
+RUN bar
+`,
+			},
+			want: map[int][]string{
+				0: {"/tmp/foo.txt"},
+			},
+		},
+		{
+			name: "simple deps",
+			args: args{
+				dockerfile: `
+FROM debian as stage1
+FROM alpine
+COPY --from=stage1 /foo /bar
+`,
+			},
+			want: map[int][]string{
+				0: {"/foo"},
+			},
+		},
+		{
+			name: "two sets deps",
+			args: args{
+				dockerfile: `
+FROM debian as stage1
+FROM ubuntu as stage2
+RUN foo
+COPY --from=stage1 /foo /bar
+FROM alpine
+COPY --from=stage2 /bar /bat
+`,
+			},
+			want: map[int][]string{
+				0: {"/foo"},
+				1: {"/bar"},
+			},
+		},
+		{
+			name: "double deps",
+			args: args{
+				dockerfile: `
+FROM debian as stage1
+FROM ubuntu as stage2
+RUN foo
+COPY --from=stage1 /foo /bar
+FROM alpine
+COPY --from=stage1 /baz /bat
+`,
+			},
+			want: map[int][]string{
+				0: {"/foo", "/baz"},
+			},
+		},
+		{
+			name: "envs in deps",
+			args: args{
+				dockerfile: `
+FROM debian as stage1
+FROM ubuntu as stage2
+RUN foo
+ENV key1 val1
+ENV key2 val2
+COPY --from=stage1 /foo/$key1 /foo/$key2 /bar
+FROM alpine
+COPY --from=stage2 /bar /bat
+`,
+			},
+			want: map[int][]string{
+				0: {"/foo/val1", "/foo/val2"},
+				1: {"/bar"},
+			},
+		},
+		{
+			name: "envs from base image in deps",
+			args: args{
+				dockerfile: `
+FROM debian as stage1
+ENV key1 baseval1
+FROM stage1 as stage2
+RUN foo
+ENV key2 val2
+COPY --from=stage1 /foo/$key1 /foo/$key2 /bar
+FROM alpine
+COPY --from=stage2 /bar /bat
+`,
+			},
+			want: map[int][]string{
+				0: {"/foo/baseval1", "/foo/val2"},
+				1: {"/bar"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f, _ := ioutil.TempFile("", "")
+			ioutil.WriteFile(f.Name(), []byte(tt.args.dockerfile), 0755)
+			opts := &config.KanikoOptions{
+				DockerfilePath: f.Name(),
+			}
+
+			got, err := CalculateDependencies(opts)
+			if err != nil {
+				t.Errorf("got error: %s,", err)
+			}
+
+			if !reflect.DeepEqual(got, tt.want) {
+				diff := cmp.Diff(got, tt.want)
+				t.Errorf("CalculateDependencies() = %v, want %v, diff %v", got, tt.want, diff)
+			}
+		})
+	}
+}
+
+func Test_filesToSave(t *testing.T) {
+	tests := []struct {
+		name  string
+		args  []string
+		want  []string
+		files []string
+	}{
+		{
+			name:  "simple",
+			args:  []string{"foo"},
+			files: []string{"foo"},
+			want:  []string{"foo"},
+		},
+		{
+			name:  "glob",
+			args:  []string{"foo*"},
+			files: []string{"foo", "foo2", "fooooo", "bar"},
+			want:  []string{"foo", "foo2", "fooooo"},
+		},
+		{
+			name:  "complex glob",
+			args:  []string{"foo*", "bar?"},
+			files: []string{"foo", "foo2", "fooooo", "bar", "bar1", "bar2", "bar33"},
+			want:  []string{"foo", "foo2", "fooooo", "bar1", "bar2"},
+		},
+		{
+			name:  "dir",
+			args:  []string{"foo"},
+			files: []string{"foo/bar", "foo/baz", "foo/bat/baz"},
+			want:  []string{"foo"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tmpDir, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Errorf("error creating tmpdir: %s", err)
+			}
+			defer os.RemoveAll(tmpDir)
+
+			for _, f := range tt.files {
+				p := filepath.Join(tmpDir, f)
+				dir := filepath.Dir(p)
+				if dir != "." {
+					if err := os.MkdirAll(dir, 0755); err != nil {
+						t.Errorf("error making dir: %s", err)
+					}
+				}
+				fp, err := os.Create(p)
+				if err != nil {
+					t.Errorf("error making file: %s", err)
+				}
+				fp.Close()
+			}
+
+			args := []string{}
+			for _, arg := range tt.args {
+				args = append(args, filepath.Join(tmpDir, arg))
+			}
+			got, err := filesToSave(args)
+			if err != nil {
+				t.Errorf("got err: %s", err)
+			}
+			want := []string{}
+			for _, w := range tt.want {
+				want = append(want, filepath.Join(tmpDir, w))
+			}
+			sort.Strings(want)
+			sort.Strings(got)
+			if !reflect.DeepEqual(got, want) {
+				t.Errorf("filesToSave() = %v, want %v", got, want)
+			}
+		})
+	}
+}
+
+func TestInitializeConfig(t *testing.T) {
+	tests := []struct {
+		description string
+		cfg         v1.ConfigFile
+		expected    v1.Config
+	}{
+		{
+			description: "env is not set in the image",
+			cfg: v1.ConfigFile{
+				Config: v1.Config{
+					Image: "test",
+				},
+			},
+			expected: v1.Config{
+				Image: "test",
+				Env: []string{
+					"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+				},
+			},
+		},
+		{
+			description: "env is set in the image",
+			cfg: v1.ConfigFile{
+				Config: v1.Config{
+					Env: []string{
+						"PATH=/usr/local/something",
+					},
+				},
+			},
+			expected: v1.Config{
+				Env: []string{
+					"PATH=/usr/local/something",
+				},
+			},
+		},
+		{
+			description: "image is empty",
+			expected: v1.Config{
+				Env: []string{
+					"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		img, err := mutate.ConfigFile(empty.Image, &tt.cfg)
+		if err != nil {
+			t.Errorf("error seen when running test %s", err)
+			t.Fail()
+		}
+		actual, _ := initializeConfig(img)
+		testutil.CheckDeepEqual(t, tt.expected, actual.Config)
+	}
+}

pkg/executor/push.go

@@ -20,9 +20,11 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/cache"
@@ -34,6 +36,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
@@ -46,14 +49,79 @@ type withUserAgent struct {
 	t http.RoundTripper
 }
 
+const (
+	UpstreamClientUaKey = "UPSTREAM_CLIENT_TYPE"
+)
+
 func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) {
-	r.Header.Set("User-Agent", fmt.Sprintf("kaniko/%s", version.Version()))
+	ua := []string{fmt.Sprintf("kaniko/%s", version.Version())}
+	if upstream := os.Getenv(UpstreamClientUaKey); upstream != "" {
+		ua = append(ua, upstream)
+	}
+	r.Header.Set("User-Agent", strings.Join(ua, ","))
 	return w.t.RoundTrip(r)
 }
 
+// CheckPushPermissions checks that the configured credentials can be used to
+// push to every specified destination.
+func CheckPushPermissions(opts *config.KanikoOptions) error {
+	if opts.NoPush {
+		return nil
+	}
+
+	checked := map[string]bool{}
+	for _, destination := range opts.Destinations {
+		destRef, err := name.NewTag(destination, name.WeakValidation)
+		if err != nil {
+			return errors.Wrap(err, "getting tag for destination")
+		}
+		if checked[destRef.Context().RepositoryStr()] {
+			continue
+		}
+
+		registryName := destRef.Repository.Registry.Name()
+		if opts.Insecure || opts.InsecureRegistries.Contains(registryName) {
+			newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
+			if err != nil {
+				return errors.Wrap(err, "getting new insecure registry")
+			}
+			destRef.Repository.Registry = newReg
+		}
+		tr := makeTransport(opts, registryName)
+		if err := remote.CheckPushPermission(destRef, creds.GetKeychain(), tr); err != nil {
+			return errors.Wrapf(err, "checking push permission for %q", destRef)
+		}
+		checked[destRef.Context().RepositoryStr()] = true
+	}
+	return nil
+}
+
 // DoPush is responsible for pushing image to the destinations specified in opts
 func DoPush(image v1.Image, opts *config.KanikoOptions) error {
 	t := timing.Start("Total Push Time")
+
+	if opts.DigestFile != "" {
+		digest, err := image.Digest()
+		if err != nil {
+			return errors.Wrap(err, "error fetching digest")
+		}
+		digestByteArray := []byte(digest.String())
+		err = ioutil.WriteFile(opts.DigestFile, digestByteArray, 0644)
+		if err != nil {
+			return errors.Wrap(err, "writing digest to file failed")
+		}
+	}
+
+	if opts.OCILayoutPath != "" {
+		path, err := layout.Write(opts.OCILayoutPath, empty.Index)
+		if err != nil {
+			return errors.Wrap(err, "writing empty layout")
+		}
+		if err := path.AppendImage(image); err != nil {
+			return errors.Wrap(err, "appending image")
+		}
+	}
+
 	destRefs := []name.Tag{}
 	for _, destination := range opts.Destinations {
 		destRef, err := name.NewTag(destination, name.WeakValidation)
@@ -80,7 +148,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error {
 	for _, destRef := range destRefs {
 		registryName := destRef.Repository.Registry.Name()
 		if opts.Insecure || opts.InsecureRegistries.Contains(registryName) {
-			newReg, err := name.NewInsecureRegistry(registryName, name.WeakValidation)
+			newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
 			if err != nil {
 				return errors.Wrap(err, "getting new insecure registry")
 			}
@@ -92,16 +160,10 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error {
 			return errors.Wrap(err, "resolving pushAuth")
 		}
 
-		// Create a transport to set our user-agent.
-		tr := http.DefaultTransport
-		if opts.SkipTLSVerify || opts.SkipTLSVerifyRegistries.Contains(registryName) {
-			tr.(*http.Transport).TLSClientConfig = &tls.Config{
-				InsecureSkipVerify: true,
-			}
-		}
+		tr := makeTransport(opts, registryName)
 		rt := &withUserAgent{t: tr}
 
-		if err := remote.Write(destRef, image, pushAuth, rt); err != nil {
+		if err := remote.Write(destRef, image, remote.WithAuth(pushAuth), remote.WithTransport(rt)); err != nil {
 			return errors.Wrap(err, fmt.Sprintf("failed to push to destination %s", destRef))
 		}
 	}
@@ -142,6 +204,17 @@ func writeImageOutputs(image v1.Image, destRefs []name.Tag) error {
 	return nil
 }
 
+func makeTransport(opts *config.KanikoOptions, registryName string) http.RoundTripper {
+	// Create a transport to set our user-agent.
+	tr := http.DefaultTransport
+	if opts.SkipTLSVerify || opts.SkipTLSVerifyRegistries.Contains(registryName) {
+		tr.(*http.Transport).TLSClientConfig = &tls.Config{
+			InsecureSkipVerify: true,
+		}
+	}
+	return tr
+}
+
 // pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache
 // if opts.Cache doesn't exist, infer the cache from the given destination
 func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath string, createdBy string) error {

pkg/executor/push_test.go

@@ -1,9 +1,12 @@
 /*
 Copyright 2018 Google LLC
+
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
+
     http://www.apache.org/licenses/LICENSE-2.0
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -14,6 +17,7 @@ limitations under the License.
 package executor
 
 import (
+
 	"fmt"
 	"os"
 	"path/filepath"
@@ -91,4 +95,106 @@ func TestWriteImageOutputs(t *testing.T) {
 			}
 		})
 	}
+=======
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"testing"
+
+	"github.com/GoogleContainerTools/kaniko/pkg/config"
+	"github.com/GoogleContainerTools/kaniko/testutil"
+	"github.com/google/go-containerregistry/pkg/v1/layout"
+	"github.com/google/go-containerregistry/pkg/v1/random"
+	"github.com/google/go-containerregistry/pkg/v1/validate"
+)
+
+func TestHeaderAdded(t *testing.T) {
+	tests := []struct {
+		name     string
+		upstream string
+		expected string
+	}{{
+		name:     "upstream env variable set",
+		upstream: "skaffold-v0.25.45",
+		expected: "kaniko/unset,skaffold-v0.25.45",
+	}, {
+		name:     "upstream env variable not set",
+		expected: "kaniko/unset",
+	},
+	}
+	for _, test := range tests {
+
+		t.Run(test.name, func(t *testing.T) {
+			rt := &withUserAgent{t: &mockRoundTripper{}}
+			if test.upstream != "" {
+				os.Setenv("UPSTREAM_CLIENT_TYPE", test.upstream)
+				defer func() { os.Unsetenv("UPSTREAM_CLIENT_TYPE") }()
+			}
+			req, err := http.NewRequest("GET", "dummy", nil)
+			if err != nil {
+				t.Fatalf("culd not create a req due to %s", err)
+			}
+			resp, err := rt.RoundTrip(req)
+			testutil.CheckError(t, false, err)
+			defer resp.Body.Close()
+			body, err := ioutil.ReadAll(resp.Body)
+			testutil.CheckErrorAndDeepEqual(t, false, err, test.expected, string(body))
+		})
+	}
+
+}
+
+type mockRoundTripper struct {
+}
+
+func (m *mockRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+	ua := r.UserAgent()
+	return &http.Response{Body: ioutil.NopCloser(bytes.NewBufferString(ua))}, nil
+}
+
+func TestOCILayoutPath(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatalf("could not create temp dir: %s", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	image, err := random.Image(1024, 4)
+	if err != nil {
+		t.Fatalf("could not create image: %s", err)
+	}
+
+	digest, err := image.Digest()
+	if err != nil {
+		t.Fatalf("could not get image digest: %s", err)
+	}
+
+	want, err := image.Manifest()
+	if err != nil {
+		t.Fatalf("could not get image manifest: %s", err)
+	}
+
+	opts := config.KanikoOptions{
+		NoPush:        true,
+		OCILayoutPath: tmpDir,
+	}
+
+	if err := DoPush(image, &opts); err != nil {
+		t.Fatalf("could not push image: %s", err)
+	}
+
+	layoutIndex, err := layout.ImageIndexFromPath(tmpDir)
+	if err != nil {
+		t.Fatalf("could not get index from layout: %s", err)
+	}
+	testutil.CheckError(t, false, validate.Index(layoutIndex))
+
+	layoutImage, err := layoutIndex.Image(digest)
+	if err != nil {
+		t.Fatalf("could not get image from layout: %s", err)
+	}
+
+	got, err := layoutImage.Manifest()
+	testutil.CheckErrorAndDeepEqual(t, false, err, want, got)
 }

pkg/snapshot/layered_map.go

@@ -103,17 +103,16 @@ func (l *LayeredMap) Add(s string) error {
 	// Use hash function and add to layers
 	newV, err := l.hasher(s)
 	if err != nil {
-		return fmt.Errorf("Error creating hash for %s: %v", s, err)
+		return fmt.Errorf("error creating hash for %s: %v", s, err)
 	}
 	l.layers[len(l.layers)-1][s] = newV
 	return nil
 }
 
-// MaybeAdd will add the specified file s to the layered map if
-// the layered map's hashing function determines it has changed. If
-// it has not changed, it will not be added. Returns true if the file
-// was added.
-func (l *LayeredMap) MaybeAdd(s string) (bool, error) {
+// CheckFileChange checks whether a given file changed
+// from the current layered map by its hashing function.
+// Returns true if the file is changed.
+func (l *LayeredMap) CheckFileChange(s string) (bool, error) {
 	oldV, ok := l.Get(s)
 	t := timing.Start("Hashing files")
 	defer timing.DefaultRun.Stop(t)
@@ -124,6 +123,5 @@ func (l *LayeredMap) MaybeAdd(s string) (bool, error) {
 	if ok && newV == oldV {
 		return false, nil
 	}
-	l.layers[len(l.layers)-1][s] = newV
 	return true, nil
 }

pkg/snapshot/snapshot.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"path/filepath"
+	"sort"
 	"syscall"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/timing"
@@ -73,44 +74,15 @@ func (s *Snapshotter) TakeSnapshot(files []string) (string, error) {
 	}
 	logrus.Info("Taking snapshot of files...")
 	logrus.Debugf("Taking snapshot of files %v", files)
-	snapshottedFiles := make(map[string]bool)
 
-	// First add to the tar any parent directories that haven't been added
-	parentDirs := map[string]struct{}{}
-	for _, file := range files {
-		for _, p := range util.ParentDirectories(file) {
-			parentDirs[p] = struct{}{}
-		}
-	}
-	filesToAdd := []string{}
-	for file := range parentDirs {
-		file = filepath.Clean(file)
-		snapshottedFiles[file] = true
-
-		// The parent directory might already be in a previous layer.
-		fileAdded, err := s.l.MaybeAdd(file)
-		if err != nil {
-			return "", fmt.Errorf("Unable to add parent dir %s to layered map: %s", file, err)
-		}
-
-		if fileAdded {
-			filesToAdd = append(filesToAdd, file)
-		}
-	}
-
-	// Next add the files themselves to the tar
-	for _, file := range files {
-		// We might have already added the file above as a parent directory of another file.
-		file = filepath.Clean(file)
-		if _, ok := snapshottedFiles[file]; ok {
-			continue
-		}
-		snapshottedFiles[file] = true
+	// Also add parent directories to keep the permission of them correctly.
+	filesToAdd := filesWithParentDirs(files)
 
+	// Add files to the layered map
+	for _, file := range filesToAdd {
 		if err := s.l.Add(file); err != nil {
-			return "", fmt.Errorf("Unable to add file %s to layered map: %s", file, err)
+			return "", fmt.Errorf("unable to add file %s to layered map: %s", file, err)
 		}
-		filesToAdd = append(filesToAdd, file)
 	}
 
 	t := util.NewTar(f)
@@ -201,16 +173,29 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
 			logrus.Debugf("Not adding %s to layer, as it's whitelisted", path)
 			continue
 		}
-		// Only add to the tar if we add it to the layeredmap.
-		maybeAdd, err := s.l.MaybeAdd(path)
+		// Only add changed files.
+		fileChanged, err := s.l.CheckFileChange(path)
 		if err != nil {
 			return nil, nil, err
 		}
-		if maybeAdd {
+		if fileChanged {
 			logrus.Debugf("Adding %s to layer, because it was changed.", path)
 			filesToAdd = append(filesToAdd, path)
 		}
 	}
+
+	// Also add parent directories to keep their permissions correctly.
+	filesToAdd = filesWithParentDirs(filesToAdd)
+
+	sort.Strings(filesToAdd)
+
+	// Add files to the layered map
+	for _, file := range filesToAdd {
+		if err := s.l.Add(file); err != nil {
+			return nil, nil, fmt.Errorf("unable to add file %s to layered map: %s", file, err)
+		}
+	}
+
 	return filesToAdd, filesToWhiteOut, nil
 }
 
@@ -230,3 +215,24 @@ func writeToTar(t util.Tar, files, whiteouts []string) error {
 	}
 	return nil
 }
+
+func filesWithParentDirs(files []string) []string {
+	filesSet := map[string]bool{}
+
+	for _, file := range files {
+		file = filepath.Clean(file)
+		filesSet[file] = true
+
+		for _, dir := range util.ParentDirectories(file) {
+			dir = filepath.Clean(dir)
+			filesSet[dir] = true
+		}
+	}
+
+	newFiles := []string{}
+	for file := range filesSet {
+		newFiles = append(newFiles, file)
+	}
+
+	return newFiles
+}

pkg/snapshot/snapshot_test.go

@@ -21,6 +21,8 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
+	"strings"
 	"testing"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
@@ -30,6 +32,7 @@ import (
 
 func TestSnapshotFSFileChange(t *testing.T) {
 	testDir, snapshotter, cleanup, err := setUpTestDir()
+	testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/")
 	defer cleanup()
 	if err != nil {
 		t.Fatal(err)
@@ -54,12 +57,18 @@ func TestSnapshotFSFileChange(t *testing.T) {
 	}
 	// Check contents of the snapshot, make sure contents is equivalent to snapshotFiles
 	tr := tar.NewReader(f)
-	fooPath := filepath.Join(testDir, "foo")
-	batPath := filepath.Join(testDir, "bar/bat")
+	fooPath := filepath.Join(testDirWithoutLeadingSlash, "foo")
+	batPath := filepath.Join(testDirWithoutLeadingSlash, "bar/bat")
 	snapshotFiles := map[string]string{
 		fooPath: "newbaz1",
 		batPath: "baz",
 	}
+	for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(fooPath) {
+		snapshotFiles[dir] = ""
+	}
+	for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(batPath) {
+		snapshotFiles[dir] = ""
+	}
 	numFiles := 0
 	for {
 		hdr, err := tr.Next()
@@ -75,19 +84,60 @@ func TestSnapshotFSFileChange(t *testing.T) {
 			t.Fatalf("Contents of %s incorrect, expected: %s, actual: %s", hdr.Name, snapshotFiles[hdr.Name], string(contents))
 		}
 	}
-	if numFiles != 2 {
+	if numFiles != len(snapshotFiles) {
 		t.Fatalf("Incorrect number of files were added, expected: 2, actual: %v", numFiles)
 	}
 }
 
+func TestSnapshotFSIsReproducible(t *testing.T) {
+	testDir, snapshotter, cleanup, err := setUpTestDir()
+	defer cleanup()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Make some changes to the filesystem
+	newFiles := map[string]string{
+		"foo":     "newbaz1",
+		"bar/bat": "baz",
+	}
+	if err := testutil.SetupFiles(testDir, newFiles); err != nil {
+		t.Fatalf("Error setting up fs: %s", err)
+	}
+	// Take another snapshot
+	tarPath, err := snapshotter.TakeSnapshotFS()
+	if err != nil {
+		t.Fatalf("Error taking snapshot of fs: %s", err)
+	}
+
+	f, err := os.Open(tarPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Check contents of the snapshot, make sure contents are sorted by name
+	tr := tar.NewReader(f)
+	var filesInTar []string
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		filesInTar = append(filesInTar, hdr.Name)
+	}
+	if !sort.StringsAreSorted(filesInTar) {
+		t.Fatalf("Expected the file in the tar archive were sorted, actual list was not sorted: %v", filesInTar)
+	}
+}
+
 func TestSnapshotFSChangePermissions(t *testing.T) {
 	testDir, snapshotter, cleanup, err := setUpTestDir()
+	testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/")
 	defer cleanup()
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Change permissions on a file
 	batPath := filepath.Join(testDir, "bar/bat")
+	batPathWithoutLeadingSlash := filepath.Join(testDirWithoutLeadingSlash, "bar/bat")
 	if err := os.Chmod(batPath, 0600); err != nil {
 		t.Fatalf("Error changing permissions on %s: %v", batPath, err)
 	}
@@ -103,7 +153,10 @@ func TestSnapshotFSChangePermissions(t *testing.T) {
 	// Check contents of the snapshot, make sure contents is equivalent to snapshotFiles
 	tr := tar.NewReader(f)
 	snapshotFiles := map[string]string{
-		batPath: "baz2",
+		batPathWithoutLeadingSlash: "baz2",
+	}
+	for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(batPath) {
+		snapshotFiles[dir] = ""
 	}
 	numFiles := 0
 	for {
@@ -111,6 +164,7 @@ func TestSnapshotFSChangePermissions(t *testing.T) {
 		if err == io.EOF {
 			break
 		}
+		t.Logf("Info %s in tar", hdr.Name)
 		numFiles++
 		if _, isFile := snapshotFiles[hdr.Name]; !isFile {
 			t.Fatalf("File %s unexpectedly in tar", hdr.Name)
@@ -120,13 +174,14 @@ func TestSnapshotFSChangePermissions(t *testing.T) {
 			t.Fatalf("Contents of %s incorrect, expected: %s, actual: %s", hdr.Name, snapshotFiles[hdr.Name], string(contents))
 		}
 	}
-	if numFiles != 1 {
+	if numFiles != len(snapshotFiles) {
 		t.Fatalf("Incorrect number of files were added, expected: 1, got: %v", numFiles)
 	}
 }
 
 func TestSnapshotFiles(t *testing.T) {
 	testDir, snapshotter, cleanup, err := setUpTestDir()
+	testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/")
 	defer cleanup()
 	if err != nil {
 		t.Fatal(err)
@@ -147,7 +202,10 @@ func TestSnapshotFiles(t *testing.T) {
 	}
 	defer os.Remove(tarPath)
 
-	expectedFiles := []string{"/", "/tmp", filepath.Join(testDir, "foo")}
+	expectedFiles := []string{
+		filepath.Join(testDirWithoutLeadingSlash, "foo"),
+	}
+	expectedFiles = append(expectedFiles, util.ParentDirectoriesWithoutLeadingSlash(filepath.Join(testDir, "foo"))...)
 
 	f, err := os.Open(tarPath)
 	if err != nil {
@@ -166,6 +224,8 @@ func TestSnapshotFiles(t *testing.T) {
 		}
 		actualFiles = append(actualFiles, hdr.Name)
 	}
+	sort.Strings(expectedFiles)
+	sort.Strings(actualFiles)
 	testutil.CheckErrorAndDeepEqual(t, false, nil, expectedFiles, actualFiles)
 }
 

pkg/util/command_util.go

@@ -37,10 +37,6 @@ import (
 func ResolveEnvironmentReplacementList(values, envs []string, isFilepath bool) ([]string, error) {
 	var resolvedValues []string
 	for _, value := range values {
-		if IsSrcRemoteFileURL(value) {
-			resolvedValues = append(resolvedValues, value)
-			continue
-		}
 		resolved, err := ResolveEnvironmentReplacement(value, envs, isFilepath)
 		logrus.Debugf("Resolved %s to %s", value, resolved)
 		if err != nil {
@@ -53,7 +49,7 @@ func ResolveEnvironmentReplacementList(values, envs []string, isFilepath bool) (
 
 // ResolveEnvironmentReplacement resolves replacing env variables in some text from envs
 // It takes in a string representation of the command, the value to be resolved, and a list of envs (config.Env)
-// Ex: fp = $foo/newdir, envs = [foo=/foodir], then this should return /foodir/newdir
+// Ex: value = $foo/newdir, envs = [foo=/foodir], then this should return /foodir/newdir
 // The dockerfile/shell package handles processing env values
 // It handles escape characters and supports expansion from the config.Env array
 // Shlex handles some of the following use cases (these and more are tested in integration tests)
@@ -63,7 +59,8 @@ func ResolveEnvironmentReplacementList(values, envs []string, isFilepath bool) (
 func ResolveEnvironmentReplacement(value string, envs []string, isFilepath bool) (string, error) {
 	shlex := shell.NewLex(parser.DefaultEscapeToken)
 	fp, err := shlex.ProcessWord(value, envs)
-	if !isFilepath {
+	// Check after replacement if value is a remote URL
+	if !isFilepath || IsSrcRemoteFileURL(fp) {
 		return fp, err
 	}
 	if err != nil {
@@ -76,6 +73,22 @@ func ResolveEnvironmentReplacement(value string, envs []string, isFilepath bool)
 	return fp, nil
 }
 
+func ResolveEnvAndWildcards(sd instructions.SourcesAndDest, buildcontext string, envs []string) ([]string, string, error) {
+	// First, resolve any environment replacement
+	resolvedEnvs, err := ResolveEnvironmentReplacementList(sd, envs, true)
+	if err != nil {
+		return nil, "", err
+	}
+	dest := resolvedEnvs[len(resolvedEnvs)-1]
+	// Resolve wildcards and get a list of resolved sources
+	srcs, err := ResolveSources(resolvedEnvs[0:len(resolvedEnvs)-1], buildcontext)
+	if err != nil {
+		return nil, "", err
+	}
+	err = IsSrcsValid(sd, srcs, buildcontext)
+	return srcs, dest, err
+}
+
 // ContainsWildcards returns true if any entry in paths contains wildcards
 func ContainsWildcards(paths []string) bool {
 	for _, path := range paths {
@@ -88,23 +101,22 @@ func ContainsWildcards(paths []string) bool {
 
 // ResolveSources resolves the given sources if the sources contains wildcards
 // It returns a list of resolved sources
-func ResolveSources(srcsAndDest instructions.SourcesAndDest, root string) ([]string, error) {
-	srcs := srcsAndDest[:len(srcsAndDest)-1]
+func ResolveSources(srcs []string, root string) ([]string, error) {
 	// If sources contain wildcards, we first need to resolve them to actual paths
-	if ContainsWildcards(srcs) {
-		logrus.Debugf("Resolving srcs %v...", srcs)
-		files, err := RelativeFiles("", root)
-		if err != nil {
-			return nil, err
-		}
-		srcs, err = matchSources(srcs, files)
-		if err != nil {
-			return nil, err
-		}
-		logrus.Debugf("Resolved sources to %v", srcs)
+	if !ContainsWildcards(srcs) {
+		return srcs, nil
 	}
-	// Check to make sure the sources are valid
-	return srcs, IsSrcsValid(srcsAndDest, srcs, root)
+	logrus.Infof("Resolving srcs %v...", srcs)
+	files, err := RelativeFiles("", root)
+	if err != nil {
+		return nil, err
+	}
+	resolved, err := matchSources(srcs, files)
+	if err != nil {
+		return nil, err
+	}
+	logrus.Debugf("Resolved sources to %v", resolved)
+	return resolved, nil
 }
 
 // matchSources returns a list of sources that match wildcards
@@ -165,20 +177,24 @@ func DestinationFilepath(src, dest, cwd string) (string, error) {
 }
 
 // URLDestinationFilepath gives the destination a file from a remote URL should be saved to
-func URLDestinationFilepath(rawurl, dest, cwd string) string {
+func URLDestinationFilepath(rawurl, dest, cwd string, envs []string) (string, error) {
 	if !IsDestDir(dest) {
 		if !filepath.IsAbs(dest) {
-			return filepath.Join(cwd, dest)
+			return filepath.Join(cwd, dest), nil
 		}
-		return dest
+		return dest, nil
 	}
 	urlBase := filepath.Base(rawurl)
+	urlBase, err := ResolveEnvironmentReplacement(urlBase, envs, true)
+	if err != nil {
+		return "", err
+	}
 	destPath := filepath.Join(dest, urlBase)
 
 	if !filepath.IsAbs(dest) {
 		destPath = filepath.Join(cwd, destPath)
 	}
-	return destPath
+	return destPath, nil
 }
 
 func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []string, root string) error {
@@ -304,13 +320,12 @@ func GetUserFromUsername(userStr string, groupStr string) (string, string, error
 	// Lookup by username
 	userObj, err := user.Lookup(userStr)
 	if err != nil {
-		if _, ok := err.(user.UnknownUserError); ok {
-			// Lookup by id
-			userObj, err = user.LookupId(userStr)
-			if err != nil {
-				return "", "", err
-			}
-		} else {
+		if _, ok := err.(user.UnknownUserError); !ok {
+			return "", "", err
+		}
+		// Lookup by id
+		userObj, err = user.LookupId(userStr)
+		if err != nil {
 			return "", "", err
 		}
 	}
@@ -320,12 +335,11 @@ func GetUserFromUsername(userStr string, groupStr string) (string, string, error
 	if groupStr != "" {
 		group, err = user.LookupGroup(groupStr)
 		if err != nil {
-			if _, ok := err.(user.UnknownGroupError); ok {
-				group, err = user.LookupGroupId(groupStr)
-				if err != nil {
-					return "", "", err
-				}
-			} else {
+			if _, ok := err.(user.UnknownGroupError); !ok {
+				return "", "", err
+			}
+			group, err = user.LookupGroupId(groupStr)
+			if err != nil {
 				return "", "", err
 			}
 		}

pkg/util/command_util_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package util
 
 import (
+	"reflect"
 	"sort"
 	"testing"
 
@@ -27,14 +28,12 @@ var testURL = "https://github.com/GoogleContainerTools/runtimes-common/blob/mast
 
 var testEnvReplacement = []struct {
 	path         string
-	command      string
 	envs         []string
 	isFilepath   bool
 	expectedPath string
 }{
 	{
-		path:    "/simple/path",
-		command: "WORKDIR /simple/path",
+		path: "/simple/path",
 		envs: []string{
 			"simple=/path/",
 		},
@@ -42,8 +41,7 @@ var testEnvReplacement = []struct {
 		expectedPath: "/simple/path",
 	},
 	{
-		path:    "/simple/path/",
-		command: "WORKDIR /simple/path/",
+		path: "/simple/path/",
 		envs: []string{
 			"simple=/path/",
 		},
@@ -51,8 +49,7 @@ var testEnvReplacement = []struct {
 		expectedPath: "/simple/path/",
 	},
 	{
-		path:    "${a}/b",
-		command: "WORKDIR ${a}/b",
+		path: "${a}/b",
 		envs: []string{
 			"a=/path/",
 			"b=/path2/",
@@ -61,8 +58,7 @@ var testEnvReplacement = []struct {
 		expectedPath: "/path/b",
 	},
 	{
-		path:    "/$a/b",
-		command: "COPY ${a}/b /c/",
+		path: "/$a/b",
 		envs: []string{
 			"a=/path/",
 			"b=/path2/",
@@ -71,8 +67,7 @@ var testEnvReplacement = []struct {
 		expectedPath: "/path/b",
 	},
 	{
-		path:    "/$a/b/",
-		command: "COPY /${a}/b /c/",
+		path: "/$a/b/",
 		envs: []string{
 			"a=/path/",
 			"b=/path2/",
@@ -81,8 +76,7 @@ var testEnvReplacement = []struct {
 		expectedPath: "/path/b/",
 	},
 	{
-		path:    "\\$foo",
-		command: "COPY \\$foo /quux",
+		path: "\\$foo",
 		envs: []string{
 			"foo=/path/",
 		},
@@ -90,13 +84,35 @@ var testEnvReplacement = []struct {
 		expectedPath: "$foo",
 	},
 	{
-		path:    "8080/$protocol",
-		command: "EXPOSE 8080/$protocol",
+		path: "8080/$protocol",
 		envs: []string{
 			"protocol=udp",
 		},
 		expectedPath: "8080/udp",
 	},
+	{
+		path: "8080/$protocol",
+		envs: []string{
+			"protocol=udp",
+		},
+		expectedPath: "8080/udp",
+	},
+	{
+		path: "$url",
+		envs: []string{
+			"url=http://example.com",
+		},
+		isFilepath:   true,
+		expectedPath: "http://example.com",
+	},
+	{
+		path: "$url",
+		envs: []string{
+			"url=http://example.com",
+		},
+		isFilepath:   false,
+		expectedPath: "http://example.com",
+	},
 }
 
 func Test_EnvReplacement(t *testing.T) {
@@ -183,6 +199,7 @@ var urlDestFilepathTests = []struct {
 	cwd          string
 	dest         string
 	expectedDest string
+	envs         []string
 }{
 	{
 		url:          "https://something/something",
@@ -202,12 +219,19 @@ var urlDestFilepathTests = []struct {
 		dest:         "/dest/",
 		expectedDest: "/dest/something",
 	},
+	{
+		url:          "https://something/$foo.tar.gz",
+		cwd:          "/test",
+		dest:         "/foo/",
+		expectedDest: "/foo/bar.tar.gz",
+		envs:         []string{"foo=bar"},
+	},
 }
 
 func Test_UrlDestFilepath(t *testing.T) {
 	for _, test := range urlDestFilepathTests {
-		actualDest := URLDestinationFilepath(test.url, test.dest, test.cwd)
-		testutil.CheckErrorAndDeepEqual(t, false, nil, test.expectedDest, actualDest)
+		actualDest, err := URLDestinationFilepath(test.url, test.dest, test.cwd, test.envs)
+		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedDest, actualDest)
 	}
 }
 
@@ -382,7 +406,7 @@ var isSrcValidTests = []struct {
 func Test_IsSrcsValid(t *testing.T) {
 	for _, test := range isSrcValidTests {
 		t.Run(test.name, func(t *testing.T) {
-			if err := GetExcludedFiles(buildContextPath); err != nil {
+			if err := GetExcludedFiles("", buildContextPath); err != nil {
 				t.Fatalf("error getting excluded files: %v", err)
 			}
 			err := IsSrcsValid(test.srcsAndDest, test.resolvedSources, buildContextPath)
@@ -400,7 +424,6 @@ var testResolveSources = []struct {
 			"context/foo",
 			"context/b*",
 			testURL,
-			"dest/",
 		},
 		expectedList: []string{
 			"context/foo",
@@ -448,3 +471,58 @@ func Test_RemoteUrls(t *testing.T) {
 	}
 
 }
+
+func TestResolveEnvironmentReplacementList(t *testing.T) {
+	type args struct {
+		values     []string
+		envs       []string
+		isFilepath bool
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []string
+		wantErr bool
+	}{
+		{
+			name: "url",
+			args: args{
+				values: []string{
+					"https://google.com/$foo", "$bar", "$url",
+				},
+				envs: []string{
+					"foo=baz",
+					"bar=bat",
+					"url=https://google.com",
+				},
+			},
+			want: []string{"https://google.com/baz", "bat", "https://google.com"},
+		},
+		{
+			name: "mixed",
+			args: args{
+				values: []string{
+					"$foo", "$bar$baz", "baz",
+				},
+				envs: []string{
+					"foo=FOO",
+					"bar=BAR",
+					"baz=BAZ",
+				},
+			},
+			want: []string{"FOO", "BARBAZ", "baz"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ResolveEnvironmentReplacementList(tt.args.values, tt.args.envs, tt.args.isFilepath)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ResolveEnvironmentReplacementList() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ResolveEnvironmentReplacementList() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/util/fs_util.go

@@ -79,10 +79,7 @@ func GetFSFromImage(root string, img v1.Image) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	// Store a map of files to their mtime. We need to set mtimes in a second pass because creating files
-	// can change the mtime of a directory.
-	extractedFiles := map[string]time.Time{}
+	extractedFiles := []string{}
 
 	for i, l := range layers {
 		logrus.Debugf("Extracting layer %d", i)
@@ -113,17 +110,10 @@ func GetFSFromImage(root string, img v1.Image) ([]string, error) {
 			if err := extractFile(root, hdr, tr); err != nil {
 				return nil, err
 			}
-			extractedFiles[filepath.Join(root, filepath.Clean(hdr.Name))] = hdr.ModTime
+			extractedFiles = append(extractedFiles, filepath.Join(root, filepath.Clean(hdr.Name)))
 		}
 	}
-
-	fileNames := []string{}
-	for f, t := range extractedFiles {
-		fileNames = append(fileNames, f)
-		os.Chtimes(f, time.Time{}, t)
-	}
-
-	return fileNames, nil
+	return extractedFiles, nil
 }
 
 // DeleteFilesystem deletes the extracted image file system
@@ -131,6 +121,10 @@ func DeleteFilesystem() error {
 	logrus.Info("Deleting filesystem...")
 	return filepath.Walk(constants.RootDir, func(path string, info os.FileInfo, _ error) error {
 		if CheckWhitelist(path) {
+			if !isExist(path) {
+				logrus.Debugf("Path %s whitelisted, but not exists", path)
+				return nil
+			}
 			if info.IsDir() {
 				return filepath.SkipDir
 			}
@@ -148,6 +142,14 @@ func DeleteFilesystem() error {
 	})
 }
 
+// isExists returns true if path exists
+func isExist(path string) bool {
+	if _, err := os.Stat(path); err == nil {
+		return true
+	}
+	return false
+}
+
 // ChildDirInWhitelist returns true if there is a child file or directory of the path in the whitelist
 func childDirInWhitelist(path string) bool {
 	for _, d := range whitelist {
@@ -198,7 +200,7 @@ func extractFile(dest string, hdr *tar.Header, tr io.Reader) error {
 	switch hdr.Typeflag {
 	case tar.TypeReg:
 		logrus.Debugf("creating file %s", path)
-		// It's possible a file is in the tar before it's directory.
+		// It's possible a file is in the tar before its directory.
 		if _, err := os.Stat(dir); os.IsNotExist(err) {
 			logrus.Debugf("base %s for file %s does not exist. Creating.", base, path)
 			if err := os.MkdirAll(dir, 0755); err != nil {
@@ -216,27 +218,16 @@ func extractFile(dest string, hdr *tar.Header, tr io.Reader) error {
 		if err != nil {
 			return err
 		}
-		// manually set permissions on file, since the default umask (022) will interfere
-		if err = os.Chmod(path, mode); err != nil {
-			return err
-		}
 		if _, err = io.Copy(currFile, tr); err != nil {
 			return err
 		}
-		if err = currFile.Chown(uid, gid); err != nil {
+		if err = setFilePermissions(path, mode, uid, gid); err != nil {
 			return err
 		}
 		currFile.Close()
 	case tar.TypeDir:
 		logrus.Debugf("creating dir %s", path)
-		if err := os.MkdirAll(path, mode); err != nil {
-			return err
-		}
-		// In some cases, MkdirAll doesn't change the permissions, so run Chmod
-		if err := os.Chmod(path, mode); err != nil {
-			return err
-		}
-		if err := os.Chown(path, uid, gid); err != nil {
+		if err := mkdirAllWithPermissions(path, mode, uid, gid); err != nil {
 			return err
 		}
 
@@ -283,7 +274,6 @@ func extractFile(dest string, hdr *tar.Header, tr io.Reader) error {
 			return err
 		}
 	}
-
 	return nil
 }
 
@@ -310,12 +300,7 @@ func checkWhitelistRoot(root string) bool {
 	if root == constants.RootDir {
 		return false
 	}
-	for _, wl := range whitelist {
-		if HasFilepathPrefix(root, wl.Path, wl.PrefixMatchOnly) {
-			return true
-		}
-	}
-	return false
+	return CheckWhitelist(root)
 }
 
 // Get whitelist from roots of mounted files
@@ -388,8 +373,7 @@ func RelativeFiles(fp string, root string) ([]string, error) {
 }
 
 // ParentDirectories returns a list of paths to all parent directories
-// Ex. /some/temp/dir -> [/some, /some/temp, /some/temp/dir]
-// This purposefully excludes the /.
+// Ex. /some/temp/dir -> [/, /some, /some/temp, /some/temp/dir]
 func ParentDirectories(path string) []string {
 	path = filepath.Clean(path)
 	dirs := strings.Split(path, "/")
@@ -405,6 +389,24 @@ func ParentDirectories(path string) []string {
 	return paths
 }
 
+// ParentDirectoriesWithoutLeadingSlash returns a list of paths to all parent directories
+// all subdirectories do not contain a leading /
+// Ex. /some/temp/dir -> [/, some, some/temp, some/temp/dir]
+func ParentDirectoriesWithoutLeadingSlash(path string) []string {
+	path = filepath.Clean(path)
+	dirs := strings.Split(path, "/")
+	dirPath := ""
+	paths := []string{constants.RootDir}
+	for index, dir := range dirs {
+		if dir == "" || index == (len(dirs)-1) {
+			continue
+		}
+		dirPath = filepath.Join(dirPath, dir)
+		paths = append(paths, dirPath)
+	}
+	return paths
+}
+
 // FilepathExists returns true if the path exists
 func FilepathExists(path string) bool {
 	_, err := os.Lstat(path)
@@ -429,10 +431,7 @@ func CreateFile(path string, reader io.Reader, perm os.FileMode, uid uint32, gid
 	if _, err := io.Copy(dest, reader); err != nil {
 		return err
 	}
-	if err := dest.Chmod(perm); err != nil {
-		return err
-	}
-	return dest.Chown(int(uid), int(gid))
+	return setFilePermissions(path, perm, int(uid), int(gid))
 }
 
 // AddVolumePath adds the given path to the volume whitelist.
@@ -492,13 +491,11 @@ func CopyDir(src, dest, buildcontext string) ([]string, error) {
 		if fi.IsDir() {
 			logrus.Debugf("Creating directory %s", destPath)
 
+			mode := fi.Mode()
 			uid := int(fi.Sys().(*syscall.Stat_t).Uid)
 			gid := int(fi.Sys().(*syscall.Stat_t).Gid)
 
-			if err := os.MkdirAll(destPath, fi.Mode()); err != nil {
-				return nil, err
-			}
-			if err := os.Chown(destPath, uid, gid); err != nil {
+			if err := mkdirAllWithPermissions(destPath, mode, uid, gid); err != nil {
 				return nil, err
 			}
 		} else if fi.Mode()&os.ModeSymlink != 0 {
@@ -557,11 +554,15 @@ func CopyFile(src, dest, buildcontext string) (bool, error) {
 }
 
 // GetExcludedFiles gets a list of files to exclude from the .dockerignore
-func GetExcludedFiles(buildcontext string) error {
-	path := filepath.Join(buildcontext, ".dockerignore")
+func GetExcludedFiles(dockerfilepath string, buildcontext string) error {
+	path := dockerfilepath + ".dockerignore"
+	if !FilepathExists(path) {
+		path = filepath.Join(buildcontext, ".dockerignore")
+	}
 	if !FilepathExists(path) {
 		return nil
 	}
+	logrus.Infof("Using dockerignore file: %v", path)
 	contents, err := ioutil.ReadFile(path)
 	if err != nil {
 		return errors.Wrap(err, "parsing .dockerignore")
@@ -591,10 +592,10 @@ func excludeFile(path, buildcontext string) bool {
 
 // HasFilepathPrefix checks  if the given file path begins with prefix
 func HasFilepathPrefix(path, prefix string, prefixMatchOnly bool) bool {
-	path = filepath.Clean(path)
 	prefix = filepath.Clean(prefix)
-	pathArray := strings.Split(path, "/")
 	prefixArray := strings.Split(prefix, "/")
+	path = filepath.Clean(path)
+	pathArray := strings.SplitN(path, "/", len(prefixArray)+1)
 
 	if len(pathArray) < len(prefixArray) {
 		return false
@@ -614,3 +615,24 @@ func HasFilepathPrefix(path, prefix string, prefixMatchOnly bool) bool {
 func Volumes() []string {
 	return volumes
 }
+
+func mkdirAllWithPermissions(path string, mode os.FileMode, uid, gid int) error {
+	if err := os.MkdirAll(path, mode); err != nil {
+		return err
+	}
+	if err := os.Chown(path, uid, gid); err != nil {
+		return err
+	}
+	// In some cases, MkdirAll doesn't change the permissions, so run Chmod
+	// Must chmod after chown because chown resets the file mode.
+	return os.Chmod(path, mode)
+}
+
+func setFilePermissions(path string, mode os.FileMode, uid, gid int) error {
+	if err := os.Chown(path, uid, gid); err != nil {
+		return err
+	}
+	// manually set permissions on file, since the default umask (022) will interfere
+	// Must chmod after chown because chown resets the file mode.
+	return os.Chmod(path, mode)
+}

pkg/util/fs_util_test.go

@@ -19,11 +19,13 @@ package util
 import (
 	"archive/tar"
 	"bytes"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"reflect"
 	"sort"
+	"strings"
 	"testing"
 
 	"github.com/GoogleContainerTools/kaniko/testutil"
@@ -177,6 +179,38 @@ func Test_ParentDirectories(t *testing.T) {
 	}
 }
 
+func Test_ParentDirectoriesWithoutLeadingSlash(t *testing.T) {
+	tests := []struct {
+		name     string
+		path     string
+		expected []string
+	}{
+		{
+			name: "regular path",
+			path: "/path/to/dir",
+			expected: []string{
+				"/",
+				"path",
+				"path/to",
+			},
+		},
+		{
+			name: "current directory",
+			path: ".",
+			expected: []string{
+				"/",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := ParentDirectoriesWithoutLeadingSlash(tt.path)
+			testutil.CheckErrorAndDeepEqual(t, false, nil, tt.expected, actual)
+		})
+	}
+}
+
 func Test_CheckWhitelist(t *testing.T) {
 	type args struct {
 		path      string
@@ -310,6 +344,84 @@ func TestHasFilepathPrefix(t *testing.T) {
 	}
 }
 
+func BenchmarkHasFilepathPrefix(b *testing.B) {
+	tests := []struct {
+		path            string
+		prefix          string
+		prefixMatchOnly bool
+	}{
+		{
+			path:            "/foo/bar",
+			prefix:          "/foo",
+			prefixMatchOnly: true,
+		},
+		{
+			path:            "/foo/bar/baz",
+			prefix:          "/foo",
+			prefixMatchOnly: true,
+		},
+		{
+			path:            "/foo/bar/baz/foo",
+			prefix:          "/foo",
+			prefixMatchOnly: true,
+		},
+		{
+			path:            "/foo/bar/baz/foo/foobar",
+			prefix:          "/foo",
+			prefixMatchOnly: true,
+		},
+		{
+			path:            "/foo/bar",
+			prefix:          "/foo/bar",
+			prefixMatchOnly: true,
+		},
+		{
+			path:            "/foo/bar/baz",
+			prefix:          "/foo/bar",
+			prefixMatchOnly: true,
+		},
+		{
+			path:            "/foo/bar/baz/foo",
+			prefix:          "/foo/bar",
+			prefixMatchOnly: true,
+		},
+		{
+			path:            "/foo/bar/baz/foo/foobar",
+			prefix:          "/foo/bar",
+			prefixMatchOnly: true,
+		},
+		{
+			path:            "/foo/bar",
+			prefix:          "/foo/bar/baz",
+			prefixMatchOnly: true,
+		},
+		{
+			path:            "/foo/bar/baz",
+			prefix:          "/foo/bar/baz",
+			prefixMatchOnly: true,
+		},
+		{
+			path:            "/foo/bar/baz/foo",
+			prefix:          "/foo/bar/baz",
+			prefixMatchOnly: true,
+		},
+		{
+			path:            "/foo/bar/baz/foo/foobar",
+			prefix:          "/foo/bar/baz",
+			prefixMatchOnly: true,
+		},
+	}
+	for _, ts := range tests {
+		name := fmt.Sprint("PathDepth=", strings.Count(ts.path, "/"), ",PrefixDepth=", strings.Count(ts.prefix, "/"))
+		b.Run(name, func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				HasFilepathPrefix(ts.path, ts.prefix, ts.prefixMatchOnly)
+			}
+		})
+	}
+}
+
 type checker func(root string, t *testing.T)
 
 func fileExists(p string) checker {
@@ -503,6 +615,30 @@ func TestExtractFile(t *testing.T) {
 				filesAreHardlinks("/bin/uncompress", "/bin/gzip"),
 			},
 		},
+		{
+			name:     "file with setuid bit",
+			contents: []byte("helloworld"),
+			hdrs:     []*tar.Header{fileHeader("./bar", "helloworld", 04644)},
+			checkers: []checker{
+				fileExists("/bar"),
+				fileMatches("/bar", []byte("helloworld")),
+				permissionsMatch("/bar", 0644|os.ModeSetuid),
+			},
+		},
+		{
+			name:     "dir with sticky bit",
+			contents: []byte("helloworld"),
+			hdrs: []*tar.Header{
+				dirHeader("./foo", 01755),
+				fileHeader("./foo/bar", "helloworld", 0644),
+			},
+			checkers: []checker{
+				fileExists("/foo/bar"),
+				fileMatches("/foo/bar", []byte("helloworld")),
+				permissionsMatch("/foo/bar", 0644),
+				permissionsMatch("/foo", 0755|os.ModeDir|os.ModeSticky),
+			},
+		},
 	}
 
 	for _, tc := range tcs {
@@ -638,3 +774,54 @@ func Test_childDirInWhitelist(t *testing.T) {
 		})
 	}
 }
+
+func Test_correctDockerignoreFileIsUsed(t *testing.T) {
+	type args struct {
+		dockerfilepath string
+		buildcontext   string
+		excluded       []string
+		included       []string
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			name: "relative dockerfile used",
+			args: args{
+				dockerfilepath: "../../integration/dockerfiles/Dockerfile_dockerignore_relative",
+				buildcontext:   "../../integration/",
+				excluded:       []string{"ignore_relative/bar"},
+				included:       []string{"ignore_relative/foo", "ignore/bar"},
+			},
+		},
+		{
+			name: "context dockerfile is used",
+			args: args{
+				dockerfilepath: "../../integration/dockerfiles/Dockerfile_test_dockerignore",
+				buildcontext:   "../../integration/",
+				excluded:       []string{"ignore/bar"},
+				included:       []string{"ignore/foo", "ignore_relative/bar"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		if err := GetExcludedFiles(tt.args.dockerfilepath, tt.args.buildcontext); err != nil {
+			t.Fatal(err)
+		}
+		for _, excl := range tt.args.excluded {
+			t.Run(tt.name+" to exclude "+excl, func(t *testing.T) {
+				if !excludeFile(excl, tt.args.buildcontext) {
+					t.Errorf("'%v' not excluded", excl)
+				}
+			})
+		}
+		for _, incl := range tt.args.included {
+			t.Run(tt.name+" to include "+incl, func(t *testing.T) {
+				if excludeFile(incl, tt.args.buildcontext) {
+					t.Errorf("'%v' not included", incl)
+				}
+			})
+		}
+	}
+}

pkg/util/image_util.go

@@ -102,7 +102,7 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) {
 
 	registryName := ref.Context().RegistryStr()
 	if opts.InsecurePull || opts.InsecureRegistries.Contains(registryName) {
-		newReg, err := name.NewInsecureRegistry(registryName, name.WeakValidation)
+		newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
 		if err != nil {
 			return nil, err
 		}
@@ -149,5 +149,5 @@ func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) {
 		cacheKey = d.String()
 	}
 
-	return cache.LocalSource(opts, cacheKey)
+	return cache.LocalSource(&opts.CacheOptions, cacheKey)
 }

pkg/util/tar_util.go

@@ -25,6 +25,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 
 	"github.com/docker/docker/pkg/archive"
@@ -54,7 +55,6 @@ func (t *Tar) Close() {
 
 // AddFileToTar adds the file at path p to the tar
 func (t *Tar) AddFileToTar(p string) error {
-	logrus.Debugf("Adding file %s to tar", p)
 	i, err := os.Lstat(p)
 	if err != nil {
 		return fmt.Errorf("Failed to get file info for %s: %s", p, err)
@@ -75,7 +75,14 @@ func (t *Tar) AddFileToTar(p string) error {
 	if err != nil {
 		return err
 	}
-	hdr.Name = p
+
+	if p != "/" {
+		// Docker uses no leading / in the tarball
+		hdr.Name = strings.TrimLeft(p, "/")
+	} else {
+		// allow entry for / to preserve permission changes etc. (currently ignored anyway by Docker runtime)
+		hdr.Name = p
+	}
 
 	hardlink, linkDst := t.checkHardlink(p, i)
 	if hardlink {
@@ -105,7 +112,8 @@ func (t *Tar) Whiteout(p string) error {
 	name := ".wh." + filepath.Base(p)
 
 	th := &tar.Header{
-		Name: filepath.Join(dir, name),
+		// Docker uses no leading / in the tarball
+		Name: strings.TrimLeft(filepath.Join(dir, name), "/"),
 		Size: 0,
 	}
 	if err := t.w.WriteHeader(th); err != nil {

pkg/util/util.go

@@ -20,14 +20,13 @@ import (
 	"crypto/md5"
 	"crypto/sha256"
 	"encoding/hex"
-	"encoding/json"
 	"io"
 	"os"
 	"strconv"
+	"sync"
 	"syscall"
 
-	"github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/partial"
+	highwayhash "github.com/minio/HighwayHash"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -47,8 +46,15 @@ func ConfigureLogging(logLevel string) error {
 
 // Hasher returns a hash function, used in snapshotting to determine if a file has changed
 func Hasher() func(string) (string, error) {
+	pool := sync.Pool{
+		New: func() interface{} {
+			b := make([]byte, highwayhash.Size*10*1024)
+			return &b
+		},
+	}
+	key := make([]byte, highwayhash.Size)
 	hasher := func(p string) (string, error) {
-		h := md5.New()
+		h, _ := highwayhash.New(key)
 		fi, err := os.Lstat(p)
 		if err != nil {
 			return "", err
@@ -66,7 +72,9 @@ func Hasher() func(string) (string, error) {
 				return "", err
 			}
 			defer f.Close()
-			if _, err := io.Copy(h, f); err != nil {
+			buf := pool.Get().(*[]byte)
+			defer pool.Put(buf)
+			if _, err := io.CopyBuffer(h, f, *buf); err != nil {
 				return "", err
 			}
 		}
@@ -130,29 +138,3 @@ func SHA256(r io.Reader) (string, error) {
 	}
 	return hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))), nil
 }
-
-type ReproducibleManifest struct {
-	Layers []v1.Descriptor
-	Config v1.Config
-}
-
-func ReproducibleDigest(img partial.WithManifestAndConfigFile) (string, error) {
-	mfst, err := img.Manifest()
-	if err != nil {
-		return "", err
-	}
-	cfg, err := img.ConfigFile()
-	if err != nil {
-		return "", err
-	}
-	rm := ReproducibleManifest{
-		Layers: mfst.Layers,
-		Config: cfg.Config,
-	}
-
-	b, err := json.Marshal(rm)
-	if err != nil {
-		return "", err
-	}
-	return string(b), nil
-}