Add support for COPY --from=<an unrelated image>. (#479)

Right now kaniko only supports COPY --from=<another stage>.
This commit adds support for the case where the referenced image is a remote image
in a registry that has not been used as a stage yet in the build.

integration/dockerfiles/Dockerfile_test_multistage

@@ -15,3 +15,4 @@ FROM fedora@sha256:c4cc32b09c6ae3f1353e7e33a8dda93dc41676b923d6d89afa996b421cc5a
 FROM base
 ARG file
 COPY --from=second /foo ${file}
+COPY --from=gcr.io/google-appengine/debian9@sha256:00109fa40230a081f5ecffe0e814725042ff62a03e2d1eae0563f1f82eaeae9b /etc/os-release /new

pkg/dockerfile/dockerfile.go

@@ -126,6 +126,7 @@ func resolveStages(stages []instructions.Stage) {
 					if val, ok := nameToIndex[c.From]; ok {
 						c.From = val
 					}
+
 				}
 			}
 		}

pkg/executor/build.go

@@ -23,6 +23,8 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/moby/buildkit/frontend/dockerfile/instructions"
+
 	"golang.org/x/sync/errgroup"
 
 	"github.com/google/go-containerregistry/pkg/name"
@@ -337,6 +339,10 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 	if err != nil {
 		return nil, err
 	}
+	// Some stages may refer to other random images, not previous stages
+	if err := fetchExtraStages(stages, opts); err != nil {
+		return nil, err
+	}
 	for index, stage := range stages {
 		sb, err := newStageBuilder(opts, stage)
 		if err != nil {
@@ -380,10 +386,10 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 			return sourceImage, nil
 		}
 		if stage.SaveStage {
-			if err := saveStageAsTarball(index, sourceImage); err != nil {
+			if err := saveStageAsTarball(strconv.Itoa(index), sourceImage); err != nil {
 				return nil, err
 			}
-			if err := extractImageToDependecyDir(index, sourceImage); err != nil {
+			if err := extractImageToDependecyDir(strconv.Itoa(index), sourceImage); err != nil {
 				return nil, err
 			}
 		}
@@ -396,8 +402,38 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 	return nil, err
 }
 
-func extractImageToDependecyDir(index int, image v1.Image) error {
-	dependencyDir := filepath.Join(constants.KanikoDir, strconv.Itoa(index))
+func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) error {
+	for _, s := range stages {
+		for _, cmd := range s.Commands {
+			c, ok := cmd.(*instructions.CopyCommand)
+			if !ok || c.From == "" {
+				continue
+			}
+
+			// FROMs at this point are guaranteed to be either an integer referring to a previous stage or
+			// a name of a remote image.
+			if _, err := strconv.Atoi(c.From); err == nil {
+				continue
+			}
+			// This must be an image name, fetch it.
+			logrus.Debugf("Found extra base image stage %s", c.From)
+			sourceImage, err := util.RetrieveRemoteImage(c.From, opts)
+			if err != nil {
+				return err
+			}
+			if err := saveStageAsTarball(c.From, sourceImage); err != nil {
+				return err
+			}
+			if err := extractImageToDependecyDir(c.From, sourceImage); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func extractImageToDependecyDir(name string, image v1.Image) error {
+	dependencyDir := filepath.Join(constants.KanikoDir, name)
 	if err := os.MkdirAll(dependencyDir, 0755); err != nil {
 		return err
 	}
@@ -406,16 +442,16 @@ func extractImageToDependecyDir(index int, image v1.Image) error {
 	return err
 }
 
-func saveStageAsTarball(stageIndex int, image v1.Image) error {
+func saveStageAsTarball(path string, image v1.Image) error {
 	destRef, err := name.NewTag("temp/tag", name.WeakValidation)
 	if err != nil {
 		return err
 	}
-	if err := os.MkdirAll(constants.KanikoIntermediateStagesDir, 0750); err != nil {
+	tarPath := filepath.Join(constants.KanikoIntermediateStagesDir, path)
+	logrus.Infof("Storing source image from stage %d at path %s", path, tarPath)
+	if err := os.MkdirAll(filepath.Dir(tarPath), 0750); err != nil {
 		return err
 	}
-	tarPath := filepath.Join(constants.KanikoIntermediateStagesDir, strconv.Itoa(stageIndex))
-	logrus.Infof("Storing source image from stage %d at path %s", stageIndex, tarPath)
 	return tarball.WriteToFile(tarPath, destRef, image)
 }
 

pkg/util/image_util.go

@@ -39,8 +39,8 @@ import (
 )
 
 var (
-	// For testing
-	retrieveRemoteImage = remoteImage
+	// RetrieveRemoteImage downloads an image from a remote location
+	RetrieveRemoteImage = remoteImage
 	retrieveTarImage    = tarballImage
 )
 
@@ -67,21 +67,8 @@ func RetrieveSourceImage(stage config.KanikoStage, opts *config.KanikoOptions) (
 		return retrieveTarImage(stage.BaseImageIndex)
 	}
 
-	// Next, check if local caching is enabled
-	// If so, look in the local cache before trying the remote registry
-	if opts.Cache && opts.CacheDir != "" {
-		cachedImage, err := cachedImage(opts, currentBaseName)
-		if cachedImage != nil {
-			return cachedImage, nil
-		}
-
-		if err != nil {
-			logrus.Warnf("Error while retrieving image from cache: %v", err)
-		}
-	}
-
 	// Otherwise, initialize image as usual
-	return retrieveRemoteImage(currentBaseName, opts)
+	return RetrieveRemoteImage(currentBaseName, opts)
 }
 
 // RetrieveConfigFile returns the config file for an image
@@ -104,6 +91,18 @@ func tarballImage(index int) (v1.Image, error) {
 
 func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) {
 	logrus.Infof("Downloading base image %s", image)
+	// First, check if local caching is enabled
+	// If so, look in the local cache before trying the remote registry
+	if opts.Cache && opts.CacheDir != "" {
+		cachedImage, err := cachedImage(opts, image)
+		if cachedImage != nil {
+			return cachedImage, nil
+		}
+
+		if err != nil {
+			logrus.Warnf("Error while retrieving image from cache: %v", err)
+		}
+	}
 	ref, err := name.ParseReference(image, name.WeakValidation)
 	if err != nil {
 		return nil, err

pkg/util/image_util_test.go

@@ -47,14 +47,14 @@ func Test_StandardImage(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	original := retrieveRemoteImage
+	original := RetrieveRemoteImage
 	defer func() {
-		retrieveRemoteImage = original
+		RetrieveRemoteImage = original
 	}()
 	mock := func(image string, opts *config.KanikoOptions) (v1.Image, error) {
 		return nil, nil
 	}
-	retrieveRemoteImage = mock
+	RetrieveRemoteImage = mock
 	actual, err := RetrieveSourceImage(config.KanikoStage{
 		Stage: stages[0],
 	}, &config.KanikoOptions{})