From 7611ea7a1d787a5892250b04e51e06149967c3d6 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Thu, 6 Dec 2018 12:44:03 -0600 Subject: [PATCH] Add support for COPY --from=. (#479) Right now kaniko only supports COPY --from=. This commit adds support for the case where the referenced image is a remote image in a registry that has not been used as a stage yet in the build. --- .../dockerfiles/Dockerfile_test_multistage | 1 + pkg/dockerfile/dockerfile.go | 1 + pkg/executor/build.go | 52 ++++++++++++++++--- pkg/util/image_util.go | 31 ++++++----- pkg/util/image_util_test.go | 6 +-- 5 files changed, 64 insertions(+), 27 deletions(-) diff --git a/integration/dockerfiles/Dockerfile_test_multistage b/integration/dockerfiles/Dockerfile_test_multistage index 25246c815..91a130497 100644 --- a/integration/dockerfiles/Dockerfile_test_multistage +++ b/integration/dockerfiles/Dockerfile_test_multistage @@ -15,3 +15,4 @@ FROM fedora@sha256:c4cc32b09c6ae3f1353e7e33a8dda93dc41676b923d6d89afa996b421cc5a FROM base ARG file COPY --from=second /foo ${file} +COPY --from=gcr.io/google-appengine/debian9@sha256:00109fa40230a081f5ecffe0e814725042ff62a03e2d1eae0563f1f82eaeae9b /etc/os-release /new diff --git a/pkg/dockerfile/dockerfile.go b/pkg/dockerfile/dockerfile.go index 2a886d0f6..6c401d829 100644 --- a/pkg/dockerfile/dockerfile.go +++ b/pkg/dockerfile/dockerfile.go @@ -126,6 +126,7 @@ func resolveStages(stages []instructions.Stage) { if val, ok := nameToIndex[c.From]; ok { c.From = val } + } } } diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 88ebafe14..17cc2dd0f 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -23,6 +23,8 @@ import ( "strconv" "time" + "github.com/moby/buildkit/frontend/dockerfile/instructions" + "golang.org/x/sync/errgroup" "github.com/google/go-containerregistry/pkg/name" @@ -337,6 +339,10 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { if err != nil { return nil, err } + // Some stages may refer to other random images, not previous stages + if err := fetchExtraStages(stages, opts); err != nil { + return nil, err + } for index, stage := range stages { sb, err := newStageBuilder(opts, stage) if err != nil { @@ -380,10 +386,10 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { return sourceImage, nil } if stage.SaveStage { - if err := saveStageAsTarball(index, sourceImage); err != nil { + if err := saveStageAsTarball(strconv.Itoa(index), sourceImage); err != nil { return nil, err } - if err := extractImageToDependecyDir(index, sourceImage); err != nil { + if err := extractImageToDependecyDir(strconv.Itoa(index), sourceImage); err != nil { return nil, err } } @@ -396,8 +402,38 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { return nil, err } -func extractImageToDependecyDir(index int, image v1.Image) error { - dependencyDir := filepath.Join(constants.KanikoDir, strconv.Itoa(index)) +func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) error { + for _, s := range stages { + for _, cmd := range s.Commands { + c, ok := cmd.(*instructions.CopyCommand) + if !ok || c.From == "" { + continue + } + + // FROMs at this point are guaranteed to be either an integer referring to a previous stage or + // a name of a remote image. + if _, err := strconv.Atoi(c.From); err == nil { + continue + } + // This must be an image name, fetch it. + logrus.Debugf("Found extra base image stage %s", c.From) + sourceImage, err := util.RetrieveRemoteImage(c.From, opts) + if err != nil { + return err + } + if err := saveStageAsTarball(c.From, sourceImage); err != nil { + return err + } + if err := extractImageToDependecyDir(c.From, sourceImage); err != nil { + return err + } + } + } + return nil +} + +func extractImageToDependecyDir(name string, image v1.Image) error { + dependencyDir := filepath.Join(constants.KanikoDir, name) if err := os.MkdirAll(dependencyDir, 0755); err != nil { return err } @@ -406,16 +442,16 @@ func extractImageToDependecyDir(index int, image v1.Image) error { return err } -func saveStageAsTarball(stageIndex int, image v1.Image) error { +func saveStageAsTarball(path string, image v1.Image) error { destRef, err := name.NewTag("temp/tag", name.WeakValidation) if err != nil { return err } - if err := os.MkdirAll(constants.KanikoIntermediateStagesDir, 0750); err != nil { + tarPath := filepath.Join(constants.KanikoIntermediateStagesDir, path) + logrus.Infof("Storing source image from stage %d at path %s", path, tarPath) + if err := os.MkdirAll(filepath.Dir(tarPath), 0750); err != nil { return err } - tarPath := filepath.Join(constants.KanikoIntermediateStagesDir, strconv.Itoa(stageIndex)) - logrus.Infof("Storing source image from stage %d at path %s", stageIndex, tarPath) return tarball.WriteToFile(tarPath, destRef, image) } diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index 8e46ce067..cb167c9de 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -39,8 +39,8 @@ import ( ) var ( - // For testing - retrieveRemoteImage = remoteImage + // RetrieveRemoteImage downloads an image from a remote location + RetrieveRemoteImage = remoteImage retrieveTarImage = tarballImage ) @@ -67,21 +67,8 @@ func RetrieveSourceImage(stage config.KanikoStage, opts *config.KanikoOptions) ( return retrieveTarImage(stage.BaseImageIndex) } - // Next, check if local caching is enabled - // If so, look in the local cache before trying the remote registry - if opts.Cache && opts.CacheDir != "" { - cachedImage, err := cachedImage(opts, currentBaseName) - if cachedImage != nil { - return cachedImage, nil - } - - if err != nil { - logrus.Warnf("Error while retrieving image from cache: %v", err) - } - } - // Otherwise, initialize image as usual - return retrieveRemoteImage(currentBaseName, opts) + return RetrieveRemoteImage(currentBaseName, opts) } // RetrieveConfigFile returns the config file for an image @@ -104,6 +91,18 @@ func tarballImage(index int) (v1.Image, error) { func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) { logrus.Infof("Downloading base image %s", image) + // First, check if local caching is enabled + // If so, look in the local cache before trying the remote registry + if opts.Cache && opts.CacheDir != "" { + cachedImage, err := cachedImage(opts, image) + if cachedImage != nil { + return cachedImage, nil + } + + if err != nil { + logrus.Warnf("Error while retrieving image from cache: %v", err) + } + } ref, err := name.ParseReference(image, name.WeakValidation) if err != nil { return nil, err diff --git a/pkg/util/image_util_test.go b/pkg/util/image_util_test.go index a155963bc..c44fe8a58 100644 --- a/pkg/util/image_util_test.go +++ b/pkg/util/image_util_test.go @@ -47,14 +47,14 @@ func Test_StandardImage(t *testing.T) { if err != nil { t.Error(err) } - original := retrieveRemoteImage + original := RetrieveRemoteImage defer func() { - retrieveRemoteImage = original + RetrieveRemoteImage = original }() mock := func(image string, opts *config.KanikoOptions) (v1.Image, error) { return nil, nil } - retrieveRemoteImage = mock + RetrieveRemoteImage = mock actual, err := RetrieveSourceImage(config.KanikoStage{ Stage: stages[0], }, &config.KanikoOptions{})