Merge pull request #1165 from JordanGoasdoue/multistage-now-respects-dependencies

feat: multistages now respect dependencies without building unnecessary stages

README.md

@@ -4,7 +4,7 @@
 
 ![kaniko logo](logo/Kaniko-Logo.png)
 
-kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster. 
+kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster.
 
 kaniko doesn't depend on a Docker daemon and executes each command within a Dockerfile completely in userspace.
 This enables building container images in environments that can't easily or securely run a Docker daemon, such as a standard Kubernetes cluster.
@@ -15,7 +15,7 @@ We'd love to hear from you!  Join us on [#kaniko Kubernetes Slack](https://kuber
 
 :mega: **Please fill out our [quick 5-question survey](https://forms.gle/HhZGEM33x4FUz9Qa6)** so that we can learn how satisfied you are with Kaniko, and what improvements we should make. Thank you! :dancers:
 
-Kaniko is not an officially supported Google project. 
+Kaniko is not an officially supported Google project.
 
 _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPMENT.md) and [CONTRIBUTING.md](CONTRIBUTING.md)._
 
@@ -50,6 +50,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME
     - [--cache](#--cache)
     - [--cache-dir](#--cache-dir)
     - [--cache-repo](#--cache-repo)
+    - [--context-sub-path](#context-sub-path)
     - [--digest-file](#--digest-file)
     - [--oci-layout-path](#--oci-layout-path)
     - [--insecure-registry](#--insecure-registry)
@@ -69,6 +70,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME
     - [--verbosity](#--verbosity)
     - [--whitelist-var-run](#--whitelist-var-run)
     - [--label](#--label)
+    - [--skip-unused-stages](#skip-unused-stages)
   - [Debug Image](#debug-image)
 - [Security](#security)
 - [Comparison with Other Tools](#comparison-with-other-tools)
@@ -302,7 +304,7 @@ There is also a utility script [`run_in_docker.sh`](./run_in_docker.sh) that can
 ./run_in_docker.sh <path to Dockerfile> <path to build context> <destination of final image>
 ```
 
-_NOTE: `run_in_docker.sh` expects a path to a 
+_NOTE: `run_in_docker.sh` expects a path to a
 Dockerfile relative to the absolute path of the build context._
 
 An example run, specifying the Dockerfile in the container directory `/workspace`, the build
@@ -558,6 +560,11 @@ Ignore /var/run when taking image snapshot. Set it to false to preserve /var/run
 
 Set this flag as `--label key=value` to set some metadata to the final image. This is equivalent as using the `LABEL` within the Dockerfile.
 
+#### --skip-unused-stages
+
+This flag builds only used stages if defined to `true`.
+Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile
+
 ### Debug Image
 
 The kaniko executor image is based on scratch and doesn't contain a shell.

cmd/executor/cmd/root.go

@@ -161,6 +161,7 @@ func addKanikoOptionsFlags() {
 	RootCmd.PersistentFlags().StringVarP(&opts.RegistryMirror, "registry-mirror", "", "", "Registry mirror to use has pull-through cache instead of docker.io.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.WhitelistVarRun, "whitelist-var-run", "", true, "Ignore /var/run directory when taking image snapshot. Set it to false to preserve /var/run/ in destination image. (Default true).")
 	RootCmd.PersistentFlags().VarP(&opts.Labels, "label", "", "Set metadata for an image. Set it repeatedly for multiple labels.")
+	RootCmd.PersistentFlags().BoolVarP(&opts.SkipUnusedStages, "skip-unused-stages", "", false, "Build only used stages if defined to true. Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile")
 }
 
 // addHiddenFlags marks certain flags as hidden from the executor help text

pkg/config/options.go

@@ -56,6 +56,7 @@ type KanikoOptions struct {
 	Cache                   bool
 	Cleanup                 bool
 	WhitelistVarRun         bool
+	SkipUnusedStages        bool
 }
 
 // WarmerOptions are options that are set by command line arguments to the cache warmer.

pkg/dockerfile/dockerfile.go

@@ -22,6 +22,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"regexp"
+	"strconv"
 	"strings"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -253,6 +254,9 @@ func MakeKanikoStages(opts *config.KanikoOptions, stages []instructions.Stage, m
 	if err := resolveStagesArgs(stages, args); err != nil {
 		return nil, errors.Wrap(err, "resolving args")
 	}
+	if opts.SkipUnusedStages {
+		stages = skipUnusedStages(stages, &targetStage, opts.Target)
+	}
 	var kanikoStages []config.KanikoStage
 	for index, stage := range stages {
 		if len(stage.Name) > 0 {
@@ -312,3 +316,53 @@ func unifyArgs(metaArgs []instructions.ArgCommand, buildArgs []string) []string
 	}
 	return args
 }
+
+// skipUnusedStages returns the list of used stages without the unnecessaries ones
+func skipUnusedStages(stages []instructions.Stage, lastStageIndex *int, target string) []instructions.Stage {
+	stagesDependencies := make(map[string]bool)
+	var onlyUsedStages []instructions.Stage
+	idx := *lastStageIndex
+
+	lastStageBaseName := stages[idx].BaseName
+
+	for i := idx; i >= 0; i-- {
+		s := stages[i]
+		if (s.Name != "" && stagesDependencies[s.Name]) || s.Name == lastStageBaseName || i == idx {
+			for _, c := range s.Commands {
+				switch cmd := c.(type) {
+				case *instructions.CopyCommand:
+					stageName := cmd.From
+					if copyFromIndex, err := strconv.Atoi(stageName); err == nil {
+						stageName = stages[copyFromIndex].Name
+					}
+					if !stagesDependencies[stageName] {
+						stagesDependencies[stageName] = true
+					}
+				}
+			}
+			if i != idx {
+				stagesDependencies[s.BaseName] = true
+			}
+		}
+	}
+	dependenciesLen := len(stagesDependencies)
+	if target == "" && dependenciesLen == 0 {
+		return stages
+	} else if dependenciesLen > 0 {
+		for i := 0; i < idx; i++ {
+			if stages[i].Name == "" {
+				continue
+			}
+			s := stages[i]
+			if stagesDependencies[s.Name] || s.Name == lastStageBaseName {
+				onlyUsedStages = append(onlyUsedStages, s)
+			}
+		}
+	}
+	onlyUsedStages = append(onlyUsedStages, stages[idx])
+	if idx > len(onlyUsedStages)-1 {
+		*lastStageIndex = len(onlyUsedStages) - 1
+	}
+
+	return onlyUsedStages
+}

pkg/dockerfile/dockerfile_test.go

@@ -456,3 +456,193 @@ func Test_ResolveStagesArgs(t *testing.T) {
 		}
 	}
 }
+
+func Test_SkipingUnusedStages(t *testing.T) {
+	tests := []struct {
+		description                   string
+		dockerfile                    string
+		targets                       []string
+		expectedSourceCodes           map[string][]string
+		expectedTargetIndexBeforeSkip map[string]int
+		expectedTargetIndexAfterSkip  map[string]int
+	}{
+		{
+			description: "dockerfile_without_copyFrom",
+			dockerfile: `
+			FROM alpine:3.11 AS base-dev
+			RUN echo dev > /hi
+			FROM alpine:3.11 AS base-prod
+			RUN echo prod > /hi
+			FROM base-dev as final-stage
+			RUN cat /hi
+			`,
+			targets: []string{"base-dev", "base-prod", ""},
+			expectedSourceCodes: map[string][]string{
+				"base-dev":  {"FROM alpine:3.11 AS base-dev"},
+				"base-prod": {"FROM alpine:3.11 AS base-prod"},
+				"":          {"FROM alpine:3.11 AS base-dev", "FROM base-dev as final-stage"},
+			},
+			expectedTargetIndexBeforeSkip: map[string]int{
+				"base-dev":  0,
+				"base-prod": 1,
+				"":          2,
+			},
+			expectedTargetIndexAfterSkip: map[string]int{
+				"base-dev":  0,
+				"base-prod": 0,
+				"":          1,
+			},
+		},
+		{
+			description: "dockerfile_with_copyFrom",
+			dockerfile: `
+			FROM alpine:3.11 AS base-dev
+			RUN echo dev > /hi
+			FROM alpine:3.11 AS base-prod
+			RUN echo prod > /hi
+			FROM alpine:3.11
+			COPY --from=base-prod /hi /finalhi
+			RUN cat /finalhi
+			`,
+			targets: []string{"base-dev", "base-prod", ""},
+			expectedSourceCodes: map[string][]string{
+				"base-dev":  {"FROM alpine:3.11 AS base-dev"},
+				"base-prod": {"FROM alpine:3.11 AS base-prod"},
+				"":          {"FROM alpine:3.11 AS base-prod", "FROM alpine:3.11"},
+			},
+			expectedTargetIndexBeforeSkip: map[string]int{
+				"base-dev":  0,
+				"base-prod": 1,
+				"":          2,
+			},
+			expectedTargetIndexAfterSkip: map[string]int{
+				"base-dev":  0,
+				"base-prod": 0,
+				"":          1,
+			},
+		},
+		{
+			description: "dockerfile_with_two_copyFrom",
+			dockerfile: `
+			FROM alpine:3.11 AS base-dev
+			RUN echo dev > /hi
+			FROM alpine:3.11 AS base-prod
+			RUN echo prod > /hi
+			FROM alpine:3.11
+			COPY --from=base-dev /hi /finalhidev
+			COPY --from=base-prod /hi /finalhiprod
+			RUN cat /finalhidev
+			RUN cat /finalhiprod
+			`,
+			targets: []string{"base-dev", "base-prod", ""},
+			expectedSourceCodes: map[string][]string{
+				"base-dev":  {"FROM alpine:3.11 AS base-dev"},
+				"base-prod": {"FROM alpine:3.11 AS base-prod"},
+				"":          {"FROM alpine:3.11 AS base-dev", "FROM alpine:3.11 AS base-prod", "FROM alpine:3.11"},
+			},
+			expectedTargetIndexBeforeSkip: map[string]int{
+				"base-dev":  0,
+				"base-prod": 1,
+				"":          2,
+			},
+			expectedTargetIndexAfterSkip: map[string]int{
+				"base-dev":  0,
+				"base-prod": 0,
+				"":          2,
+			},
+		},
+		{
+			description: "dockerfile_with_two_copyFrom_and_arg",
+			dockerfile: `
+			FROM debian:9.11 as base
+			COPY . .
+			FROM scratch as second
+			ENV foopath context/foo
+			COPY --from=0 $foopath context/b* /foo/
+			FROM second as third
+			COPY --from=base /context/foo /new/foo
+			FROM base as fourth
+			# Make sure that we snapshot intermediate images correctly
+			RUN date > /date
+			ENV foo bar
+			# This base image contains symlinks with relative paths to whitelisted directories
+			# We need to test they're extracted correctly
+			FROM fedora@sha256:c4cc32b09c6ae3f1353e7e33a8dda93dc41676b923d6d89afa996b421cc5aa48
+			FROM fourth
+			ARG file=/foo2
+			COPY --from=second /foo ${file}
+			COPY --from=debian:9.11 /etc/os-release /new
+			`,
+			targets: []string{"base", ""},
+			expectedSourceCodes: map[string][]string{
+				"base":   {"FROM debian:9.11 as base"},
+				"second": {"FROM debian:9.11 as base", "FROM scratch as second"},
+				"":       {"FROM debian:9.11 as base", "FROM scratch as second", "FROM base as fourth", "FROM fourth"},
+			},
+			expectedTargetIndexBeforeSkip: map[string]int{
+				"base":   0,
+				"second": 1,
+				"":       5,
+			},
+			expectedTargetIndexAfterSkip: map[string]int{
+				"base":   0,
+				"second": 1,
+				"":       3,
+			},
+		},
+		{
+			description: "dockerfile_without_final_dependencies",
+			dockerfile: `
+			FROM alpine:3.11
+			FROM debian:9.11 as base
+			RUN echo foo > /foo
+			FROM debian:9.11 as fizz
+			RUN echo fizz >> /fizz
+			COPY --from=base /foo /fizz
+			FROM alpine:3.11 as buzz
+			RUN echo buzz > /buzz
+			FROM alpine:3.11 as final
+			RUN echo bar > /bar
+			`,
+			targets: []string{"final", "buzz", "fizz", ""},
+			expectedSourceCodes: map[string][]string{
+				"final": {"FROM alpine:3.11 as final"},
+				"buzz":  {"FROM alpine:3.11 as buzz"},
+				"fizz":  {"FROM debian:9.11 as base", "FROM debian:9.11 as fizz"},
+				"":      {"FROM alpine:3.11", "FROM debian:9.11 as base", "FROM debian:9.11 as fizz", "FROM alpine:3.11 as buzz", "FROM alpine:3.11 as final"},
+			},
+			expectedTargetIndexBeforeSkip: map[string]int{
+				"final": 4,
+				"buzz":  3,
+				"fizz":  2,
+				"":      4,
+			},
+			expectedTargetIndexAfterSkip: map[string]int{
+				"final": 0,
+				"buzz":  0,
+				"fizz":  1,
+				"":      4,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		stages, _, err := Parse([]byte(test.dockerfile))
+		testutil.CheckError(t, false, err)
+		actualSourceCodes := make(map[string][]string)
+		for _, target := range test.targets {
+			targetIndex, err := targetStage(stages, target)
+			testutil.CheckError(t, false, err)
+			targetIndexBeforeSkip := targetIndex
+			onlyUsedStages := skipUnusedStages(stages, &targetIndex, target)
+			for _, s := range onlyUsedStages {
+				actualSourceCodes[target] = append(actualSourceCodes[target], s.SourceCode)
+			}
+			t.Run(test.description, func(t *testing.T) {
+				testutil.CheckDeepEqual(t, test.expectedSourceCodes[target], actualSourceCodes[target])
+				testutil.CheckDeepEqual(t, test.expectedTargetIndexBeforeSkip[target], targetIndexBeforeSkip)
+				testutil.CheckDeepEqual(t, test.expectedTargetIndexAfterSkip[target], targetIndex)
+			})
+		}
+	}
+}