diff --git a/README.md b/README.md index 897d832d3..969c1417b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ![kaniko logo](logo/Kaniko-Logo.png) -kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster. +kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster. kaniko doesn't depend on a Docker daemon and executes each command within a Dockerfile completely in userspace. This enables building container images in environments that can't easily or securely run a Docker daemon, such as a standard Kubernetes cluster. @@ -15,7 +15,7 @@ We'd love to hear from you! Join us on [#kaniko Kubernetes Slack](https://kuber :mega: **Please fill out our [quick 5-question survey](https://forms.gle/HhZGEM33x4FUz9Qa6)** so that we can learn how satisfied you are with Kaniko, and what improvements we should make. Thank you! :dancers: -Kaniko is not an officially supported Google project. +Kaniko is not an officially supported Google project. _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPMENT.md) and [CONTRIBUTING.md](CONTRIBUTING.md)._ @@ -50,6 +50,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME - [--cache](#--cache) - [--cache-dir](#--cache-dir) - [--cache-repo](#--cache-repo) + - [--context-sub-path](#context-sub-path) - [--digest-file](#--digest-file) - [--oci-layout-path](#--oci-layout-path) - [--insecure-registry](#--insecure-registry) @@ -69,6 +70,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME - [--verbosity](#--verbosity) - [--whitelist-var-run](#--whitelist-var-run) - [--label](#--label) + - [--skip-unused-stages](#skip-unused-stages) - [Debug Image](#debug-image) - [Security](#security) - [Comparison with Other Tools](#comparison-with-other-tools) @@ -302,7 +304,7 @@ There is also a utility script [`run_in_docker.sh`](./run_in_docker.sh) that can ./run_in_docker.sh ``` -_NOTE: `run_in_docker.sh` expects a path to a +_NOTE: `run_in_docker.sh` expects a path to a Dockerfile relative to the absolute path of the build context._ An example run, specifying the Dockerfile in the container directory `/workspace`, the build @@ -558,6 +560,11 @@ Ignore /var/run when taking image snapshot. Set it to false to preserve /var/run Set this flag as `--label key=value` to set some metadata to the final image. This is equivalent as using the `LABEL` within the Dockerfile. +#### --skip-unused-stages + +This flag builds only used stages if defined to `true`. +Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile + ### Debug Image The kaniko executor image is based on scratch and doesn't contain a shell. diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index db27d75f3..f0b50cc61 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -161,6 +161,7 @@ func addKanikoOptionsFlags() { RootCmd.PersistentFlags().StringVarP(&opts.RegistryMirror, "registry-mirror", "", "", "Registry mirror to use has pull-through cache instead of docker.io.") RootCmd.PersistentFlags().BoolVarP(&opts.WhitelistVarRun, "whitelist-var-run", "", true, "Ignore /var/run directory when taking image snapshot. Set it to false to preserve /var/run/ in destination image. (Default true).") RootCmd.PersistentFlags().VarP(&opts.Labels, "label", "", "Set metadata for an image. Set it repeatedly for multiple labels.") + RootCmd.PersistentFlags().BoolVarP(&opts.SkipUnusedStages, "skip-unused-stages", "", false, "Build only used stages if defined to true. Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile") } // addHiddenFlags marks certain flags as hidden from the executor help text diff --git a/pkg/config/options.go b/pkg/config/options.go index 47436a5fd..6c8690284 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -56,6 +56,7 @@ type KanikoOptions struct { Cache bool Cleanup bool WhitelistVarRun bool + SkipUnusedStages bool } // WarmerOptions are options that are set by command line arguments to the cache warmer. diff --git a/pkg/dockerfile/dockerfile.go b/pkg/dockerfile/dockerfile.go index 3e41f40de..5ca8f1eea 100644 --- a/pkg/dockerfile/dockerfile.go +++ b/pkg/dockerfile/dockerfile.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "net/http" "regexp" + "strconv" "strings" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -253,6 +254,9 @@ func MakeKanikoStages(opts *config.KanikoOptions, stages []instructions.Stage, m if err := resolveStagesArgs(stages, args); err != nil { return nil, errors.Wrap(err, "resolving args") } + if opts.SkipUnusedStages { + stages = skipUnusedStages(stages, &targetStage, opts.Target) + } var kanikoStages []config.KanikoStage for index, stage := range stages { if len(stage.Name) > 0 { @@ -312,3 +316,53 @@ func unifyArgs(metaArgs []instructions.ArgCommand, buildArgs []string) []string } return args } + +// skipUnusedStages returns the list of used stages without the unnecessaries ones +func skipUnusedStages(stages []instructions.Stage, lastStageIndex *int, target string) []instructions.Stage { + stagesDependencies := make(map[string]bool) + var onlyUsedStages []instructions.Stage + idx := *lastStageIndex + + lastStageBaseName := stages[idx].BaseName + + for i := idx; i >= 0; i-- { + s := stages[i] + if (s.Name != "" && stagesDependencies[s.Name]) || s.Name == lastStageBaseName || i == idx { + for _, c := range s.Commands { + switch cmd := c.(type) { + case *instructions.CopyCommand: + stageName := cmd.From + if copyFromIndex, err := strconv.Atoi(stageName); err == nil { + stageName = stages[copyFromIndex].Name + } + if !stagesDependencies[stageName] { + stagesDependencies[stageName] = true + } + } + } + if i != idx { + stagesDependencies[s.BaseName] = true + } + } + } + dependenciesLen := len(stagesDependencies) + if target == "" && dependenciesLen == 0 { + return stages + } else if dependenciesLen > 0 { + for i := 0; i < idx; i++ { + if stages[i].Name == "" { + continue + } + s := stages[i] + if stagesDependencies[s.Name] || s.Name == lastStageBaseName { + onlyUsedStages = append(onlyUsedStages, s) + } + } + } + onlyUsedStages = append(onlyUsedStages, stages[idx]) + if idx > len(onlyUsedStages)-1 { + *lastStageIndex = len(onlyUsedStages) - 1 + } + + return onlyUsedStages +} diff --git a/pkg/dockerfile/dockerfile_test.go b/pkg/dockerfile/dockerfile_test.go index 8fb612297..63c66f4b2 100644 --- a/pkg/dockerfile/dockerfile_test.go +++ b/pkg/dockerfile/dockerfile_test.go @@ -456,3 +456,193 @@ func Test_ResolveStagesArgs(t *testing.T) { } } } + +func Test_SkipingUnusedStages(t *testing.T) { + tests := []struct { + description string + dockerfile string + targets []string + expectedSourceCodes map[string][]string + expectedTargetIndexBeforeSkip map[string]int + expectedTargetIndexAfterSkip map[string]int + }{ + { + description: "dockerfile_without_copyFrom", + dockerfile: ` + FROM alpine:3.11 AS base-dev + RUN echo dev > /hi + FROM alpine:3.11 AS base-prod + RUN echo prod > /hi + FROM base-dev as final-stage + RUN cat /hi + `, + targets: []string{"base-dev", "base-prod", ""}, + expectedSourceCodes: map[string][]string{ + "base-dev": {"FROM alpine:3.11 AS base-dev"}, + "base-prod": {"FROM alpine:3.11 AS base-prod"}, + "": {"FROM alpine:3.11 AS base-dev", "FROM base-dev as final-stage"}, + }, + expectedTargetIndexBeforeSkip: map[string]int{ + "base-dev": 0, + "base-prod": 1, + "": 2, + }, + expectedTargetIndexAfterSkip: map[string]int{ + "base-dev": 0, + "base-prod": 0, + "": 1, + }, + }, + { + description: "dockerfile_with_copyFrom", + dockerfile: ` + FROM alpine:3.11 AS base-dev + RUN echo dev > /hi + FROM alpine:3.11 AS base-prod + RUN echo prod > /hi + FROM alpine:3.11 + COPY --from=base-prod /hi /finalhi + RUN cat /finalhi + `, + targets: []string{"base-dev", "base-prod", ""}, + expectedSourceCodes: map[string][]string{ + "base-dev": {"FROM alpine:3.11 AS base-dev"}, + "base-prod": {"FROM alpine:3.11 AS base-prod"}, + "": {"FROM alpine:3.11 AS base-prod", "FROM alpine:3.11"}, + }, + expectedTargetIndexBeforeSkip: map[string]int{ + "base-dev": 0, + "base-prod": 1, + "": 2, + }, + expectedTargetIndexAfterSkip: map[string]int{ + "base-dev": 0, + "base-prod": 0, + "": 1, + }, + }, + { + description: "dockerfile_with_two_copyFrom", + dockerfile: ` + FROM alpine:3.11 AS base-dev + RUN echo dev > /hi + FROM alpine:3.11 AS base-prod + RUN echo prod > /hi + FROM alpine:3.11 + COPY --from=base-dev /hi /finalhidev + COPY --from=base-prod /hi /finalhiprod + RUN cat /finalhidev + RUN cat /finalhiprod + `, + targets: []string{"base-dev", "base-prod", ""}, + expectedSourceCodes: map[string][]string{ + "base-dev": {"FROM alpine:3.11 AS base-dev"}, + "base-prod": {"FROM alpine:3.11 AS base-prod"}, + "": {"FROM alpine:3.11 AS base-dev", "FROM alpine:3.11 AS base-prod", "FROM alpine:3.11"}, + }, + expectedTargetIndexBeforeSkip: map[string]int{ + "base-dev": 0, + "base-prod": 1, + "": 2, + }, + expectedTargetIndexAfterSkip: map[string]int{ + "base-dev": 0, + "base-prod": 0, + "": 2, + }, + }, + { + description: "dockerfile_with_two_copyFrom_and_arg", + dockerfile: ` + FROM debian:9.11 as base + COPY . . + FROM scratch as second + ENV foopath context/foo + COPY --from=0 $foopath context/b* /foo/ + FROM second as third + COPY --from=base /context/foo /new/foo + FROM base as fourth + # Make sure that we snapshot intermediate images correctly + RUN date > /date + ENV foo bar + # This base image contains symlinks with relative paths to whitelisted directories + # We need to test they're extracted correctly + FROM fedora@sha256:c4cc32b09c6ae3f1353e7e33a8dda93dc41676b923d6d89afa996b421cc5aa48 + FROM fourth + ARG file=/foo2 + COPY --from=second /foo ${file} + COPY --from=debian:9.11 /etc/os-release /new + `, + targets: []string{"base", ""}, + expectedSourceCodes: map[string][]string{ + "base": {"FROM debian:9.11 as base"}, + "second": {"FROM debian:9.11 as base", "FROM scratch as second"}, + "": {"FROM debian:9.11 as base", "FROM scratch as second", "FROM base as fourth", "FROM fourth"}, + }, + expectedTargetIndexBeforeSkip: map[string]int{ + "base": 0, + "second": 1, + "": 5, + }, + expectedTargetIndexAfterSkip: map[string]int{ + "base": 0, + "second": 1, + "": 3, + }, + }, + { + description: "dockerfile_without_final_dependencies", + dockerfile: ` + FROM alpine:3.11 + FROM debian:9.11 as base + RUN echo foo > /foo + FROM debian:9.11 as fizz + RUN echo fizz >> /fizz + COPY --from=base /foo /fizz + FROM alpine:3.11 as buzz + RUN echo buzz > /buzz + FROM alpine:3.11 as final + RUN echo bar > /bar + `, + targets: []string{"final", "buzz", "fizz", ""}, + expectedSourceCodes: map[string][]string{ + "final": {"FROM alpine:3.11 as final"}, + "buzz": {"FROM alpine:3.11 as buzz"}, + "fizz": {"FROM debian:9.11 as base", "FROM debian:9.11 as fizz"}, + "": {"FROM alpine:3.11", "FROM debian:9.11 as base", "FROM debian:9.11 as fizz", "FROM alpine:3.11 as buzz", "FROM alpine:3.11 as final"}, + }, + expectedTargetIndexBeforeSkip: map[string]int{ + "final": 4, + "buzz": 3, + "fizz": 2, + "": 4, + }, + expectedTargetIndexAfterSkip: map[string]int{ + "final": 0, + "buzz": 0, + "fizz": 1, + "": 4, + }, + }, + } + + for _, test := range tests { + stages, _, err := Parse([]byte(test.dockerfile)) + testutil.CheckError(t, false, err) + actualSourceCodes := make(map[string][]string) + for _, target := range test.targets { + targetIndex, err := targetStage(stages, target) + testutil.CheckError(t, false, err) + targetIndexBeforeSkip := targetIndex + onlyUsedStages := skipUnusedStages(stages, &targetIndex, target) + for _, s := range onlyUsedStages { + actualSourceCodes[target] = append(actualSourceCodes[target], s.SourceCode) + } + t.Run(test.description, func(t *testing.T) { + testutil.CheckDeepEqual(t, test.expectedSourceCodes[target], actualSourceCodes[target]) + testutil.CheckDeepEqual(t, test.expectedTargetIndexBeforeSkip[target], targetIndexBeforeSkip) + testutil.CheckDeepEqual(t, test.expectedTargetIndexAfterSkip[target], targetIndex) + }) + } + } +}