public_git
/
kaniko
mirror of https://github.com/GoogleContainerTools/kaniko
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1165 from JordanGoasdoue/multistage-now-respects-dependencies 

feat: multistages now respect dependencies without building unnecessary stages
This commit is contained in:
Tejal Desai 2020-05-01 10:34:47 -07:00 committed by GitHub
parent 7eb2458ba4 8cbc7a8ca2
commit 7ccf05fae3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 256 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
README.md
View File

@ -4,7 +4,7 @@
![kaniko logo](logo/Kaniko-Logo.png)
kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster.
kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster.
kaniko doesn't depend on a Docker daemon and executes each command within a Dockerfile completely in userspace.
This enables building container images in environments that can't easily or securely run a Docker daemon, such as a standard Kubernetes cluster.
@ -15,7 +15,7 @@ We'd love to hear from you! Join us on [#kaniko Kubernetes Slack](https://kuber
:mega: **Please fill out our [quick 5-question survey](https://forms.gle/HhZGEM33x4FUz9Qa6)** so that we can learn how satisfied you are with Kaniko, and what improvements we should make. Thank you! :dancers:
Kaniko is not an officially supported Google project.
Kaniko is not an officially supported Google project.
_If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPMENT.md) and [CONTRIBUTING.md](CONTRIBUTING.md)._
@ -50,6 +50,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME
- [--cache](#--cache)
- [--cache-dir](#--cache-dir)
- [--cache-repo](#--cache-repo)
- [--context-sub-path](#context-sub-path)
- [--digest-file](#--digest-file)
- [--oci-layout-path](#--oci-layout-path)
- [--insecure-registry](#--insecure-registry)
@ -69,6 +70,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME
- [--verbosity](#--verbosity)
- [--whitelist-var-run](#--whitelist-var-run)
- [--label](#--label)
- [--skip-unused-stages](#skip-unused-stages)
- [Debug Image](#debug-image)
- [Security](#security)
- [Comparison with Other Tools](#comparison-with-other-tools)
@ -302,7 +304,7 @@ There is also a utility script [`run_in_docker.sh`](./run_in_docker.sh) that can
./run_in_docker.sh <path to Dockerfile> <path to build context> <destination of final image>
```
_NOTE: `run_in_docker.sh` expects a path to a
_NOTE: `run_in_docker.sh` expects a path to a
Dockerfile relative to the absolute path of the build context._
An example run, specifying the Dockerfile in the container directory `/workspace`, the build
@ -558,6 +560,11 @@ Ignore /var/run when taking image snapshot. Set it to false to preserve /var/run
Set this flag as `--label key=value` to set some metadata to the final image. This is equivalent as using the `LABEL` within the Dockerfile.
#### --skip-unused-stages
This flag builds only used stages if defined to `true`.
Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile
### Debug Image
The kaniko executor image is based on scratch and doesn't contain a shell.

1
cmd/executor/cmd/root.go
View File

@ -161,6 +161,7 @@ func addKanikoOptionsFlags() {
RootCmd.PersistentFlags().StringVarP(&opts.RegistryMirror, "registry-mirror", "", "", "Registry mirror to use has pull-through cache instead of docker.io.")
RootCmd.PersistentFlags().BoolVarP(&opts.WhitelistVarRun, "whitelist-var-run", "", true, "Ignore /var/run directory when taking image snapshot. Set it to false to preserve /var/run/ in destination image. (Default true).")
RootCmd.PersistentFlags().VarP(&opts.Labels, "label", "", "Set metadata for an image. Set it repeatedly for multiple labels.")
RootCmd.PersistentFlags().BoolVarP(&opts.SkipUnusedStages, "skip-unused-stages", "", false, "Build only used stages if defined to true. Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile")
}
// addHiddenFlags marks certain flags as hidden from the executor help text

1
pkg/config/options.go
View File

@ -56,6 +56,7 @@ type KanikoOptions struct {
Cache bool
Cleanup bool
WhitelistVarRun bool
SkipUnusedStages bool
}
// WarmerOptions are options that are set by command line arguments to the cache warmer.

54
pkg/dockerfile/dockerfile.go
View File

@ -22,6 +22,7 @@ import (
"io/ioutil"
"net/http"
"regexp"
"strconv"
"strings"
v1 "github.com/google/go-containerregistry/pkg/v1"
@ -253,6 +254,9 @@ func MakeKanikoStages(opts *config.KanikoOptions, stages []instructions.Stage, m
if err := resolveStagesArgs(stages, args); err != nil {
return nil, errors.Wrap(err, "resolving args")
}
if opts.SkipUnusedStages {
stages = skipUnusedStages(stages, &targetStage, opts.Target)
}
var kanikoStages []config.KanikoStage
for index, stage := range stages {
if len(stage.Name) > 0 {
@ -312,3 +316,53 @@ func unifyArgs(metaArgs []instructions.ArgCommand, buildArgs []string) []string
}
return args
}
// skipUnusedStages returns the list of used stages without the unnecessaries ones
func skipUnusedStages(stages []instructions.Stage, lastStageIndex *int, target string) []instructions.Stage {
stagesDependencies := make(map[string]bool)
var onlyUsedStages []instructions.Stage
idx := *lastStageIndex
lastStageBaseName := stages[idx].BaseName
for i := idx; i >= 0; i-- {
s := stages[i]
if (s.Name != "" && stagesDependencies[s.Name]) || s.Name == lastStageBaseName || i == idx {
for _, c := range s.Commands {
switch cmd := c.(type) {
case *instructions.CopyCommand:
stageName := cmd.From
if copyFromIndex, err := strconv.Atoi(stageName); err == nil {
stageName = stages[copyFromIndex].Name
}
if !stagesDependencies[stageName] {
stagesDependencies[stageName] = true
}
}
}
if i != idx {
stagesDependencies[s.BaseName] = true
}
}
}
dependenciesLen := len(stagesDependencies)
if target == "" && dependenciesLen == 0 {
return stages
} else if dependenciesLen > 0 {
for i := 0; i < idx; i++ {
if stages[i].Name == "" {
continue
}
s := stages[i]
if stagesDependencies[s.Name] || s.Name == lastStageBaseName {
onlyUsedStages = append(onlyUsedStages, s)
}
}
}
onlyUsedStages = append(onlyUsedStages, stages[idx])
if idx > len(onlyUsedStages)-1 {
*lastStageIndex = len(onlyUsedStages) - 1
}
return onlyUsedStages
}

190
pkg/dockerfile/dockerfile_test.go
View File

@ -456,3 +456,193 @@ func Test_ResolveStagesArgs(t *testing.T) {
}
}
}
func Test_SkipingUnusedStages(t *testing.T) {
tests := []struct {
description string
dockerfile string
targets []string
expectedSourceCodes map[string][]string
expectedTargetIndexBeforeSkip map[string]int
expectedTargetIndexAfterSkip map[string]int
}{
{
description: "dockerfile_without_copyFrom",
dockerfile: `
FROM alpine:3.11 AS base-dev
RUN echo dev > /hi
FROM alpine:3.11 AS base-prod
RUN echo prod > /hi
FROM base-dev as final-stage
RUN cat /hi
`,
targets: []string{"base-dev", "base-prod", ""},
expectedSourceCodes: map[string][]string{
"base-dev": {"FROM alpine:3.11 AS base-dev"},
"base-prod": {"FROM alpine:3.11 AS base-prod"},
"": {"FROM alpine:3.11 AS base-dev", "FROM base-dev as final-stage"},
},
expectedTargetIndexBeforeSkip: map[string]int{
"base-dev": 0,
"base-prod": 1,
"": 2,
},
expectedTargetIndexAfterSkip: map[string]int{
"base-dev": 0,
"base-prod": 0,
"": 1,
},
},
{
description: "dockerfile_with_copyFrom",
dockerfile: `
FROM alpine:3.11 AS base-dev
RUN echo dev > /hi
FROM alpine:3.11 AS base-prod
RUN echo prod > /hi
FROM alpine:3.11
COPY --from=base-prod /hi /finalhi
RUN cat /finalhi
`,
targets: []string{"base-dev", "base-prod", ""},
expectedSourceCodes: map[string][]string{
"base-dev": {"FROM alpine:3.11 AS base-dev"},
"base-prod": {"FROM alpine:3.11 AS base-prod"},
"": {"FROM alpine:3.11 AS base-prod", "FROM alpine:3.11"},
},
expectedTargetIndexBeforeSkip: map[string]int{
"base-dev": 0,
"base-prod": 1,
"": 2,
},
expectedTargetIndexAfterSkip: map[string]int{
"base-dev": 0,
"base-prod": 0,
"": 1,
},
},
{
description: "dockerfile_with_two_copyFrom",
dockerfile: `
FROM alpine:3.11 AS base-dev
RUN echo dev > /hi
FROM alpine:3.11 AS base-prod
RUN echo prod > /hi
FROM alpine:3.11
COPY --from=base-dev /hi /finalhidev
COPY --from=base-prod /hi /finalhiprod
RUN cat /finalhidev
RUN cat /finalhiprod
`,
targets: []string{"base-dev", "base-prod", ""},
expectedSourceCodes: map[string][]string{
"base-dev": {"FROM alpine:3.11 AS base-dev"},
"base-prod": {"FROM alpine:3.11 AS base-prod"},
"": {"FROM alpine:3.11 AS base-dev", "FROM alpine:3.11 AS base-prod", "FROM alpine:3.11"},
},
expectedTargetIndexBeforeSkip: map[string]int{
"base-dev": 0,
"base-prod": 1,
"": 2,
},
expectedTargetIndexAfterSkip: map[string]int{
"base-dev": 0,
"base-prod": 0,
"": 2,
},
},
{
description: "dockerfile_with_two_copyFrom_and_arg",
dockerfile: `
FROM debian:9.11 as base
COPY . .
FROM scratch as second
ENV foopath context/foo
COPY --from=0 $foopath context/b* /foo/
FROM second as third
COPY --from=base /context/foo /new/foo
FROM base as fourth
# Make sure that we snapshot intermediate images correctly
RUN date > /date
ENV foo bar
# This base image contains symlinks with relative paths to whitelisted directories
# We need to test they're extracted correctly
FROM fedora@sha256:c4cc32b09c6ae3f1353e7e33a8dda93dc41676b923d6d89afa996b421cc5aa48
FROM fourth
ARG file=/foo2
COPY --from=second /foo ${file}
COPY --from=debian:9.11 /etc/os-release /new
`,
targets: []string{"base", ""},
expectedSourceCodes: map[string][]string{
"base": {"FROM debian:9.11 as base"},
"second": {"FROM debian:9.11 as base", "FROM scratch as second"},
"": {"FROM debian:9.11 as base", "FROM scratch as second", "FROM base as fourth", "FROM fourth"},
},
expectedTargetIndexBeforeSkip: map[string]int{
"base": 0,
"second": 1,
"": 5,
},
expectedTargetIndexAfterSkip: map[string]int{
"base": 0,
"second": 1,
"": 3,
},
},
{
description: "dockerfile_without_final_dependencies",
dockerfile: `
FROM alpine:3.11
FROM debian:9.11 as base
RUN echo foo > /foo
FROM debian:9.11 as fizz
RUN echo fizz >> /fizz
COPY --from=base /foo /fizz
FROM alpine:3.11 as buzz
RUN echo buzz > /buzz
FROM alpine:3.11 as final
RUN echo bar > /bar
`,
targets: []string{"final", "buzz", "fizz", ""},
expectedSourceCodes: map[string][]string{
"final": {"FROM alpine:3.11 as final"},
"buzz": {"FROM alpine:3.11 as buzz"},
"fizz": {"FROM debian:9.11 as base", "FROM debian:9.11 as fizz"},
"": {"FROM alpine:3.11", "FROM debian:9.11 as base", "FROM debian:9.11 as fizz", "FROM alpine:3.11 as buzz", "FROM alpine:3.11 as final"},
},
expectedTargetIndexBeforeSkip: map[string]int{
"final": 4,
"buzz": 3,
"fizz": 2,
"": 4,
},
expectedTargetIndexAfterSkip: map[string]int{
"final": 0,
"buzz": 0,
"fizz": 1,
"": 4,
},
},
}
for _, test := range tests {
stages, _, err := Parse([]byte(test.dockerfile))
testutil.CheckError(t, false, err)
actualSourceCodes := make(map[string][]string)
for _, target := range test.targets {
targetIndex, err := targetStage(stages, target)
testutil.CheckError(t, false, err)
targetIndexBeforeSkip := targetIndex
onlyUsedStages := skipUnusedStages(stages, &targetIndex, target)
for _, s := range onlyUsedStages {
actualSourceCodes[target] = append(actualSourceCodes[target], s.SourceCode)
}
t.Run(test.description, func(t *testing.T) {
testutil.CheckDeepEqual(t, test.expectedSourceCodes[target], actualSourceCodes[target])
testutil.CheckDeepEqual(t, test.expectedTargetIndexBeforeSkip[target], targetIndexBeforeSkip)
testutil.CheckDeepEqual(t, test.expectedTargetIndexAfterSkip[target], targetIndex)
})
}
}
}