public_git
/
kaniko
mirror of https://github.com/GoogleContainerTools/kaniko
1
0
Fork 0
Code Issues Releases Wiki Activity

Optimize file copying and stage saving between stages. (#605) 

This change calculates the exact files and directories needed between
stages used in the COPY command. Instead of saving the entire
stage as a tarball, we now save only the necessary files.
This commit is contained in:
dlorenc 2019-03-13 07:47:28 -07:00 committed by GitHub
parent 6ce3dfb93a
commit 246cc92a33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 469 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Gopkg.lock generated
View File

@ -680,6 +680,14 @@
revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
version = "v1.0.2"
[[projects]]
branch = "master"
digest = "1:15057fc7395024283a7d2639b8afc61c5b6df3fe260ce06ff5834c8464f16b5c"
name = "github.com/otiai10/copy"
packages = ["."]
pruneopts = "NUT"
revision = "7e9a647135a142c2669943d4a4d29be015ce9392"
[[projects]]
branch = "master"
digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2"
@ -1204,6 +1212,7 @@
"github.com/moby/buildkit/frontend/dockerfile/instructions",
"github.com/moby/buildkit/frontend/dockerfile/parser",
"github.com/moby/buildkit/frontend/dockerfile/shell",
"github.com/otiai10/copy",
"github.com/pkg/errors",
"github.com/sirupsen/logrus",
"github.com/spf13/cobra",

4
pkg/commands/add.go
View File

@ -47,7 +47,7 @@ type AddCommand struct {
func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
srcs, dest, err := resolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs)
srcs, dest, err := util.ResolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs)
if err != nil {
return err
}
@ -114,7 +114,7 @@ func (a *AddCommand) String() string {
func (a *AddCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) {
replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
srcs, _, err := resolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs)
srcs, _, err := util.ResolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs)
if err != nil {
return nil, err
}

16
pkg/commands/copy.go
View File

@ -45,7 +45,7 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu
replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
srcs, dest, err := resolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs)
srcs, dest, err := util.ResolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs)
if err != nil {
return err
}
@ -100,18 +100,6 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu
return nil
}
func resolveEnvAndWildcards(sd instructions.SourcesAndDest, buildcontext string, envs []string) ([]string, string, error) {
// First, resolve any environment replacement
resolvedEnvs, err := util.ResolveEnvironmentReplacementList(sd, envs, true)
if err != nil {
return nil, "", err
}
dest := resolvedEnvs[len(resolvedEnvs)-1]
// Resolve wildcards and get a list of resolved sources
srcs, err := util.ResolveSources(resolvedEnvs, buildcontext)
return srcs, dest, err
}
// FilesToSnapshot should return an empty array if still nil; no files were changed
func (c *CopyCommand) FilesToSnapshot() []string {
return c.snapshotFiles
@ -129,7 +117,7 @@ func (c *CopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerf
}
replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
srcs, _, err := resolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs)
srcs, _, err := util.ResolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs)
if err != nil {
return nil, err
}

1
pkg/config/stage.go
View File

@ -26,4 +26,5 @@ type KanikoStage struct {
BaseImageStoredLocally bool
SaveStage bool
MetaArgs []instructions.ArgCommand
Index int
}

12
pkg/dockerfile/dockerfile.go
View File

@ -25,6 +25,8 @@ import (
"strconv"
"strings"
"github.com/sirupsen/logrus"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
@ -67,6 +69,7 @@ func Stages(opts *config.KanikoOptions) ([]config.KanikoStage, error) {
return nil, errors.Wrap(err, "resolving base name")
}
stage.Name = resolvedBaseName
logrus.Infof("Resolved base name %s to %s", stage.BaseName, stage.Name)
kanikoStages = append(kanikoStages, config.KanikoStage{
Stage: stage,
BaseImageIndex: baseImageIndex(index, stages),
@ -74,6 +77,7 @@ func Stages(opts *config.KanikoOptions) ([]config.KanikoStage, error) {
SaveStage: saveStage(index, stages),
Final: index == targetStage,
MetaArgs: metaArgs,
Index: index,
})
if index == targetStage {
break
@ -175,14 +179,6 @@ func saveStage(index int, stages []instructions.Stage) bool {
return true
}
}
for _, cmd := range stage.Commands {
switch c := cmd.(type) {
case *instructions.CopyCommand:
if c.From == strconv.Itoa(index) {
return true
}
}
}
}
return false
}

2
pkg/dockerfile/dockerfile_test.go
View File

@ -114,7 +114,7 @@ func Test_SaveStage(t *testing.T) {
{
name: "reference stage in later copy command",
index: 0,
expected: true,
expected: false,
},
{
name: "reference stage in later from command",

104
pkg/executor/build.go
View File

@ -23,6 +23,8 @@ import (
"strconv"
"time"
"github.com/otiai10/copy"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
@ -60,10 +62,11 @@ type stageBuilder struct {
opts *config.KanikoOptions
cmds []commands.DockerCommand
args *dockerfile.BuildArgs
crossStageDeps map[int][]string
}
// newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage
func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*stageBuilder, error) {
func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string) (*stageBuilder, error) {
sourceImage, err := util.RetrieveSourceImage(stage, opts)
if err != nil {
return nil, err
@ -96,6 +99,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*sta
snapshotter: snapshotter,
baseImageDigest: digest.String(),
opts: opts,
crossStageDeps: crossStageDeps,
}
for _, cmd := range s.stage.Commands {
@ -207,6 +211,10 @@ func (s *stageBuilder) build() error {
break
}
}
if len(s.crossStageDeps[s.stage.Index]) > 0 {
shouldUnpack = true
}
if shouldUnpack {
t := timing.Start("FS Unpacking")
if _, err := util.GetFSFromImage(constants.RootDir, s.image); err != nil {
@ -353,6 +361,63 @@ func (s *stageBuilder) saveSnapshotToImage(createdBy string, tarPath string) err
}
func CalculateDependencies(opts *config.KanikoOptions) (map[int][]string, error) {
stages, err := dockerfile.Stages(opts)
if err != nil {
return nil, err
}
images := []v1.Image{}
depGraph := map[int][]string{}
for _, s := range stages {
ba := dockerfile.NewBuildArgs(opts.BuildArgs)
ba.AddMetaArgs(s.MetaArgs)
var image v1.Image
var err error
if s.BaseImageStoredLocally {
image = images[s.BaseImageIndex]
} else if s.Name == constants.NoBaseImage {
image = empty.Image
} else {
image, err = util.RetrieveSourceImage(s, opts)
if err != nil {
return nil, err
}
}
initializeConfig(image)
cfg, err := image.ConfigFile()
if err != nil {
return nil, err
}
for _, c := range s.Commands {
switch cmd := c.(type) {
case *instructions.CopyCommand:
if cmd.From != "" {
i, err := strconv.Atoi(cmd.From)
if err != nil {
continue
}
resolved, err := util.ResolveEnvironmentReplacementList(cmd.SourcesAndDest, cfg.Config.Env, true)
if err != nil {
return nil, err
}
depGraph[i] = append(depGraph[i], resolved[0:len(resolved)-1]...)
}
case *instructions.EnvCommand:
if err := util.UpdateConfigEnv(cmd.Env, &cfg.Config, ba.ReplacementEnvs(cfg.Config.Env)); err != nil {
return nil, err
}
image, err = mutate.Config(image, cfg.Config)
if err != nil {
return nil, err
}
}
}
images = append(images, image)
}
return depGraph, nil
}
// DoBuild executes building the Dockerfile
func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
t := timing.Start("Total Build Time")
@ -369,8 +434,14 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
return nil, err
}
crossStageDependencies, err := CalculateDependencies(opts)
if err != nil {
return nil, err
}
logrus.Infof("Built cross stage deps: %v", crossStageDependencies)
for index, stage := range stages {
sb, err := newStageBuilder(opts, stage)
sb, err := newStageBuilder(opts, stage, crossStageDependencies)
if err != nil {
return nil, err
}
@ -405,10 +476,21 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
if err := saveStageAsTarball(strconv.Itoa(index), sourceImage); err != nil {
return nil, err
}
if err := extractImageToDependecyDir(strconv.Itoa(index), sourceImage); err != nil {
return nil, err
}
}
filesToSave, err := filesToSave(crossStageDependencies[index])
if err != nil {
return nil, err
}
dstDir := filepath.Join(constants.KanikoDir, strconv.Itoa(index))
if err := os.MkdirAll(dstDir, 0644); err != nil {
return nil, err
}
for _, p := range filesToSave {
logrus.Infof("Saving file %s for later use.", p)
copy.Copy(p, filepath.Join(dstDir, p))
}
// Delete the filesystem
if err := util.DeleteFilesystem(); err != nil {
return nil, err
@ -418,6 +500,18 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
return nil, err
}
func filesToSave(deps []string) ([]string, error) {
allFiles := []string{}
for _, src := range deps {
srcs, err := filepath.Glob(src)
if err != nil {
return nil, err
}
allFiles = append(allFiles, srcs...)
}
return allFiles, nil
}
func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) error {
t := timing.Start("Fetching Extra Stages")
defer timing.DefaultRun.Stop(t)

209
pkg/executor/build_test.go
View File

@ -17,14 +17,19 @@ limitations under the License.
package executor
import (
"io/ioutil"
"os"
"path/filepath"
"reflect"
"sort"
"testing"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-cmp/cmp"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
)
func Test_reviewConfig(t *testing.T) {
@ -180,3 +185,201 @@ func Test_stageBuilder_shouldTakeSnapshot(t *testing.T) {
})
}
}
func TestCalculateDependencies(t *testing.T) {
type args struct {
dockerfile string
}
tests := []struct {
name string
args args
want map[int][]string
}{
{
name: "no deps",
args: args{
dockerfile: `
FROM debian as stage1
RUN foo
FROM stage1
RUN bar
`,
},
want: map[int][]string{},
},
{
name: "simple deps",
args: args{
dockerfile: `
FROM debian as stage1
FROM alpine
COPY --from=stage1 /foo /bar
`,
},
want: map[int][]string{
0: {"/foo"},
},
},
{
name: "two sets deps",
args: args{
dockerfile: `
FROM debian as stage1
FROM ubuntu as stage2
RUN foo
COPY --from=stage1 /foo /bar
FROM alpine
COPY --from=stage2 /bar /bat
`,
},
want: map[int][]string{
0: {"/foo"},
1: {"/bar"},
},
},
{
name: "double deps",
args: args{
dockerfile: `
FROM debian as stage1
FROM ubuntu as stage2
RUN foo
COPY --from=stage1 /foo /bar
FROM alpine
COPY --from=stage1 /baz /bat
`,
},
want: map[int][]string{
0: {"/foo", "/baz"},
},
},
{
name: "envs in deps",
args: args{
dockerfile: `
FROM debian as stage1
FROM ubuntu as stage2
RUN foo
ENV key1 val1
ENV key2 val2
COPY --from=stage1 /foo/$key1 /foo/$key2 /bar
FROM alpine
COPY --from=stage2 /bar /bat
`,
},
want: map[int][]string{
0: {"/foo/val1", "/foo/val2"},
1: {"/bar"},
},
},
{
name: "envs from base image in deps",
args: args{
dockerfile: `
FROM debian as stage1
ENV key1 baseval1
FROM stage1 as stage2
RUN foo
ENV key2 val2
COPY --from=stage1 /foo/$key1 /foo/$key2 /bar
FROM alpine
COPY --from=stage2 /bar /bat
`,
},
want: map[int][]string{
0: {"/foo/baseval1", "/foo/val2"},
1: {"/bar"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f, _ := ioutil.TempFile("", "")
ioutil.WriteFile(f.Name(), []byte(tt.args.dockerfile), 0755)
opts := &config.KanikoOptions{
DockerfilePath: f.Name(),
}
if got, _ := CalculateDependencies(opts); !reflect.DeepEqual(got, tt.want) {
diff := cmp.Diff(got, tt.want)
t.Errorf("CalculateDependencies() = %v, want %v, diff %v", got, tt.want, diff)
}
})
}
}
func Test_filesToSave(t *testing.T) {
tests := []struct {
name string
args []string
want []string
files []string
}{
{
name: "simple",
args: []string{"foo"},
files: []string{"foo"},
want: []string{"foo"},
},
{
name: "glob",
args: []string{"foo*"},
files: []string{"foo", "foo2", "fooooo", "bar"},
want: []string{"foo", "foo2", "fooooo"},
},
{
name: "complex glob",
args: []string{"foo*", "bar?"},
files: []string{"foo", "foo2", "fooooo", "bar", "bar1", "bar2", "bar33"},
want: []string{"foo", "foo2", "fooooo", "bar1", "bar2"},
},
{
name: "dir",
args: []string{"foo"},
files: []string{"foo/bar", "foo/baz", "foo/bat/baz"},
want: []string{"foo"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "")
if err != nil {
t.Errorf("error creating tmpdir: %s", err)
}
defer os.RemoveAll(tmpDir)
for _, f := range tt.files {
p := filepath.Join(tmpDir, f)
dir := filepath.Dir(p)
if dir != "." {
if err := os.MkdirAll(dir, 0755); err != nil {
t.Errorf("error making dir: %s", err)
}
}
fp, err := os.Create(p)
if err != nil {
t.Errorf("error making file: %s", err)
}
fp.Close()
}
args := []string{}
for _, arg := range tt.args {
args = append(args, filepath.Join(tmpDir, arg))
}
got, err := filesToSave(args)
if err != nil {
t.Errorf("got err: %s", err)
}
want := []string{}
for _, w := range tt.want {
want = append(want, filepath.Join(tmpDir, w))
}
sort.Strings(want)
sort.Strings(got)
if !reflect.DeepEqual(got, want) {
t.Errorf("filesToSave() = %v, want %v", got, want)
}
})
}
}

0
pkg/executor/foo Normal file
View File

45
pkg/util/command_util.go
View File

@ -78,6 +78,22 @@ func ResolveEnvironmentReplacement(value string, envs []string, isFilepath bool)
return fp, nil
}
func ResolveEnvAndWildcards(sd instructions.SourcesAndDest, buildcontext string, envs []string) ([]string, string, error) {
// First, resolve any environment replacement
resolvedEnvs, err := ResolveEnvironmentReplacementList(sd, envs, true)
if err != nil {
return nil, "", err
}
dest := resolvedEnvs[len(resolvedEnvs)-1]
// Resolve wildcards and get a list of resolved sources
srcs, err := ResolveSources(resolvedEnvs[0:len(resolvedEnvs)-1], buildcontext)
if err != nil {
return nil, "", err
}
err = IsSrcsValid(sd, srcs, buildcontext)
return srcs, dest, err
}
// ContainsWildcards returns true if any entry in paths contains wildcards
func ContainsWildcards(paths []string) bool {
for _, path := range paths {
@ -90,23 +106,22 @@ func ContainsWildcards(paths []string) bool {
// ResolveSources resolves the given sources if the sources contains wildcards
// It returns a list of resolved sources
func ResolveSources(srcsAndDest instructions.SourcesAndDest, root string) ([]string, error) {
srcs := srcsAndDest[:len(srcsAndDest)-1]
func ResolveSources(srcs []string, root string) ([]string, error) {
// If sources contain wildcards, we first need to resolve them to actual paths
if ContainsWildcards(srcs) {
logrus.Debugf("Resolving srcs %v...", srcs)
files, err := RelativeFiles("", root)
if err != nil {
return nil, err
}
srcs, err = matchSources(srcs, files)
if err != nil {
return nil, err
}
logrus.Debugf("Resolved sources to %v", srcs)
if !ContainsWildcards(srcs) {
return srcs, nil
}
// Check to make sure the sources are valid
return srcs, IsSrcsValid(srcsAndDest, srcs, root)
logrus.Infof("Resolving srcs %v...", srcs)
files, err := RelativeFiles("", root)
if err != nil {
return nil, err
}
resolved, err := matchSources(srcs, files)
if err != nil {
return nil, err
}
logrus.Debugf("Resolved sources to %v", resolved)
return resolved, nil
}
// matchSources returns a list of sources that match wildcards

1
pkg/util/command_util_test.go
View File

@ -408,7 +408,6 @@ var testResolveSources = []struct {
"context/foo",
"context/b*",
testURL,
"dest/",
},
expectedList: []string{
"context/foo",

21
vendor/github.com/otiai10/copy/LICENSE generated vendored Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2018 otiai10
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

93
vendor/github.com/otiai10/copy/copy.go generated vendored Normal file
View File

@ -0,0 +1,93 @@
package copy
import (
"io"
"io/ioutil"
"os"
"path/filepath"
)
// Copy copies src to dest, doesn't matter if src is a directory or a file
func Copy(src, dest string) error {
info, err := os.Lstat(src)
if err != nil {
return err
}
return copy(src, dest, info)
}
// copy dispatches copy-funcs according to the mode.
// Because this "copy" could be called recursively,
// "info" MUST be given here, NOT nil.
func copy(src, dest string, info os.FileInfo) error {
if info.Mode()&os.ModeSymlink != 0 {
return lcopy(src, dest, info)
}
if info.IsDir() {
return dcopy(src, dest, info)
}
return fcopy(src, dest, info)
}
// fcopy is for just a file,
// with considering existence of parent directory
// and file permission.
func fcopy(src, dest string, info os.FileInfo) error {
if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil {
return err
}
f, err := os.Create(dest)
if err != nil {
return err
}
defer f.Close()
if err = os.Chmod(f.Name(), info.Mode()); err != nil {
return err
}
s, err := os.Open(src)
if err != nil {
return err
}
defer s.Close()
_, err = io.Copy(f, s)
return err
}
// dcopy is for a directory,
// with scanning contents inside the directory
// and pass everything to "copy" recursively.
func dcopy(srcdir, destdir string, info os.FileInfo) error {
if err := os.MkdirAll(destdir, info.Mode()); err != nil {
return err
}
contents, err := ioutil.ReadDir(srcdir)
if err != nil {
return err
}
for _, content := range contents {
cs, cd := filepath.Join(srcdir, content.Name()), filepath.Join(destdir, content.Name())
if err := copy(cs, cd, content); err != nil {
// If any error, exit immediately
return err
}
}
return nil
}
// lcopy is for a symlink,
// with just creating a new symlink by replicating src symlink.
func lcopy(src, dest string, info os.FileInfo) error {
src, err := os.Readlink(src)
if err != nil {
return err
}
return os.Symlink(src, dest)
}

1
vendor/github.com/otiai10/copy/testdata/case03/case01 generated vendored Symbolic link
View File

@ -0,0 +1 @@
./testdata/case01