Updated PR Branch with master

2026-10-09 08:16:12 +02:00 · 2019-04-09 02:18:28 +05:30
parent 8b459b57aa
commit fa2a2c803b
75 changed files with 2221 additions and 647 deletions
@@ -23,6 +23,8 @@ import (
 	"strconv"
 	"time"

+	"github.com/otiai10/copy"
+
 	"github.com/google/go-containerregistry/pkg/v1/partial"

 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
@@ -60,10 +62,11 @@ type stageBuilder struct {
 	opts            *config.KanikoOptions
 	cmds            []commands.DockerCommand
 	args            *dockerfile.BuildArgs
+	crossStageDeps  map[int][]string
 }

 // newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage
-func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*stageBuilder, error) {
+func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string) (*stageBuilder, error) {
 	sourceImage, err := util.RetrieveSourceImage(stage, opts)
 	if err != nil {
 		return nil, err
@@ -96,6 +99,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*sta
 		snapshotter:     snapshotter,
 		baseImageDigest: digest.String(),
 		opts:            opts,
+		crossStageDeps:  crossStageDeps,
 	}

 	for _, cmd := range s.stage.Commands {
@@ -207,6 +211,10 @@ func (s *stageBuilder) build() error {
 			break
 		}
 	}
+	if len(s.crossStageDeps[s.stage.Index]) > 0 {
+		shouldUnpack = true
+	}
+
 	if shouldUnpack {
 		t := timing.Start("FS Unpacking")
 		if _, err := util.GetFSFromImage(constants.RootDir, s.image); err != nil {
@@ -353,6 +361,68 @@ func (s *stageBuilder) saveSnapshotToImage(createdBy string, tarPath string) err

 }

+func CalculateDependencies(opts *config.KanikoOptions) (map[int][]string, error) {
+	stages, err := dockerfile.Stages(opts)
+	if err != nil {
+		return nil, err
+	}
+	images := []v1.Image{}
+	depGraph := map[int][]string{}
+	for _, s := range stages {
+		ba := dockerfile.NewBuildArgs(opts.BuildArgs)
+		ba.AddMetaArgs(s.MetaArgs)
+		var image v1.Image
+		var err error
+		if s.BaseImageStoredLocally {
+			image = images[s.BaseImageIndex]
+		} else if s.Name == constants.NoBaseImage {
+			image = empty.Image
+		} else {
+			image, err = util.RetrieveSourceImage(s, opts)
+			if err != nil {
+				return nil, err
+			}
+		}
+		cfg, err := initializeConfig(image)
+		if err != nil {
+			return nil, err
+		}
+		for _, c := range s.Commands {
+			switch cmd := c.(type) {
+			case *instructions.CopyCommand:
+				if cmd.From != "" {
+					i, err := strconv.Atoi(cmd.From)
+					if err != nil {
+						continue
+					}
+					resolved, err := util.ResolveEnvironmentReplacementList(cmd.SourcesAndDest, ba.ReplacementEnvs(cfg.Config.Env), true)
+					if err != nil {
+						return nil, err
+					}
+
+					depGraph[i] = append(depGraph[i], resolved[0:len(resolved)-1]...)
+				}
+			case *instructions.EnvCommand:
+				if err := util.UpdateConfigEnv(cmd.Env, &cfg.Config, ba.ReplacementEnvs(cfg.Config.Env)); err != nil {
+					return nil, err
+				}
+				image, err = mutate.Config(image, cfg.Config)
+				if err != nil {
+					return nil, err
+				}
+			case *instructions.ArgCommand:
+				k, v, err := commands.ParseArg(cmd.Key, cmd.Value, cfg.Config.Env, ba)
+				if err != nil {
+					return nil, err
+				}
+				ba.AddArg(k, v)
+			}
+		}
+		images = append(images, image)
+	}
+	return depGraph, nil
+}
+
 // DoBuild executes building the Dockerfile
 func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 	t := timing.Start("Total Build Time")
@@ -369,8 +439,14 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 		return nil, err
 	}

+	crossStageDependencies, err := CalculateDependencies(opts)
+	if err != nil {
+		return nil, err
+	}
+	logrus.Infof("Built cross stage deps: %v", crossStageDependencies)
+
 	for index, stage := range stages {
-		sb, err := newStageBuilder(opts, stage)
+		sb, err := newStageBuilder(opts, stage, crossStageDependencies)
 		if err != nil {
 			return nil, err
 		}
@@ -405,10 +481,21 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 			if err := saveStageAsTarball(strconv.Itoa(index), sourceImage); err != nil {
 				return nil, err
 			}
-			if err := extractImageToDependecyDir(strconv.Itoa(index), sourceImage); err != nil {
-				return nil, err
-			}
 		}
+
+		filesToSave, err := filesToSave(crossStageDependencies[index])
+		if err != nil {
+			return nil, err
+		}
+		dstDir := filepath.Join(constants.KanikoDir, strconv.Itoa(index))
+		if err := os.MkdirAll(dstDir, 0644); err != nil {
+			return nil, err
+		}
+		for _, p := range filesToSave {
+			logrus.Infof("Saving file %s for later use.", p)
+			copy.Copy(p, filepath.Join(dstDir, p))
+		}
+
 		// Delete the filesystem
 		if err := util.DeleteFilesystem(); err != nil {
 			return nil, err
@@ -418,6 +505,18 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
 	return nil, err
 }

+func filesToSave(deps []string) ([]string, error) {
+	allFiles := []string{}
+	for _, src := range deps {
+		srcs, err := filepath.Glob(src)
+		if err != nil {
+			return nil, err
+		}
+		allFiles = append(allFiles, srcs...)
+	}
+	return allFiles, nil
+}
+
 func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) error {
 	t := timing.Start("Fetching Extra Stages")
 	defer timing.DefaultRun.Stop(t)
@@ -17,14 +17,19 @@ limitations under the License.
 package executor

 import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
 	"testing"

-	"github.com/moby/buildkit/frontend/dockerfile/instructions"
-
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
 	"github.com/GoogleContainerTools/kaniko/testutil"
-	"github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-cmp/cmp"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )

 func Test_reviewConfig(t *testing.T) {
@@ -180,3 +185,223 @@ func Test_stageBuilder_shouldTakeSnapshot(t *testing.T) {
 		})
 	}
 }
+
+func TestCalculateDependencies(t *testing.T) {
+	type args struct {
+		dockerfile string
+	}
+	tests := []struct {
+		name string
+		args args
+		want map[int][]string
+	}{
+		{
+			name: "no deps",
+			args: args{
+				dockerfile: `
+FROM debian as stage1
+RUN foo
+FROM stage1
+RUN bar
+`,
+			},
+			want: map[int][]string{},
+		},
+		{
+			name: "args",
+			args: args{
+				dockerfile: `
+ARG myFile=foo
+FROM debian as stage1
+RUN foo
+FROM stage1
+ARG myFile
+COPY --from=stage1 /tmp/$myFile.txt .
+RUN bar
+`,
+			},
+			want: map[int][]string{
+				0: {"/tmp/foo.txt"},
+			},
+		},
+		{
+			name: "simple deps",
+			args: args{
+				dockerfile: `
+FROM debian as stage1
+FROM alpine
+COPY --from=stage1 /foo /bar
+`,
+			},
+			want: map[int][]string{
+				0: {"/foo"},
+			},
+		},
+		{
+			name: "two sets deps",
+			args: args{
+				dockerfile: `
+FROM debian as stage1
+FROM ubuntu as stage2
+RUN foo
+COPY --from=stage1 /foo /bar
+FROM alpine
+COPY --from=stage2 /bar /bat
+`,
+			},
+			want: map[int][]string{
+				0: {"/foo"},
+				1: {"/bar"},
+			},
+		},
+		{
+			name: "double deps",
+			args: args{
+				dockerfile: `
+FROM debian as stage1
+FROM ubuntu as stage2
+RUN foo
+COPY --from=stage1 /foo /bar
+FROM alpine
+COPY --from=stage1 /baz /bat
+`,
+			},
+			want: map[int][]string{
+				0: {"/foo", "/baz"},
+			},
+		},
+		{
+			name: "envs in deps",
+			args: args{
+				dockerfile: `
+FROM debian as stage1
+FROM ubuntu as stage2
+RUN foo
+ENV key1 val1
+ENV key2 val2
+COPY --from=stage1 /foo/$key1 /foo/$key2 /bar
+FROM alpine
+COPY --from=stage2 /bar /bat
+`,
+			},
+			want: map[int][]string{
+				0: {"/foo/val1", "/foo/val2"},
+				1: {"/bar"},
+			},
+		},
+		{
+			name: "envs from base image in deps",
+			args: args{
+				dockerfile: `
+FROM debian as stage1
+ENV key1 baseval1
+FROM stage1 as stage2
+RUN foo
+ENV key2 val2
+COPY --from=stage1 /foo/$key1 /foo/$key2 /bar
+FROM alpine
+COPY --from=stage2 /bar /bat
+`,
+			},
+			want: map[int][]string{
+				0: {"/foo/baseval1", "/foo/val2"},
+				1: {"/bar"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f, _ := ioutil.TempFile("", "")
+			ioutil.WriteFile(f.Name(), []byte(tt.args.dockerfile), 0755)
+			opts := &config.KanikoOptions{
+				DockerfilePath: f.Name(),
+			}
+
+			got, err := CalculateDependencies(opts)
+			if err != nil {
+				t.Errorf("got error: %s,", err)
+			}
+
+			if !reflect.DeepEqual(got, tt.want) {
+				diff := cmp.Diff(got, tt.want)
+				t.Errorf("CalculateDependencies() = %v, want %v, diff %v", got, tt.want, diff)
+			}
+		})
+	}
+}
+
+func Test_filesToSave(t *testing.T) {
+	tests := []struct {
+		name  string
+		args  []string
+		want  []string
+		files []string
+	}{
+		{
+			name:  "simple",
+			args:  []string{"foo"},
+			files: []string{"foo"},
+			want:  []string{"foo"},
+		},
+		{
+			name:  "glob",
+			args:  []string{"foo*"},
+			files: []string{"foo", "foo2", "fooooo", "bar"},
+			want:  []string{"foo", "foo2", "fooooo"},
+		},
+		{
+			name:  "complex glob",
+			args:  []string{"foo*", "bar?"},
+			files: []string{"foo", "foo2", "fooooo", "bar", "bar1", "bar2", "bar33"},
+			want:  []string{"foo", "foo2", "fooooo", "bar1", "bar2"},
+		},
+		{
+			name:  "dir",
+			args:  []string{"foo"},
+			files: []string{"foo/bar", "foo/baz", "foo/bat/baz"},
+			want:  []string{"foo"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tmpDir, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Errorf("error creating tmpdir: %s", err)
+			}
+			defer os.RemoveAll(tmpDir)
+
+			for _, f := range tt.files {
+				p := filepath.Join(tmpDir, f)
+				dir := filepath.Dir(p)
+				if dir != "." {
+					if err := os.MkdirAll(dir, 0755); err != nil {
+						t.Errorf("error making dir: %s", err)
+					}
+				}
+				fp, err := os.Create(p)
+				if err != nil {
+					t.Errorf("error making file: %s", err)
+				}
+				fp.Close()
+			}
+
+			args := []string{}
+			for _, arg := range tt.args {
+				args = append(args, filepath.Join(tmpDir, arg))
+			}
+			got, err := filesToSave(args)
+			if err != nil {
+				t.Errorf("got err: %s", err)
+			}
+			want := []string{}
+			for _, w := range tt.want {
+				want = append(want, filepath.Join(tmpDir, w))
+			}
+			sort.Strings(want)
+			sort.Strings(got)
+			if !reflect.DeepEqual(got, want) {
+				t.Errorf("filesToSave() = %v, want %v", got, want)
+			}
+		})
+	}
+}
@@ -47,6 +47,30 @@ func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) {
 	return w.t.RoundTrip(r)
 }

+// CheckPushPermissionos checks that the configured credentials can be used to
+// push to every specified destination.
+func CheckPushPermissions(opts *config.KanikoOptions) error {
+	if opts.NoPush {
+		return nil
+	}
+
+	checked := map[string]bool{}
+	for _, destination := range opts.Destinations {
+		destRef, err := name.NewTag(destination, name.WeakValidation)
+		if err != nil {
+			return errors.Wrap(err, "getting tag for destination")
+		}
+		if checked[destRef.Context().RepositoryStr()] {
+			continue
+		}
+		if err := remote.CheckPushPermission(destRef, creds.GetKeychain(), http.DefaultTransport); err != nil {
+			return errors.Wrapf(err, "checking push permission for %q", destRef)
+		}
+		checked[destRef.Context().RepositoryStr()] = true
+	}
+	return nil
+}
+
 // DoPush is responsible for pushing image to the destinations specified in opts
 func DoPush(image v1.Image, opts *config.KanikoOptions) error {
 	t := timing.Start("Total Push Time")