From ed883b9015155e76a4b62e0c62decbfcd28d5952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordan=20Goasdou=C3=A9?= Date: Thu, 26 Mar 2020 01:17:51 +0100 Subject: [PATCH 1/6] feat: can now resolves args from stage --- pkg/dockerfile/dockerfile.go | 53 +++++++++++++++++--- pkg/dockerfile/dockerfile_test.go | 83 ++++++++++++++++++++++++++++--- 2 files changed, 123 insertions(+), 13 deletions(-) diff --git a/pkg/dockerfile/dockerfile.go b/pkg/dockerfile/dockerfile.go index f41875c56..7ce6591cc 100644 --- a/pkg/dockerfile/dockerfile.go +++ b/pkg/dockerfile/dockerfile.go @@ -62,18 +62,20 @@ func Stages(opts *config.KanikoOptions) ([]config.KanikoStage, error) { return nil, err } resolveStages(stages) + args := unifyArgs(metaArgs, opts.BuildArgs) + if err := resolveStagesArgs(stages, args); err != nil { + return nil, errors.Wrap(err, "resolving args") + } var kanikoStages []config.KanikoStage for index, stage := range stages { - resolvedBaseName, err := util.ResolveEnvironmentReplacement(stage.BaseName, opts.BuildArgs, false) - if err != nil { - return nil, errors.Wrap(err, "resolving base name") + if len(stage.Name) > 0 { + logrus.Infof("Resolved base name %s to %s", stage.BaseName, stage.Name) } - stage.Name = resolvedBaseName - logrus.Infof("Resolved base name %s to %s", stage.BaseName, stage.Name) + baseImageIndex := baseImageIndex(index, stages) kanikoStages = append(kanikoStages, config.KanikoStage{ Stage: stage, - BaseImageIndex: baseImageIndex(index, stages), - BaseImageStoredLocally: (baseImageIndex(index, stages) != -1), + BaseImageIndex: baseImageIndex, + BaseImageStoredLocally: (baseImageIndex != -1), SaveStage: saveStage(index, stages), Final: index == targetStage, MetaArgs: metaArgs, @@ -87,6 +89,29 @@ func Stages(opts *config.KanikoOptions) ([]config.KanikoStage, error) { return kanikoStages, nil } +// unifyArgs returns the unified args between metaArgs and --build-arg +// by default --build-arg overrides metaArgs except when --build-arg is empty +func unifyArgs(metaArgs []instructions.ArgCommand, buildArgs []string) []string { + argsMap := make(map[string]string) + for _, a := range metaArgs { + if a.Value != nil { + argsMap[a.Key] = *a.Value + } + } + splitter := "=" + for _, a := range buildArgs { + s := strings.Split(a, splitter) + if len(s) > 1 && s[1] != "" { + argsMap[s[0]] = s[1] + } + } + var args []string + for k, v := range argsMap { + args = append(args, fmt.Sprintf("%s=%s", k, v)) + } + return args +} + // baseImageIndex returns the index of the stage the current stage is built off // returns -1 if the current stage isn't built off a previous stage func baseImageIndex(currentStage int, stages []instructions.Stage) int { @@ -228,6 +253,20 @@ func resolveStages(stages []instructions.Stage) { } } +// resolveStagesArgs resolves all the args from list of stages +func resolveStagesArgs(stages []instructions.Stage, args []string) error { + for i, s := range stages { + resolvedBaseName, err := util.ResolveEnvironmentReplacement(s.BaseName, args, false) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("resolving base name %s", s.BaseName)) + } + if s.BaseName != resolvedBaseName { + stages[i].BaseName = resolvedBaseName + } + } + return nil +} + // ParseCommands parses an array of commands into an array of instructions.Command; used for onbuild func ParseCommands(cmdArray []string) ([]instructions.Command, error) { var cmds []instructions.Command diff --git a/pkg/dockerfile/dockerfile_test.go b/pkg/dockerfile/dockerfile_test.go index d903fdb52..e891b4a40 100644 --- a/pkg/dockerfile/dockerfile_test.go +++ b/pkg/dockerfile/dockerfile_test.go @@ -17,6 +17,7 @@ limitations under the License. package dockerfile import ( + "fmt" "io/ioutil" "os" "strconv" @@ -32,10 +33,10 @@ func Test_Stages_ArgValueWithQuotes(t *testing.T) { ARG IMAGE="ubuntu:16.04" FROM ${IMAGE} RUN echo hi > /hi - + FROM scratch AS second COPY --from=0 /hi /hi2 - + FROM scratch COPY --from=second /hi2 /hi3 ` @@ -193,10 +194,10 @@ func Test_resolveStages(t *testing.T) { dockerfile := ` FROM scratch RUN echo hi > /hi - + FROM scratch AS second COPY --from=0 /hi /hi2 - + FROM scratch AS tHiRd COPY --from=second /hi2 /hi3 COPY --from=1 /hi2 /hi3 @@ -230,10 +231,10 @@ func Test_targetStage(t *testing.T) { dockerfile := ` FROM scratch RUN echo hi > /hi - + FROM scratch AS second COPY --from=0 /hi /hi2 - + FROM scratch COPY --from=second /hi2 /hi3 ` @@ -364,3 +365,73 @@ func Test_baseImageIndex(t *testing.T) { }) } } + +func Test_ResolveStagesArgs(t *testing.T) { + dockerfile := ` + ARG IMAGE="ubuntu:16.04" + ARG LAST_STAGE_VARIANT + FROM ${IMAGE} as base + RUN echo hi > /hi + FROM base AS base-dev + RUN echo dev >> /hi + FROM base AS base-prod + RUN echo prod >> /hi + FROM base-${LAST_STAGE_VARIANT} + RUN cat /hi + ` + + buildArgLastVariants := []string{"dev", "prod"} + buildArgImages := []string{"alpine:3.11", ""} + var expectedImage string + + for _, buildArgLastVariant := range buildArgLastVariants { + for _, buildArgImage := range buildArgImages { + if buildArgImage != "" { + expectedImage = buildArgImage + } else { + expectedImage = "ubuntu:16.04" + } + buildArgs := []string{fmt.Sprintf("IMAGE=%s", buildArgImage), fmt.Sprintf("LAST_STAGE_VARIANT=%s", buildArgLastVariant)} + + stages, metaArgs, err := Parse([]byte(dockerfile)) + if err != nil { + t.Fatal(err) + } + stagesLen := len(stages) + resolveStages(stages) + + args := unifyArgs(metaArgs, buildArgs) + if err := resolveStagesArgs(stages, args); err != nil { + t.Fatalf("fail to resolves args %v: %v", buildArgs, err) + } + tests := []struct { + name string + actualSourceCode string + actualBaseName string + expectedSourceCode string + expectedBaseName string + }{ + { + name: "Test_BuildArg_From_First_Stage", + actualSourceCode: stages[0].SourceCode, + actualBaseName: stages[0].BaseName, + expectedSourceCode: "FROM ${IMAGE} as base", + expectedBaseName: expectedImage, + }, + { + name: "Test_BuildArg_From_Last_Stage", + actualSourceCode: stages[stagesLen-1].SourceCode, + actualBaseName: stages[stagesLen-1].BaseName, + expectedSourceCode: "FROM base-${LAST_STAGE_VARIANT}", + expectedBaseName: fmt.Sprintf("base-%s", buildArgLastVariant), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testutil.CheckDeepEqual(t, test.expectedSourceCode, test.actualSourceCode) + testutil.CheckDeepEqual(t, test.expectedBaseName, test.actualBaseName) + }) + } + } + } +} From 3aaec5015bc9c39e922333c29cbaab9a945f99dd Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Fri, 3 Apr 2020 15:34:59 -0300 Subject: [PATCH 2/6] feat: allow a subdir within a context Signed-off-by: Carlos Alexandro Becker --- cmd/executor/cmd/root.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index ff198060d..a044a305b 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -39,10 +39,11 @@ import ( ) var ( - opts = &config.KanikoOptions{} - force bool - logLevel string - logFormat string + opts = &config.KanikoOptions{} + ctxSubPath string + force bool + logLevel string + logFormat string ) func init() { @@ -131,6 +132,7 @@ var RootCmd = &cobra.Command{ func addKanikoOptionsFlags() { RootCmd.PersistentFlags().StringVarP(&opts.DockerfilePath, "dockerfile", "f", "Dockerfile", "Path to the dockerfile to be built.") RootCmd.PersistentFlags().StringVarP(&opts.SrcContext, "context", "c", "/workspace/", "Path to the dockerfile build context.") + RootCmd.PersistentFlags().StringVarP(&ctxSubPath, "context-sub-path", "", "", "Sub path within the given context.") RootCmd.PersistentFlags().StringVarP(&opts.Bucket, "bucket", "b", "", "Name of the GCS bucket from which to access build context as tarball.") RootCmd.PersistentFlags().VarP(&opts.Destinations, "destination", "d", "Registry the final image should be pushed to. Set it repeatedly for multiple destinations.") RootCmd.PersistentFlags().StringVarP(&opts.SnapshotMode, "snapshotMode", "", "full", "Change the file attributes inspected during snapshotting") @@ -259,6 +261,9 @@ func resolveSourceContext() error { if err != nil { return err } + if ctxSubPath != "" { + opts.SrcContext = filepath.Join(opts.SrcContext, ctxSubPath) + } logrus.Debugf("Build context located at %s", opts.SrcContext) return nil } From 92b9582ff9bb487942981a451836d31ae3db747b Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Fri, 3 Apr 2020 16:21:29 -0300 Subject: [PATCH 3/6] fix: better error if not exists Signed-off-by: Carlos Alexandro Becker --- cmd/executor/cmd/root.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index a044a305b..db27d75f3 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -263,6 +263,9 @@ func resolveSourceContext() error { } if ctxSubPath != "" { opts.SrcContext = filepath.Join(opts.SrcContext, ctxSubPath) + if _, err := os.Stat(opts.SrcContext); os.IsNotExist(err) { + return err + } } logrus.Debugf("Build context located at %s", opts.SrcContext) return nil From 0fc7b8a4f4b7f5452102dcb6aea9241638e798c8 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Fri, 3 Apr 2020 16:34:22 -0300 Subject: [PATCH 4/6] test: add it Signed-off-by: Carlos Alexandro Becker --- integration/integration_test.go | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/integration/integration_test.go b/integration/integration_test.go index 0ae62e4b4..53d4dd364 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -260,6 +260,50 @@ func TestGitBuildcontext(t *testing.T) { checkContainerDiffOutput(t, diff, expected) } +func TestGitBuildcontextSubPath(t *testing.T) { + repo := getGitRepo() + dockerfile := "Dockerfile_test_run_2" + + // Build with docker + dockerImage := GetDockerImage(config.imageRepo, "Dockerfile_test_git") + dockerCmd := exec.Command("docker", + append([]string{ + "build", + "-t", dockerImage, + "-f", dockerfile, + filepath.Join(repo, integrationPath, dockerfilesPath), + })...) + out, err := RunCommandWithoutTest(dockerCmd) + if err != nil { + t.Errorf("Failed to build image %s with docker command %q: %s %s", dockerImage, dockerCmd.Args, err, string(out)) + } + + // Build with kaniko + kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_git") + dockerRunFlags := []string{"run", "--net=host"} + dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount) + dockerRunFlags = append( + dockerRunFlags, + ExecutorImage, + "-f", dockerfile, + "-d", kanikoImage, + "-c", fmt.Sprintf("git://%s", repo), + "--context-sub-path", filepath.Join(integrationPath, dockerfilesPath), + ) + + kanikoCmd := exec.Command("docker", dockerRunFlags...) + + out, err = RunCommandWithoutTest(kanikoCmd) + if err != nil { + t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out)) + } + + diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache") + + expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage) + checkContainerDiffOutput(t, diff, expected) +} + func TestBuildViaRegistryMirror(t *testing.T) { repo := getGitRepo() dockerfile := "integration/dockerfiles/Dockerfile_registry_mirror" From 8b50908e48c103b848fd7c78113840ba206b5b15 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Fri, 3 Apr 2020 17:15:40 -0300 Subject: [PATCH 5/6] test: fixed Signed-off-by: Carlos Alexandro Becker --- integration/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/integration_test.go b/integration/integration_test.go index 53d4dd364..5cb854782 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -271,7 +271,7 @@ func TestGitBuildcontextSubPath(t *testing.T) { "build", "-t", dockerImage, "-f", dockerfile, - filepath.Join(repo, integrationPath, dockerfilesPath), + repo + ":" + filepath.Join(integrationPath, dockerfilesPath), })...) out, err := RunCommandWithoutTest(dockerCmd) if err != nil { From 4bd1444be03302bd1d5ab0161dec6e9158606cb0 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Wed, 8 Apr 2020 14:08:57 -0300 Subject: [PATCH 6/6] docs: --sub-path Signed-off-by: Carlos Alexandro Becker --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 2e0de1929..726dc2708 100644 --- a/README.md +++ b/README.md @@ -432,6 +432,12 @@ If `--destination=gcr.io/kaniko-project/test`, then cached layers will be stored _This flag must be used in conjunction with the `--cache=true` flag._ +#### --context-sub-path + +Set a sub path within the given `--context`. + +Its particularly useful when your context is, for example, a git repository, +and you want to build one of its subfolders instead of the root folder. #### --digest-file