From 57c2df4d97d8743dae8263c5304635903ca4d3f9 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Mon, 15 Jun 2020 17:24:04 -0300 Subject: [PATCH 1/4] feat: new git flags Signed-off-by: Carlos Alexandro Becker --- cmd/executor/cmd/root.go | 9 +++- pkg/buildcontext/buildcontext.go | 10 ++++- pkg/buildcontext/git.go | 71 ++++++++++++++++++++++++++++++-- pkg/config/options.go | 7 ++++ 4 files changed, 91 insertions(+), 6 deletions(-) diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index eed04861f..5156322d6 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -175,6 +175,9 @@ func addKanikoOptionsFlags() { RootCmd.PersistentFlags().VarP(&opts.Labels, "label", "", "Set metadata for an image. Set it repeatedly for multiple labels.") RootCmd.PersistentFlags().BoolVarP(&opts.SkipUnusedStages, "skip-unused-stages", "", false, "Build only used stages if defined to true. Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile") RootCmd.PersistentFlags().BoolVarP(&opts.RunV2, "use-new-run", "", false, "Experimental run command to detect file system changes. This new run command does no rely on snapshotting to detect changes.") + RootCmd.PersistentFlags().StringVarP(&opts.Git.Branch, "git-branch", "", "", "Branch to clone if build context is a git repository") + RootCmd.PersistentFlags().BoolVarP(&opts.Git.SingleBranch, "git-single-branch", "", false, "Whether to clone a single branch") + RootCmd.PersistentFlags().BoolVarP(&opts.Git.RecurseSubmodules, "git-recurse-submodules", "", false, "Whether to clone recursing submodules") } // addHiddenFlags marks certain flags as hidden from the executor help text @@ -266,7 +269,11 @@ func resolveSourceContext() error { opts.SrcContext = opts.Bucket } } - contextExecutor, err := buildcontext.GetBuildContext(opts.SrcContext) + contextExecutor, err := buildcontext.GetBuildContext(opts.SrcContext, buildcontext.BuildOptions{ + GitBranch: opts.Git.Branch, + GitSingleBranch: opts.Git.SingleBranch, + GitRecurseSubmodules: opts.Git.RecurseSubmodules, + }) if err != nil { return err } diff --git a/pkg/buildcontext/buildcontext.go b/pkg/buildcontext/buildcontext.go index 5a81c5033..08d8a7c98 100644 --- a/pkg/buildcontext/buildcontext.go +++ b/pkg/buildcontext/buildcontext.go @@ -28,6 +28,12 @@ const ( TarBuildContextPrefix = "tar://" ) +type BuildOptions struct { + GitBranch string + GitSingleBranch bool + GitRecurseSubmodules bool +} + // BuildContext unifies calls to download and unpack the build context. type BuildContext interface { // Unpacks a build context and returns the directory where it resides @@ -36,7 +42,7 @@ type BuildContext interface { // GetBuildContext parses srcContext for the prefix and returns related buildcontext // parser -func GetBuildContext(srcContext string) (BuildContext, error) { +func GetBuildContext(srcContext string, opts BuildOptions) (BuildContext, error) { split := strings.SplitAfter(srcContext, "://") if len(split) > 1 { prefix := split[0] @@ -50,7 +56,7 @@ func GetBuildContext(srcContext string) (BuildContext, error) { case constants.LocalDirBuildContextPrefix: return &Dir{context: context}, nil case constants.GitBuildContextPrefix: - return &Git{context: context}, nil + return &Git{context: context, opts: opts}, nil case constants.HTTPSBuildContextPrefix: if util.ValidAzureBlobStorageHost(srcContext) { return &AzureBlob{context: srcContext}, nil diff --git a/pkg/buildcontext/git.go b/pkg/buildcontext/git.go index 3ab829356..1f37ae19e 100644 --- a/pkg/buildcontext/git.go +++ b/pkg/buildcontext/git.go @@ -17,14 +17,20 @@ limitations under the License. package buildcontext import ( + "fmt" "os" "strings" "github.com/GoogleContainerTools/kaniko/pkg/constants" + "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/storage/filesystem" + "github.com/sirupsen/logrus" ) const ( @@ -43,24 +49,83 @@ var ( // Git unifies calls to download and unpack the build context. type Git struct { context string + opts BuildOptions } // UnpackTarFromBuildContext will provide the directory where Git Repository is Cloned func (g *Git) UnpackTarFromBuildContext() (string, error) { directory := constants.BuildContextDir parts := strings.Split(g.context, "#") + url := getGitPullMethod() + "://" + parts[0] options := git.CloneOptions{ - URL: getGitPullMethod() + "://" + parts[0], - Auth: getGitAuth(), - Progress: os.Stdout, + URL: url, + Auth: getGitAuth(), + Progress: os.Stdout, + SingleBranch: g.opts.GitSingleBranch, + RecurseSubmodules: getRecurseSubmodules(g.opts.GitRecurseSubmodules), } if len(parts) > 1 { options.ReferenceName = plumbing.ReferenceName(parts[1]) } + + if branch := g.opts.GitBranch; branch != "" { + ref, err := getGitReferenceName(directory, url, branch) + if err != nil { + return directory, err + } + options.ReferenceName = ref + } + + logrus.Debugf("Getting source from reference %s", options.ReferenceName) _, err := git.PlainClone(directory, false, &options) return directory, err } +func getGitReferenceName(directory string, url string, branch string) (plumbing.ReferenceName, error) { + var remote = git.NewRemote( + filesystem.NewStorage( + osfs.New(directory), + cache.NewObjectLRUDefault(), + ), + &config.RemoteConfig{ + URLs: []string{url}, + }, + ) + + refs, err := remote.List(&git.ListOptions{ + Auth: getGitAuth(), + }) + if err != nil { + return plumbing.HEAD, err + } + + if ref := plumbing.NewBranchReferenceName(branch); gitRefExists(ref, refs) { + return ref, nil + } + + if ref := plumbing.NewTagReferenceName(branch); gitRefExists(ref, refs) { + return ref, nil + } + + return plumbing.HEAD, fmt.Errorf("invalid branch: %s", branch) +} + +func gitRefExists(ref plumbing.ReferenceName, refs []*plumbing.Reference) bool { + for _, ref2 := range refs { + if ref.String() == ref2.Name().String() { + return true + } + } + return false +} + +func getRecurseSubmodules(v bool) git.SubmoduleRescursivity { + if v { + return git.DefaultSubmoduleRecursionDepth + } + return git.NoRecurseSubmodules +} + func getGitAuth() transport.AuthMethod { username := os.Getenv(gitAuthUsernameEnvKey) password := os.Getenv(gitAuthPasswordEnvKey) diff --git a/pkg/config/options.go b/pkg/config/options.go index 74db79b6f..5ba57914d 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -58,6 +58,13 @@ type KanikoOptions struct { IgnoreVarRun bool SkipUnusedStages bool RunV2 bool + Git KanikoGitOptions +} + +type KanikoGitOptions struct { + Branch string + SingleBranch bool + RecurseSubmodules bool } // WarmerOptions are options that are set by command line arguments to the cache warmer. From e483bee97cc1e39c2e7671843c20d14dd8b05bdf Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Wed, 24 Jun 2020 15:31:32 -0300 Subject: [PATCH 2/4] feat: git options Signed-off-by: Carlos Alexandro Becker --- cmd/executor/cmd/root.go | 4 +--- pkg/config/options.go | 38 ++++++++++++++++++++++++++++++++++++++ pkg/config/options_test.go | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 pkg/config/options_test.go diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 5156322d6..695ddca32 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -175,9 +175,7 @@ func addKanikoOptionsFlags() { RootCmd.PersistentFlags().VarP(&opts.Labels, "label", "", "Set metadata for an image. Set it repeatedly for multiple labels.") RootCmd.PersistentFlags().BoolVarP(&opts.SkipUnusedStages, "skip-unused-stages", "", false, "Build only used stages if defined to true. Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile") RootCmd.PersistentFlags().BoolVarP(&opts.RunV2, "use-new-run", "", false, "Experimental run command to detect file system changes. This new run command does no rely on snapshotting to detect changes.") - RootCmd.PersistentFlags().StringVarP(&opts.Git.Branch, "git-branch", "", "", "Branch to clone if build context is a git repository") - RootCmd.PersistentFlags().BoolVarP(&opts.Git.SingleBranch, "git-single-branch", "", false, "Whether to clone a single branch") - RootCmd.PersistentFlags().BoolVarP(&opts.Git.RecurseSubmodules, "git-recurse-submodules", "", false, "Whether to clone recursing submodules") + RootCmd.PersistentFlags().Var(&opts.Git, "git", "Branch to clone if build context is a git repository") } // addHiddenFlags marks certain flags as hidden from the executor help text diff --git a/pkg/config/options.go b/pkg/config/options.go index 5ba57914d..669b84f58 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -17,6 +17,10 @@ limitations under the License. package config import ( + "errors" + "fmt" + "strconv" + "strings" "time" ) @@ -67,6 +71,40 @@ type KanikoGitOptions struct { RecurseSubmodules bool } +var ErrInvalidGitFlag = errors.New("invalid git flag, must be in the key=value format") + +func (k *KanikoGitOptions) Type() string { + return "gitoptions" +} + +func (k *KanikoGitOptions) String() string { + return fmt.Sprintf("branch=%s,single-branch=%t,recurse-submodules=%t", k.Branch, k.SingleBranch, k.RecurseSubmodules) +} + +func (k *KanikoGitOptions) Set(s string) error { + var parts = strings.SplitN(s, "=", 2) + if len(parts) != 2 { + return fmt.Errorf("%w: %s", ErrInvalidGitFlag, s) + } + switch parts[0] { + case "branch": + k.Branch = parts[1] + case "single-branch": + v, err := strconv.ParseBool(parts[1]) + if err != nil { + return err + } + k.SingleBranch = v + case "recurse-submodules": + v, err := strconv.ParseBool(parts[1]) + if err != nil { + return err + } + k.RecurseSubmodules = v + } + return nil +} + // WarmerOptions are options that are set by command line arguments to the cache warmer. type WarmerOptions struct { CacheOptions diff --git a/pkg/config/options_test.go b/pkg/config/options_test.go new file mode 100644 index 000000000..cc6ac6693 --- /dev/null +++ b/pkg/config/options_test.go @@ -0,0 +1,37 @@ +package config + +import ( + "testing" + + "github.com/GoogleContainerTools/kaniko/testutil" +) + +func TestKanikoGitOptions(t *testing.T) { + t.Run("invalid pair", func(t *testing.T) { + var g = &KanikoGitOptions{} + testutil.CheckError(t, true, g.Set("branch")) + }) + + t.Run("sets values", func(t *testing.T) { + var g = &KanikoGitOptions{} + testutil.CheckNoError(t, g.Set("branch=foo")) + testutil.CheckNoError(t, g.Set("recurse-submodules=true")) + testutil.CheckNoError(t, g.Set("single-branch=true")) + testutil.CheckDeepEqual(t, KanikoGitOptions{ + Branch: "foo", + SingleBranch: true, + RecurseSubmodules: true, + }, *g) + }) + + t.Run("sets bools other than true", func(t *testing.T) { + var g = KanikoGitOptions{} + testutil.CheckError(t, true, g.Set("recurse-submodules=")) + testutil.CheckError(t, true, g.Set("single-branch=zaza")) + testutil.CheckNoError(t, g.Set("recurse-submodules=false")) + testutil.CheckDeepEqual(t, KanikoGitOptions{ + SingleBranch: false, + RecurseSubmodules: false, + }, g) + }) +} From 94b2b6b39111c87793d34d8a087cf49b19674f5e Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Thu, 13 Aug 2020 14:03:38 -0700 Subject: [PATCH 3/4] gofmt --- pkg/buildcontext/git.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/buildcontext/git.go b/pkg/buildcontext/git.go index 452e06961..0f20eb0c4 100644 --- a/pkg/buildcontext/git.go +++ b/pkg/buildcontext/git.go @@ -21,7 +21,6 @@ import ( "os" "strings" - "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" @@ -31,6 +30,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage/filesystem" "github.com/sirupsen/logrus" + + "github.com/GoogleContainerTools/kaniko/pkg/constants" ) const ( @@ -69,7 +70,6 @@ func (g *Git) UnpackTarFromBuildContext() (string, error) { options.ReferenceName = plumbing.ReferenceName(parts[1]) } - if branch := g.opts.GitBranch; branch != "" { ref, err := getGitReferenceName(directory, url, branch) if err != nil { @@ -79,7 +79,7 @@ func (g *Git) UnpackTarFromBuildContext() (string, error) { } logrus.Debugf("Getting source from reference %s", options.ReferenceName) - _, err := git.PlainClone(directory, false, &options) + r, err := git.PlainClone(directory, false, &options) if err == nil && len(parts) > 2 { // ... retrieving the commit being pointed by HEAD @@ -101,7 +101,6 @@ func (g *Git) UnpackTarFromBuildContext() (string, error) { return directory, err } } - return directory, err } From df7f4cc9dc1bf256010db1313cac93f09f6c176f Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Thu, 13 Aug 2020 14:44:07 -0700 Subject: [PATCH 4/4] add boilerplate --- pkg/config/options_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/config/options_test.go b/pkg/config/options_test.go index cc6ac6693..f268c9303 100644 --- a/pkg/config/options_test.go +++ b/pkg/config/options_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2020 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package config import (