From 6dcde20d7a4f7fc610425ca4595c95b32b9e0eaf Mon Sep 17 00:00:00 2001 From: xiaomudk Date: Thu, 3 Nov 2022 14:51:30 +0800 Subject: [PATCH] Add subcommand init for checks and installs helmfile deps (#389) * Add subcommand init for checks and installs helmfile deps Signed-off-by: xiaomudk --- .github/workflows/ci.yaml | 22 +++ cmd/init.go | 35 ++++ cmd/root.go | 1 + docs/index.md | 12 +- pkg/app/app.go | 8 + pkg/app/config.go | 4 + pkg/app/init.go | 238 +++++++++++++++++++++++++++ pkg/app/init_test.go | 89 ++++++++++ pkg/config/init.go | 30 ++++ pkg/helmexec/exec.go | 18 +- test/e2e/helmfile-init/init_linux.sh | 49 ++++++ 11 files changed, 501 insertions(+), 5 deletions(-) create mode 100644 cmd/init.go create mode 100644 pkg/app/init.go create mode 100644 pkg/app/init_test.go create mode 100644 pkg/config/init.go create mode 100644 test/e2e/helmfile-init/init_linux.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b4fbab09..a043bffc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -132,3 +132,25 @@ jobs: TERM: xterm EXTRA_HELMFILE_FLAGS: ${{ matrix.extra-helmfile-flags }} run: make integration + e2e_tests: + needs: tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Install package + run: | + sudo apt-get -y install expect + - name: Download built binaries + uses: actions/download-artifact@v2 + with: + name: built-binaries-${{ github.run_id }} + - name: Extract tar to get built binaries + run: tar -xvf built-binaries.tar + - name: Display built binaries + run: ls -l helmfile diff-yamls yamldiff + - name: Run helmfile init + env: + TERM: xterm + run: bash test/e2e/helmfile-init/init_linux.sh diff --git a/cmd/init.go b/cmd/init.go new file mode 100644 index 00000000..3d5124bf --- /dev/null +++ b/cmd/init.go @@ -0,0 +1,35 @@ +package cmd + +import ( + "github.com/spf13/cobra" + + "github.com/helmfile/helmfile/pkg/app" + "github.com/helmfile/helmfile/pkg/config" +) + +// NewInitCmd helmfile checks and installs deps +func NewInitCmd(globalCfg *config.GlobalImpl) *cobra.Command { + options := config.NewInitOptions() + cmd := &cobra.Command{ + Use: "init", + Short: "Initialize the helmfile, includes version checking and installation of helm and plug-ins", + RunE: func(cmd *cobra.Command, args []string) error { + initImpl := config.NewInitImpl(globalCfg, options) + err := config.NewCLIConfigImpl(initImpl.GlobalImpl) + if err != nil { + return err + } + + if err := initImpl.ValidateConfig(); err != nil { + return err + } + + a := app.New(initImpl) + return toCLIError(initImpl.GlobalImpl, a.Init(initImpl)) + }, + } + f := cmd.Flags() + f.BoolVar(&options.Force, "force", false, "Do not prompt, install dependencies required by helmfile") + + return cmd +} diff --git a/cmd/root.go b/cmd/root.go index ee0962a6..0343d395 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -81,6 +81,7 @@ func NewRootCmd(globalConfig *config.GlobalOptions) (*cobra.Command, error) { } cmd.AddCommand( + NewInitCmd(globalImpl), NewApplyCmd(globalImpl), NewBuildCmd(globalImpl), NewCacheCmd(globalImpl), diff --git a/docs/index.md b/docs/index.md index 9eada075..caacff2c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -505,6 +505,7 @@ Available Commands: diff Diff releases defined in state file fetch Fetch charts from state file help Help about any command + init Initialize the helmfile, includes version checking and installation of helm and plug-ins lint Lint charts from state file (helm lint) list List releases defined in state file repos Repos releases defined in state file @@ -532,7 +533,7 @@ Flags: -n, --namespace string Set namespace. Uses the namespace set in the context by default, and is available in templates as {{ .Namespace }} --no-color Output without color -q, --quiet Silence output. Equivalent to log-level warn - -l, --selector stringArray Only run using the releases that match labels. Labels can take the form of foo=bar or foo!=bar. + -l, --selector stringArray Only run using the releases that match labels. Labels can take the form of foo=bar or foo!=bar. A release must match all labels in a group in order to be used. Multiple groups can be specified at once. "--selector tier=frontend,tier!=proxy --selector tier=backend" will match all frontend, non-proxy releases AND all backend releases. The name of a release can be used as a label: "--selector name=myrelease" @@ -543,6 +544,11 @@ Flags: Use "helmfile [command] --help" for more information about a command. ``` +### init + +The `helmfile init` sub-command checks the dependencies required for helmfile operation, such as `helm`, `helm diff plugin`, `helm secrets plugin`, `helm helm-git plugin`, `helm s3 plugin`. When it does not exist or the version is too low, it can be installed automatically. + + ### sync The `helmfile sync` sub-command sync your cluster state as described in your `helmfile`. The default helmfile is `helmfile.yaml`, but any YAML file can be passed by specifying a `--file path/to/your/yaml/file` flag. @@ -1172,9 +1178,9 @@ mysetting: | The possibility is endless. Try importing values from your golang app, bash script, jsonnet, or anything! -Then `envExec` same as `exec`, but it can receive a dict as the envs. +Then `envExec` same as `exec`, but it can receive a dict as the envs. -A usual usage of `envExec` would look like this: +A usual usage of `envExec` would look like this: ```yaml mysetting: | diff --git a/pkg/app/app.go b/pkg/app/app.go index 040b11f3..50e74476 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -93,6 +93,14 @@ func Init(app *App) *App { return app } +func (a *App) Init(c InitConfigProvider) error { + runner := &helmexec.ShellRunner{ + Logger: a.Logger, + } + helmfileInit := NewHelmfileInit(a.OverrideHelmBinary, c, a.Logger, runner) + return helmfileInit.Initialize() +} + func (a *App) Deps(c DepsConfigProvider) error { return a.ForEachState(func(run *Run) (_ bool, errs []error) { prepErr := run.withPreparedCharts("deps", state.ChartPrepareOptions{ diff --git a/pkg/app/config.go b/pkg/app/config.go index 583c42ad..cfac09d1 100644 --- a/pkg/app/config.go +++ b/pkg/app/config.go @@ -247,6 +247,10 @@ type ListConfigProvider interface { type CacheConfigProvider interface{} +type InitConfigProvider interface { + Force() bool +} + // when enable reuse-values, reuse the last release's values and merge in any overrides values. type valuesControlMode interface { ReuseValues() bool diff --git a/pkg/app/init.go b/pkg/app/init.go new file mode 100644 index 00000000..e5076a73 --- /dev/null +++ b/pkg/app/init.go @@ -0,0 +1,238 @@ +package app + +import ( + "fmt" + "io" + "net/http" + "os" + "os/exec" + "runtime" + "strings" + + "github.com/Masterminds/semver/v3" + "go.uber.org/zap" + "helm.sh/helm/v3/pkg/cli" + + "github.com/helmfile/helmfile/pkg/helmexec" +) + +const ( + HelmRequiredVersion = "v2.10.0" + HelmRecommendedVersion = "v3.10.1" + HelmDiffRecommendedVersion = "v3.4.0" + HelmSecretsRecommendedVersion = "v4.1.1" + HelmGitRecommendedVersion = "v0.12.0" + HelmS3RecommendedVersion = "v0.14.0" + HelmInstallCommand = "https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3" +) + +var ( + manuallyInstallCode = 1 + windowPackageManagers = map[string]string{ + "scoop": fmt.Sprintf("scoop install helm@%s", strings.TrimLeft(HelmRecommendedVersion, "v")), + "choco": fmt.Sprintf("choco install kubernetes-helm --version %s", strings.TrimLeft(HelmRecommendedVersion, "v")), + } + helmPlugins = []helmRecommendedPlugin{ + { + name: "diff", + version: HelmDiffRecommendedVersion, + repo: "https://github.com/databus23/helm-diff", + }, + { + name: "secrets", + version: HelmSecretsRecommendedVersion, + repo: "https://github.com/jkroepke/helm-secrets", + }, + { + name: "s3", + version: HelmS3RecommendedVersion, + repo: "https://github.com/hypnoglow/helm-s3.git", + }, + { + name: "helm-git", + version: HelmGitRecommendedVersion, + repo: "https://github.com/aslafy-z/helm-git.git", + }, + } +) + +type helmRecommendedPlugin struct { + name string + version string + repo string +} + +type HelmfileInit struct { + helmBinary string + configProvider InitConfigProvider + logger *zap.SugaredLogger + runner helmexec.Runner +} + +func downloadfile(filepath string, url string) error { + file, err := os.Create(filepath) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + resp, err := http.Get(url) + if err != nil { + return err + } + if resp.StatusCode/100 != 2 { + return fmt.Errorf("download %s error, code: %d", url, resp.StatusCode) + } + defer func() { _ = resp.Body.Close() }() + _, err = io.Copy(file, resp.Body) + if err != nil { + return err + } + return nil +} + +func NewHelmfileInit(helmBinary string, c InitConfigProvider, logger *zap.SugaredLogger, runner helmexec.Runner) *HelmfileInit { + return &HelmfileInit{helmBinary: helmBinary, configProvider: c, logger: logger, runner: runner} +} +func (h *HelmfileInit) UpdateHelm() error { + return h.InstallHelm() +} + +func (h *HelmfileInit) installHelmOnWindows() error { + for name, command := range windowPackageManagers { + _, err := exec.LookPath(name) + if err != nil { + continue + } + err = h.WhetherContinue(fmt.Sprintf("use: '%s'", command)) + if err != nil { + return err + } + _, err = h.runner.Execute("cmd", []string{ + "/c", + command, + }, nil, true) + return err + } + + return &Error{msg: "windows platform, please install helm manually, installation steps: https://helm.sh/docs/intro/install/", code: &manuallyInstallCode} +} + +func (h *HelmfileInit) InstallHelm() error { + if runtime.GOOS == "windows" { + return h.installHelmOnWindows() + } + + err := h.WhetherContinue(fmt.Sprintf("use: '%s'", HelmInstallCommand)) + if err != nil { + return err + } + getHelmScript, err := os.CreateTemp("", "get-helm-3.sh") + defer func() { + _ = getHelmScript.Close() + _ = os.Remove(getHelmScript.Name()) + }() + if err != nil { + return err + } + err = downloadfile(getHelmScript.Name(), HelmInstallCommand) + if err != nil { + return err + } + _, err = h.runner.Execute("bash", []string{ + getHelmScript.Name(), + "--version", + HelmRecommendedVersion, + }, nil, true) + if err != nil { + return err + } + h.helmBinary = DefaultHelmBinary + return nil +} + +func (h *HelmfileInit) WhetherContinue(ask string) error { + if h.configProvider.Force() { + return nil + } + askYes := AskForConfirmation(ask) + if !askYes { + return &Error{msg: "cancel automatic installation, please install manually", code: &manuallyInstallCode} + } + return nil +} + +func (h *HelmfileInit) CheckHelmPlugins() error { + settings := cli.New() + helm := helmexec.New(h.helmBinary, false, h.logger, "", h.runner) + for _, p := range helmPlugins { + pluginVersion, err := helmexec.GetPluginVersion(p.name, settings.PluginsDirectory) + if err != nil { + if !strings.Contains(err.Error(), "not installed") { + return err + } + + err = h.WhetherContinue(fmt.Sprintf("The helm plugin %s is not installed, do you need to install it", p.name)) + if err != nil { + return err + } + + err = helm.AddPlugin(p.name, p.repo, p.version) + if err != nil { + return err + } + pluginVersion, _ = helmexec.GetPluginVersion(p.name, settings.PluginsDirectory) + } + requiredVersion, _ := semver.NewVersion(p.version) + if pluginVersion.LessThan(requiredVersion) { + err = h.WhetherContinue(fmt.Sprintf("The helm plugin %s version is too low, do you need to update it", p.name)) + if err != nil { + return err + } + err = helm.UpdatePlugin(p.name) + if err != nil { + return err + } + } + } + return nil +} + +func (h *HelmfileInit) CheckHelm() error { + helmExits := true + _, err := exec.LookPath(h.helmBinary) + if err != nil { + helmExits = false + } + if !helmExits { + h.logger.Info("helm not found, needs to be installed") + err = h.InstallHelm() + if err != nil { + return err + } + } + helmversion, err := helmexec.GetHelmVersion(h.helmBinary, h.runner) + if err != nil { + return err + } + requiredHelmVersion, _ := semver.NewVersion(HelmRequiredVersion) + if helmversion.LessThan(requiredHelmVersion) { + h.logger.Infof("helm version is too low, the current version is %s, the required version is %s", helmversion, requiredHelmVersion) + err = h.UpdateHelm() + if err != nil { + return err + } + } + return nil +} +func (h *HelmfileInit) Initialize() error { + err := h.CheckHelm() + if err != nil { + return err + } + err = h.CheckHelmPlugins() + if err != nil { + return err + } + h.logger.Info("helmfile initialization completed!") + return nil +} diff --git a/pkg/app/init_test.go b/pkg/app/init_test.go new file mode 100644 index 00000000..00a4feea --- /dev/null +++ b/pkg/app/init_test.go @@ -0,0 +1,89 @@ +package app + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path" + "regexp" + "testing" +) + +func TestDownloadfile(t *testing.T) { + var ts *httptest.Server + cases := []struct { + name string + handler func(http.ResponseWriter, *http.Request) + url string + filepath string + wantContent string + wantError string + }{ + { + name: "download success", + handler: func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "helmfile") + }, + wantContent: "helmfile", + }, + { + name: "download 404", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(404) + fmt.Fprint(w, "not found") + }, + wantError: "download .*? error, code: 404", + }, + { + name: "download 500", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + fmt.Fprint(w, "server error") + }, + wantError: "download .*? error, code: 500", + }, + { + name: "download path error", + handler: func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "helmfile") + }, + filepath: "abc/down.txt", + wantError: "open .*? no such file or directory", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + dir := t.TempDir() + downfile := path.Join(dir, "down.txt") + if c.filepath != "" { + downfile = path.Join(dir, c.filepath) + } + + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.handler(w, r) + })) + defer ts.Close() + url := ts.URL + if c.url != "" { + url = c.url + } + err := downloadfile(downfile, url) + if c.wantError != "" { + if err == nil { + t.Errorf("download got no error, want error: %v", c.wantError) + } else if matched, regexErr := regexp.MatchString(c.wantError, err.Error()); regexErr != nil || !matched { + t.Errorf("download got error: %v, want error: %v", err, c.wantError) + } + return + } + content, err := os.ReadFile(downfile) + if err != nil { + t.Errorf("read download file error: %v", err) + } + if string(content) != c.wantContent { + t.Errorf("download file content got: %v, want content: %v", string(content), c.wantContent) + } + }) + } +} diff --git a/pkg/config/init.go b/pkg/config/init.go new file mode 100644 index 00000000..f5dde152 --- /dev/null +++ b/pkg/config/init.go @@ -0,0 +1,30 @@ +package config + +// InitOptions is the options for the init command +type InitOptions struct { + Force bool +} + +// NewInitOptions creates a new InitOptions +func NewInitOptions() *InitOptions { + return &InitOptions{} +} + +// InitImpl is impl for InitOptions +type InitImpl struct { + *GlobalImpl + *InitOptions +} + +// NewInitImpl creates a new InitImpl +func NewInitImpl(g *GlobalImpl, b *InitOptions) *InitImpl { + return &InitImpl{ + GlobalImpl: g, + InitOptions: b, + } +} + +// Force returns the Force. +func (b *InitImpl) Force() bool { + return b.InitOptions.Force +} diff --git a/pkg/helmexec/exec.go b/pkg/helmexec/exec.go index 144bc4f2..3ad41525 100644 --- a/pkg/helmexec/exec.go +++ b/pkg/helmexec/exec.go @@ -79,7 +79,7 @@ func parseHelmVersion(versionStr string) (semver.Version, error) { return *ver, nil } -func getHelmVersion(helmBinary string, runner Runner) (semver.Version, error) { +func GetHelmVersion(helmBinary string, runner Runner) (semver.Version, error) { // Autodetect from `helm version` outBytes, err := runner.Execute(helmBinary, []string{"version", "--client", "--short"}, nil, false) if err != nil { @@ -115,7 +115,7 @@ func redactedURL(chart string) string { // nolint: golint func New(helmBinary string, enableLiveOutput bool, logger *zap.SugaredLogger, kubeContext string, runner Runner) *execer { // TODO: proper error handling - version, err := getHelmVersion(helmBinary, runner) + version, err := GetHelmVersion(helmBinary, runner) if err != nil { panic(err) } @@ -521,6 +521,20 @@ func (helm *execer) TestRelease(context HelmContext, name string, flags ...strin return err } +func (helm *execer) AddPlugin(name, path, version string) error { + helm.logger.Infof("Install helm plugin %v", name) + out, err := helm.exec([]string{"plugin", "install", path, "--version", version}, map[string]string{}, nil) + helm.info(out) + return err +} + +func (helm *execer) UpdatePlugin(name string) error { + helm.logger.Infof("Update helm plugin %v", name) + out, err := helm.exec([]string{"plugin", "update", name}, map[string]string{}, nil) + helm.info(out) + return err +} + func (helm *execer) exec(args []string, env map[string]string, overrideEnableLiveOutput *bool) ([]byte, error) { cmdargs := args if len(helm.extra) > 0 { diff --git a/test/e2e/helmfile-init/init_linux.sh b/test/e2e/helmfile-init/init_linux.sh new file mode 100644 index 00000000..aa918aa9 --- /dev/null +++ b/test/e2e/helmfile-init/init_linux.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# vim: set tabstop=4 shiftwidth=4 + +set -e +set -o pipefail + +# IMPORTS ----------------------------------------------------------------------------------------------------------- + +# determine working directory to use to relative paths irrespective of starting directory +dir="${BASH_SOURCE%/*}" +if [[ ! -d "${dir}" ]]; then dir="${PWD}"; fi + +. "${dir}/../../integration/lib/output.sh" + +helmfile="./helmfile" +helm_dir="${PWD}/${dir}/.helm" +helm=`which helm` +export HELM_DATA_HOME="${helm_dir}/data" +export HELM_HOME="${HELM_DATA_HOME}" +export HELM_PLUGINS="${HELM_DATA_HOME}/plugins" +export HELM_CONFIG_HOME="${helm_dir}/config" + +function cleanup() { + set +e + info "Deleting ${helm_dir}" + rm -rf ${helm_dir} # remove helm data so reinstalling plugins does not fail +} + +function removehelm() { + [ -f $helm ] && rm -rf $helm +} + +set -e +trap cleanup EXIT + +removehelm + +expect <