diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7680c2d1..c928a336 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,16 +52,32 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - helm-version: - - v3.4.2 - - v3.5.4 - - v3.6.3 - - v3.7.2 - - v3.8.2 - - v3.9.4 - plugin-secrets-version: - - 3.15.0 - - 4.0.0 + include: + # We intend to support 2 helm minor version at a time. + # What's why we include only 2 helm minor versions in this matrix. + # See https://github.com/helmfile/helmfile/pull/286#issuecomment-1250161182 for more context. + - helm-version: v3.8.2 + plugin-secrets-version: 3.15.0 + extra-helmfile-flags: + - helm-version: v3.8.2 + # We assume that the helm-secrets plugin is supposed to + # work with the two most recent helm minor versions. + # Once it turned out to be not practically true, + # we will mark this combination as failable, + # and instruct users to upgrade helm and helm-secrets at once. + plugin-secrets-version: 4.0.0 + extra-helmfile-flags: + - helm-version: v3.9.4 + plugin-secrets-version: 3.15.0 + extra-helmfile-flags: + - helm-version: v3.9.4 + plugin-secrets-version: 4.0.0 + extra-helmfile-flags: + # In case you need to test some optional helmfile features, + # enable it via extra-helmfile-flags below. + - helm-version: v3.9.4 + plugin-secrets-version: 4.0.0 + extra-helmfile-flags: "--enable-live-output" steps: - uses: actions/checkout@v2 - name: Cache libraries @@ -102,4 +118,5 @@ jobs: HELM_SECRETS_VERSION: ${{ matrix.plugin-secrets-version }} HELMFILE_HELM3: 1 TERM: xterm + EXTRA_HELMFILE_FLAGS: ${{ matrix.extra-helmfile-flags }} run: make integration diff --git a/cmd/root.go b/cmd/root.go index 587d3e77..5a027843 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -122,6 +122,8 @@ A release must match all labels in a group in order to be used. Multiple groups "--selector tier=frontend,tier!=proxy --selector tier=backend" will match all frontend, non-proxy releases AND all backend releases. The name of a release can be used as a label: "--selector name=myrelease"`) fs.BoolVar(&globalOptions.AllowNoMatchingRelease, "allow-no-matching-release", false, `Do not exit with an error code if the provided selector has no matching releases.`) + fs.BoolVar(&globalOptions.EnableLiveOutput, "enable-live-output", globalOptions.EnableLiveOutput, `Show live output from the Helm binary Stdout/Stderr into Helmfile own Stdout/Stderr. + It only applies for the Helm CLI commands, Stdout/Stderr for Hooks are still displayed only when it's execution finishes.`) // avoid 'pflag: help requested' error (#251) fs.BoolP("help", "h", false, "help for helmfile") } diff --git a/pkg/app/app.go b/pkg/app/app.go index 32ed2b60..5c58717a 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -28,6 +28,7 @@ import ( type App struct { OverrideKubeContext string OverrideHelmBinary string + EnableLiveOutput bool Logger *zap.SugaredLogger Env string @@ -64,6 +65,7 @@ func New(conf ConfigProvider) *App { return Init(&App{ OverrideKubeContext: conf.KubeContext(), OverrideHelmBinary: conf.HelmBinary(), + EnableLiveOutput: conf.EnableLiveOutput(), Logger: conf.Logger(), Env: conf.Env(), Namespace: conf.Namespace(), @@ -84,6 +86,10 @@ func Init(app *App) *App { panic(fmt.Sprintf("Failed to initialize vals runtime: %v", err)) } + if app.EnableLiveOutput { + app.Logger.Info("Live output is enabled") + } + return app } @@ -213,6 +219,9 @@ func (a *App) Template(c TemplateConfigProvider) error { return a.ForEachState(func(run *Run) (ok bool, errs []error) { includeCRDs := c.IncludeCRDs() + // Live output should never be enabled for the "template" subcommand to avoid breaking `helmfile template | kubectl apply -f -` + run.helm.SetEnableLiveOutput(false) + // `helm template` in helm v2 does not support local chart. // So, we set forceDownload=true for helm v2 only prepErr := run.withPreparedCharts("template", state.ChartPrepareOptions{ @@ -717,6 +726,7 @@ func (a *App) loadDesiredStateFromYaml(file string, opts ...LoadOpts) (*state.He overrideKubeContext: a.OverrideKubeContext, overrideHelmBinary: a.OverrideHelmBinary, + enableLiveOutput: a.EnableLiveOutput, getHelm: a.getHelm, valsRuntime: a.valsRuntime, } @@ -755,7 +765,7 @@ func (a *App) getHelm(st *state.HelmState) helmexec.Interface { key := createHelmKey(bin, kubectx) if _, ok := a.helms[key]; !ok { - a.helms[key] = helmexec.New(bin, a.Logger, kubectx, &helmexec.ShellRunner{ + a.helms[key] = helmexec.New(bin, a.EnableLiveOutput, a.Logger, kubectx, &helmexec.ShellRunner{ Logger: a.Logger, }) } diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 97954f7e..ced53445 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -2488,12 +2488,12 @@ func (mock *mockRunner) ExecuteStdIn(cmd string, args []string, env map[string]s return []byte{}, nil } -func (mock *mockRunner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) { +func (mock *mockRunner) Execute(cmd string, args []string, env map[string]string, enableLiveOutput bool) ([]byte, error) { return []byte{}, nil } func MockExecer(logger *zap.SugaredLogger, kubeContext string) helmexec.Interface { - execer := helmexec.New("helm", logger, kubeContext, &mockRunner{}) + execer := helmexec.New("helm", false, logger, kubeContext, &mockRunner{}) return execer } @@ -2538,6 +2538,8 @@ func (helm *mockHelmExec) SetExtraArgs(args ...string) { } func (helm *mockHelmExec) SetHelmBinary(bin string) { } +func (helm *mockHelmExec) SetEnableLiveOutput(enableLiveOutput bool) { +} func (helm *mockHelmExec) AddRepo(name, repository, cafile, certfile, keyfile, username, password string, managed string, passCredentials string, skipTLSVerify string) error { helm.repos = append(helm.repos, mockRepo{Name: name}) return nil diff --git a/pkg/app/config.go b/pkg/app/config.go index a66eff6b..7753cf64 100644 --- a/pkg/app/config.go +++ b/pkg/app/config.go @@ -5,6 +5,7 @@ import "go.uber.org/zap" type ConfigProvider interface { Args() string HelmBinary() string + EnableLiveOutput() bool FileOrDir() string KubeContext() string diff --git a/pkg/app/desired_state_file_loader.go b/pkg/app/desired_state_file_loader.go index ffa11735..719e01de 100644 --- a/pkg/app/desired_state_file_loader.go +++ b/pkg/app/desired_state_file_loader.go @@ -24,6 +24,7 @@ const ( type desiredStateLoader struct { overrideKubeContext string overrideHelmBinary string + enableLiveOutput bool env string namespace string @@ -162,7 +163,7 @@ func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *e } func (a *desiredStateLoader) underlying() *state.StateCreator { - c := state.NewCreator(a.logger, a.fs, a.valsRuntime, a.getHelm, a.overrideHelmBinary, a.remote) + c := state.NewCreator(a.logger, a.fs, a.valsRuntime, a.getHelm, a.overrideHelmBinary, a.remote, a.enableLiveOutput) c.LoadFile = a.loadFile return c } diff --git a/pkg/app/mocks_test.go b/pkg/app/mocks_test.go index 40a96836..94f2b437 100644 --- a/pkg/app/mocks_test.go +++ b/pkg/app/mocks_test.go @@ -46,6 +46,9 @@ func (helm *noCallHelmExec) SetExtraArgs(args ...string) { func (helm *noCallHelmExec) SetHelmBinary(bin string) { helm.doPanic() } +func (helm *noCallHelmExec) SetEnableLiveOutput(enableLiveOutput bool) { + helm.doPanic() +} func (helm *noCallHelmExec) AddRepo(name, repository, cafile, certfile, keyfile, username, password string, managed string, passCredentials string, skipTLSVerify string) error { helm.doPanic() return nil diff --git a/pkg/config/global.go b/pkg/config/global.go index fa511472..7e902836 100644 --- a/pkg/config/global.go +++ b/pkg/config/global.go @@ -44,6 +44,8 @@ type GlobalOptions struct { AllowNoMatchingRelease bool // logger is the logger to use. logger *zap.SugaredLogger + // EnableLiveOutput enables live output from the Helm binary stdout/stderr into Helmfile own stdout/stderr + EnableLiveOutput bool } // Logger returns the logger to use. @@ -120,6 +122,11 @@ func (g *GlobalImpl) StateValuesFiles() []string { return g.GlobalOptions.StateValuesFile } +// EnableLiveOutput return when to pipe the stdout and stderr from Helm live to the helmfile stdout +func (g *GlobalImpl) EnableLiveOutput() bool { + return g.GlobalOptions.EnableLiveOutput +} + // Logger returns the logger func (g *GlobalImpl) Logger() *zap.SugaredLogger { return g.GlobalOptions.logger diff --git a/pkg/event/bus.go b/pkg/event/bus.go index fe0b245c..92b69bff 100644 --- a/pkg/event/bus.go +++ b/pkg/event/bus.go @@ -118,7 +118,7 @@ func (bus *Bus) Trigger(evt string, evtErr error, context map[string]interface{} } } - bytes, err := bus.Runner.Execute(command, args, map[string]string{}) + bytes, err := bus.Runner.Execute(command, args, map[string]string{}, false) bus.Logger.Debugf("hook[%s]: %s\n", name, string(bytes)) if hook.ShowLogs { prefix := fmt.Sprintf("\nhook[%s] logs | ", evt) diff --git a/pkg/event/bus_test.go b/pkg/event/bus_test.go index b7e509cb..f2cc2d89 100644 --- a/pkg/event/bus_test.go +++ b/pkg/event/bus_test.go @@ -19,7 +19,7 @@ func (r *runner) ExecuteStdIn(cmd string, args []string, env map[string]string, return []byte(""), nil } -func (r *runner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) { +func (r *runner) Execute(cmd string, args []string, env map[string]string, enableLiveOutput bool) ([]byte, error) { if cmd == "ng" { return nil, fmt.Errorf("cmd failed due to invalid cmd: %s", cmd) } diff --git a/pkg/exectest/helm.go b/pkg/exectest/helm.go index 8f01947b..4e1c1a7a 100644 --- a/pkg/exectest/helm.go +++ b/pkg/exectest/helm.go @@ -89,6 +89,8 @@ func (helm *Helm) SetExtraArgs(args ...string) { } func (helm *Helm) SetHelmBinary(bin string) { } +func (helm *Helm) SetEnableLiveOutput(enableLiveOutput bool) { +} func (helm *Helm) AddRepo(name, repository, cafile, certfile, keyfile, username, password string, managed string, passCredentials string, skipTLSVerify string) error { helm.Repo = []string{name, repository, cafile, certfile, keyfile, username, password, managed, passCredentials, skipTLSVerify} return nil diff --git a/pkg/helmexec/exec.go b/pkg/helmexec/exec.go index 74a7b135..d57f8545 100644 --- a/pkg/helmexec/exec.go +++ b/pkg/helmexec/exec.go @@ -28,6 +28,7 @@ type decryptedSecret struct { type execer struct { helmBinary string + enableLiveOutput bool version semver.Version runner Runner logger *zap.SugaredLogger @@ -78,7 +79,7 @@ func parseHelmVersion(versionStr string) (semver.Version, error) { func getHelmVersion(helmBinary string, runner Runner) (semver.Version, error) { // Autodetect from `helm version` - outBytes, err := runner.Execute(helmBinary, []string{"version", "--client", "--short"}, nil) + outBytes, err := runner.Execute(helmBinary, []string{"version", "--client", "--short"}, nil, false) if err != nil { return semver.Version{}, fmt.Errorf("error determining helm version: %w", err) } @@ -110,7 +111,7 @@ func redactedURL(chart string) string { // New for running helm commands // nolint: golint -func New(helmBinary string, logger *zap.SugaredLogger, kubeContext string, runner Runner) *execer { +func New(helmBinary string, enableLiveOutput bool, logger *zap.SugaredLogger, kubeContext string, runner Runner) *execer { // TODO: proper error handling version, err := getHelmVersion(helmBinary, runner) if err != nil { @@ -118,6 +119,7 @@ func New(helmBinary string, logger *zap.SugaredLogger, kubeContext string, runne } return &execer{ helmBinary: helmBinary, + enableLiveOutput: enableLiveOutput, version: version, logger: logger, kubeContext: kubeContext, @@ -134,6 +136,10 @@ func (helm *execer) SetHelmBinary(bin string) { helm.helmBinary = bin } +func (helm *execer) SetEnableLiveOutput(enableLiveOutput bool) { + helm.enableLiveOutput = enableLiveOutput +} + func (helm *execer) AddRepo(name, repository, cafile, certfile, keyfile, username, password string, managed string, passCredentials string, skipTLSVerify string) error { var args []string var out []byte @@ -174,7 +180,7 @@ func (helm *execer) AddRepo(name, repository, cafile, certfile, keyfile, usernam args = append(args, "--insecure-skip-tls-verify") } helm.logger.Infof("Adding repo %v %v", name, repository) - out, err = helm.exec(args, map[string]string{}) + out, err = helm.exec(args, map[string]string{}, nil) default: helm.logger.Errorf("ERROR: unknown type '%v' for repository %v", managed, name) out = nil @@ -186,7 +192,7 @@ func (helm *execer) AddRepo(name, repository, cafile, certfile, keyfile, usernam func (helm *execer) UpdateRepo() error { helm.logger.Info("Updating repo") - out, err := helm.exec([]string{"repo", "update"}, map[string]string{}) + out, err := helm.exec([]string{"repo", "update"}, map[string]string{}, nil) helm.info(out) return err } @@ -210,14 +216,14 @@ func (helm *execer) RegistryLogin(repository string, username string, password s func (helm *execer) BuildDeps(name, chart string) error { helm.logger.Infof("Building dependency release=%v, chart=%v", name, chart) - out, err := helm.exec([]string{"dependency", "build", chart}, map[string]string{}) + out, err := helm.exec([]string{"dependency", "build", chart}, map[string]string{}, nil) helm.info(out) return err } func (helm *execer) UpdateDeps(chart string) error { helm.logger.Infof("Updating dependency %v", chart) - out, err := helm.exec([]string{"dependency", "update", chart}, map[string]string{}) + out, err := helm.exec([]string{"dependency", "update", chart}, map[string]string{}, nil) helm.info(out) return err } @@ -233,7 +239,7 @@ func (helm *execer) SyncRelease(context HelmContext, name, chart string, flags . env["HELM_TILLER_HISTORY_MAX"] = strconv.Itoa(context.HistoryMax) } - out, err := helm.exec(append(append(preArgs, "upgrade", "--install", "--reset-values", name, chart), flags...), env) + out, err := helm.exec(append(append(preArgs, "upgrade", "--install", "--reset-values", name, chart), flags...), env, nil) helm.write(nil, out) return err } @@ -242,7 +248,7 @@ func (helm *execer) ReleaseStatus(context HelmContext, name string, flags ...str helm.logger.Infof("Getting status %v", name) preArgs := context.GetTillerlessArgs(helm) env := context.getTillerlessEnv() - out, err := helm.exec(append(append(preArgs, "status", name), flags...), env) + out, err := helm.exec(append(append(preArgs, "status", name), flags...), env, nil) helm.write(nil, out) return err } @@ -258,7 +264,8 @@ func (helm *execer) List(context HelmContext, filter string, flags ...string) (s args = []string{"list", filter} } - out, err := helm.exec(append(append(preArgs, args...), flags...), env) + enableLiveOutput := false + out, err := helm.exec(append(append(preArgs, args...), flags...), env, &enableLiveOutput) // In v2 we have been expecting `helm list FILTER` prints nothing. // In v3 helm still prints the header like `NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION`, // which confuses helmfile's existing logic that treats any non-empty output from `helm list` is considered as the indication @@ -308,7 +315,8 @@ func (helm *execer) DecryptSecret(context HelmContext, name string, flags ...str if pluginVersion.Major() > 3 { secretArg = "decrypt" } - secretBytes, err := helm.exec(append(append(preArgs, "secrets", secretArg, absPath), flags...), env) + enableLiveOutput := false + secretBytes, err := helm.exec(append(append(preArgs, "secrets", secretArg, absPath), flags...), env, &enableLiveOutput) if err != nil { secret.err = err return "", err @@ -370,7 +378,7 @@ func (helm *execer) TemplateRelease(name string, chart string, flags ...string) args = []string{"template", chart, "--name", name} } - out, err := helm.exec(append(args, flags...), map[string]string{}) + out, err := helm.exec(append(args, flags...), map[string]string{}, nil) var outputToFile bool @@ -407,7 +415,12 @@ func (helm *execer) DiffRelease(context HelmContext, name, chart string, suppres } preArgs := context.GetTillerlessArgs(helm) env := context.getTillerlessEnv() - out, err := helm.exec(append(append(preArgs, "diff", "upgrade", "--reset-values", "--allow-unreleased", name, chart), flags...), env) + var overrideEnableLiveOutput *bool = nil + if suppressDiff { + enableLiveOutput := false + overrideEnableLiveOutput = &enableLiveOutput + } + out, err := helm.exec(append(append(preArgs, "diff", "upgrade", "--reset-values", "--allow-unreleased", name, chart), flags...), env, overrideEnableLiveOutput) // Do our best to write STDOUT only when diff existed // Unfortunately, this works only when you run helmfile with `--detailed-exitcode` detailedExitcodeEnabled := false @@ -433,14 +446,14 @@ func (helm *execer) DiffRelease(context HelmContext, name, chart string, suppres func (helm *execer) Lint(name, chart string, flags ...string) error { helm.logger.Infof("Linting release=%v, chart=%v", name, chart) - out, err := helm.exec(append([]string{"lint", chart}, flags...), map[string]string{}) + out, err := helm.exec(append([]string{"lint", chart}, flags...), map[string]string{}, nil) helm.write(nil, out) return err } func (helm *execer) Fetch(chart string, flags ...string) error { helm.logger.Infof("Fetching %v", redactedURL(chart)) - out, err := helm.exec(append([]string{"fetch", chart}, flags...), map[string]string{}) + out, err := helm.exec(append([]string{"fetch", chart}, flags...), map[string]string{}, nil) helm.info(out) return err } @@ -463,7 +476,7 @@ func (helm *execer) ChartPull(chart string, flags ...string) error { } else { helmArgs = []string{"chart", "pull", chart} } - out, err := helm.exec(append(helmArgs, flags...), map[string]string{"HELM_EXPERIMENTAL_OCI": "1"}) + out, err := helm.exec(append(helmArgs, flags...), map[string]string{"HELM_EXPERIMENTAL_OCI": "1"}, nil) helm.info(out) return err } @@ -478,7 +491,7 @@ func (helm *execer) ChartExport(chart string, path string, flags ...string) erro } else { helmArgs = []string{"chart", "export", chart} } - out, err := helm.exec(append(append(helmArgs, "--destination", path), flags...), map[string]string{"HELM_EXPERIMENTAL_OCI": "1"}) + out, err := helm.exec(append(append(helmArgs, "--destination", path), flags...), map[string]string{"HELM_EXPERIMENTAL_OCI": "1"}, nil) helm.info(out) return err } @@ -487,7 +500,7 @@ func (helm *execer) DeleteRelease(context HelmContext, name string, flags ...str helm.logger.Infof("Deleting %v", name) preArgs := context.GetTillerlessArgs(helm) env := context.getTillerlessEnv() - out, err := helm.exec(append(append(preArgs, "delete", name), flags...), env) + out, err := helm.exec(append(append(preArgs, "delete", name), flags...), env, nil) helm.write(nil, out) return err } @@ -497,12 +510,12 @@ func (helm *execer) TestRelease(context HelmContext, name string, flags ...strin preArgs := context.GetTillerlessArgs(helm) env := context.getTillerlessEnv() args := []string{"test", name} - out, err := helm.exec(append(append(preArgs, args...), flags...), env) + out, err := helm.exec(append(append(preArgs, args...), flags...), env, nil) helm.write(nil, out) return err } -func (helm *execer) exec(args []string, env map[string]string) ([]byte, error) { +func (helm *execer) exec(args []string, env map[string]string, overrideEnableLiveOutput *bool) ([]byte, error) { cmdargs := args if len(helm.extra) > 0 { cmdargs = append(cmdargs, helm.extra...) @@ -512,7 +525,11 @@ func (helm *execer) exec(args []string, env map[string]string) ([]byte, error) { } cmd := fmt.Sprintf("exec: %s %s", helm.helmBinary, strings.Join(cmdargs, " ")) helm.logger.Debug(cmd) - outBytes, err := helm.runner.Execute(helm.helmBinary, cmdargs, env) + enableLiveOutput := helm.enableLiveOutput + if overrideEnableLiveOutput != nil { + enableLiveOutput = *overrideEnableLiveOutput + } + outBytes, err := helm.runner.Execute(helm.helmBinary, cmdargs, env, enableLiveOutput) return outBytes, err } @@ -534,7 +551,7 @@ func (helm *execer) azcli(name string) ([]byte, error) { cmdargs := append(strings.Split("acr helm repo add --name", " "), name) cmd := fmt.Sprintf("exec: az %s", strings.Join(cmdargs, " ")) helm.logger.Debug(cmd) - outBytes, err := helm.runner.Execute("az", cmdargs, map[string]string{}) + outBytes, err := helm.runner.Execute("az", cmdargs, map[string]string{}, false) helm.logger.Debugf("%s: %s", cmd, outBytes) return outBytes, err } diff --git a/pkg/helmexec/exec_test.go b/pkg/helmexec/exec_test.go index 7dbc19d0..24343620 100644 --- a/pkg/helmexec/exec_test.go +++ b/pkg/helmexec/exec_test.go @@ -29,13 +29,13 @@ func (mock *mockRunner) ExecuteStdIn(cmd string, args []string, env map[string]s return mock.output, mock.err } -func (mock *mockRunner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) { +func (mock *mockRunner) Execute(cmd string, args []string, env map[string]string, enableLiveOutput bool) ([]byte, error) { return mock.output, mock.err } // nolint: golint func MockExecer(logger *zap.SugaredLogger, kubeContext string) *execer { - execer := New("helm", logger, kubeContext, &mockRunner{}) + execer := New("helm", false, logger, kubeContext, &mockRunner{}) return execer } @@ -82,6 +82,17 @@ func Test_SetHelmBinary(t *testing.T) { } } +func Test_SetEnableLiveOutput(t *testing.T) { + helm := MockExecer(NewLogger(os.Stdout, "info"), "dev") + if helm.enableLiveOutput { + t.Error("helmexec.enableLiveOutput should not be enabled by default") + } + helm.SetEnableLiveOutput(true) + if !helm.enableLiveOutput { + t.Errorf("helmexec.SetEnableLiveOutput() - actual = %t expect = true", helm.enableLiveOutput) + } +} + func Test_AddRepo_Helm_3_3_2(t *testing.T) { var buffer bytes.Buffer logger := NewLogger(&buffer, "debug") @@ -582,7 +593,7 @@ func Test_exec(t *testing.T) { logger := NewLogger(&buffer, "debug") helm := MockExecer(logger, "") env := map[string]string{} - _, err := helm.exec([]string{"version"}, env) + _, err := helm.exec([]string{"version"}, env, nil) expected := `exec: helm version ` if err != nil { @@ -593,14 +604,14 @@ func Test_exec(t *testing.T) { } helm = MockExecer(logger, "dev") - ret, _ := helm.exec([]string{"diff"}, env) + ret, _ := helm.exec([]string{"diff"}, env, nil) if len(ret) != 0 { t.Error("helmexec.exec() - expected empty return value") } buffer.Reset() helm = MockExecer(logger, "dev") - _, err = helm.exec([]string{"diff", "release", "chart", "--timeout 10", "--wait", "--wait-for-jobs"}, env) + _, err = helm.exec([]string{"diff", "release", "chart", "--timeout 10", "--wait", "--wait-for-jobs"}, env, nil) expected = `exec: helm --kube-context dev diff release chart --timeout 10 --wait --wait-for-jobs ` if err != nil { @@ -611,7 +622,7 @@ func Test_exec(t *testing.T) { } buffer.Reset() - _, err = helm.exec([]string{"version"}, env) + _, err = helm.exec([]string{"version"}, env, nil) expected = `exec: helm --kube-context dev version ` if err != nil { @@ -623,7 +634,7 @@ func Test_exec(t *testing.T) { buffer.Reset() helm.SetExtraArgs("foo") - _, err = helm.exec([]string{"version"}, env) + _, err = helm.exec([]string{"version"}, env, nil) expected = `exec: helm --kube-context dev version foo ` if err != nil { @@ -636,7 +647,7 @@ func Test_exec(t *testing.T) { buffer.Reset() helm = MockExecer(logger, "") helm.SetHelmBinary("overwritten") - _, err = helm.exec([]string{"version"}, env) + _, err = helm.exec([]string{"version"}, env, nil) expected = `exec: overwritten version ` if err != nil { @@ -896,20 +907,20 @@ exec: helm --kube-context dev template https://example_user:example_password@rep func Test_IsHelm3(t *testing.T) { helm2Runner := mockRunner{output: []byte("Client: v2.16.0+ge13bc94\n")} - helm := New("helm", NewLogger(os.Stdout, "info"), "dev", &helm2Runner) + helm := New("helm", false, NewLogger(os.Stdout, "info"), "dev", &helm2Runner) if helm.IsHelm3() { t.Error("helmexec.IsHelm3() - Detected Helm 3 with Helm 2 version") } helm3Runner := mockRunner{output: []byte("v3.0.0+ge29ce2a\n")} - helm = New("helm", NewLogger(os.Stdout, "info"), "dev", &helm3Runner) + helm = New("helm", false, NewLogger(os.Stdout, "info"), "dev", &helm3Runner) if !helm.IsHelm3() { t.Error("helmexec.IsHelm3() - Failed to detect Helm 3") } t.Setenv(envvar.Helm3, "1") helm2Runner = mockRunner{output: []byte("Client: v2.16.0+ge13bc94\n")} - helm = New("helm", NewLogger(os.Stdout, "info"), "dev", &helm2Runner) + helm = New("helm", false, NewLogger(os.Stdout, "info"), "dev", &helm2Runner) if !helm.IsHelm3() { t.Errorf("helmexec.IsHelm3() - Helm3 not detected when %s is set", envvar.Helm3) } @@ -940,14 +951,14 @@ func Test_GetPluginVersion(t *testing.T) { func Test_GetVersion(t *testing.T) { helm2Runner := mockRunner{output: []byte("Client: v2.16.1+ge13bc94\n")} - helm := New("helm", NewLogger(os.Stdout, "info"), "dev", &helm2Runner) + helm := New("helm", false, NewLogger(os.Stdout, "info"), "dev", &helm2Runner) ver := helm.GetVersion() if ver.Major != 2 || ver.Minor != 16 || ver.Patch != 1 { t.Errorf("helmexec.GetVersion - did not detect correct Helm2 version; it was: %+v", ver) } helm3Runner := mockRunner{output: []byte("v3.2.4+ge29ce2a\n")} - helm = New("helm", NewLogger(os.Stdout, "info"), "dev", &helm3Runner) + helm = New("helm", false, NewLogger(os.Stdout, "info"), "dev", &helm3Runner) ver = helm.GetVersion() if ver.Major != 3 || ver.Minor != 2 || ver.Patch != 4 { t.Errorf("helmexec.GetVersion - did not detect correct Helm3 version; it was: %+v", ver) @@ -956,7 +967,7 @@ func Test_GetVersion(t *testing.T) { func Test_IsVersionAtLeast(t *testing.T) { helm2Runner := mockRunner{output: []byte("Client: v2.16.1+ge13bc94\n")} - helm := New("helm", NewLogger(os.Stdout, "info"), "dev", &helm2Runner) + helm := New("helm", false, NewLogger(os.Stdout, "info"), "dev", &helm2Runner) if !helm.IsVersionAtLeast("2.1.0") { t.Error("helmexec.IsVersionAtLeast - 2.16.1 not atleast 2.1") } diff --git a/pkg/helmexec/helmexec.go b/pkg/helmexec/helmexec.go index d5400450..39158739 100644 --- a/pkg/helmexec/helmexec.go +++ b/pkg/helmexec/helmexec.go @@ -11,6 +11,7 @@ type Version struct { type Interface interface { SetExtraArgs(args ...string) SetHelmBinary(bin string) + SetEnableLiveOutput(enableLiveOutput bool) AddRepo(name, repository, cafile, certfile, keyfile, username, password string, managed string, passCredentials string, skipTLSVerify string) error UpdateRepo() error diff --git a/pkg/helmexec/runner.go b/pkg/helmexec/runner.go index ec573d71..fc994846 100644 --- a/pkg/helmexec/runner.go +++ b/pkg/helmexec/runner.go @@ -1,6 +1,7 @@ package helmexec import ( + "bufio" "bytes" "errors" "fmt" @@ -16,7 +17,7 @@ import ( // Runner interface for shell commands type Runner interface { - Execute(cmd string, args []string, env map[string]string) ([]byte, error) + Execute(cmd string, args []string, env map[string]string, enableLiveOutput bool) ([]byte, error) ExecuteStdIn(cmd string, args []string, env map[string]string, stdin io.Reader) ([]byte, error) } @@ -28,13 +29,18 @@ type ShellRunner struct { } // Execute a shell command -func (shell ShellRunner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) { +func (shell ShellRunner) Execute(cmd string, args []string, env map[string]string, enableLiveOutput bool) ([]byte, error) { preparedCmd := exec.Command(cmd, args...) preparedCmd.Dir = shell.Dir preparedCmd.Env = mergeEnv(os.Environ(), env) - return Output(preparedCmd, &logWriterGenerator{ - log: shell.Logger, - }) + + if !enableLiveOutput { + return Output(preparedCmd, &logWriterGenerator{ + log: shell.Logger, + }) + } else { + return LiveOutput(preparedCmd, os.Stdout) + } } // Execute a shell command @@ -94,6 +100,43 @@ func Output(c *exec.Cmd, logWriterGenerators ...*logWriterGenerator) ([]byte, er return stdout.Bytes(), err } +func LiveOutput(c *exec.Cmd, stdout io.Writer) ([]byte, error) { + reader, writer := io.Pipe() + scannerStopped := make(chan struct{}) + + go func() { + defer close(scannerStopped) + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + fmt.Fprintln(stdout, scanner.Text()) + } + }() + + c.Stdout = writer + c.Stderr = writer + err := c.Start() + if err == nil { + err = c.Wait() + _ = writer.Close() + <-scannerStopped + } + + if err != nil { + switch ee := err.(type) { + case *exec.ExitError: + // Propagate any non-zero exit status from the external command, rather than throwing it away, + // so that helmfile could return its own exit code accordingly + waitStatus := ee.Sys().(syscall.WaitStatus) + exitStatus := waitStatus.ExitStatus() + err = newExitError(c.Path, c.Args, exitStatus, ee, "", "") + default: + panic(fmt.Sprintf("unexpected error: %v", err)) + } + } + + return nil, err +} + func mergeEnv(orig []string, new map[string]string) []string { wanted := env2map(orig) for k, v := range new { diff --git a/pkg/helmexec/runner_test.go b/pkg/helmexec/runner_test.go new file mode 100644 index 00000000..d174c893 --- /dev/null +++ b/pkg/helmexec/runner_test.go @@ -0,0 +1,88 @@ +package helmexec + +import ( + "bytes" + "os/exec" + "reflect" + "strings" + "testing" +) + +func TestShellRunner_Execute(t *testing.T) { + tests := []struct { + name string + want []byte + stdoutWant string + enableLiveOutput bool + }{ + { + name: "echo_template_no_live_output", + want: []byte("template\n"), + enableLiveOutput: false, + }, + { + name: "echo_template_enable_live_output", + want: nil, + enableLiveOutput: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buffer bytes.Buffer + shell := ShellRunner{ + Logger: NewLogger(&buffer, "debug"), + } + got, err := shell.Execute("echo", strings.Split("template", " "), map[string]string{}, tt.enableLiveOutput) + + if err != nil { + t.Errorf("Execute() has produced an error = %v", err) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ExecuteStdIn() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLiveOutput(t *testing.T) { + tests := []struct { + name string + cmd *exec.Cmd + wantW string + wantErr bool + }{ + { + name: "echo_template", + cmd: exec.Command("echo", "template"), + wantW: "template\n", + wantErr: false, + }, + { + name: "helm_template", + cmd: exec.Command("helm", "template"), + wantW: `Error: "helm template" requires at least 1 argument + +Usage: helm template [NAME] [CHART] [flags] +`, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := &bytes.Buffer{} + got, err := LiveOutput(tt.cmd, w) + if (err != nil) != tt.wantErr { + t.Errorf("LiveOutput() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotW := w.String(); gotW != tt.wantW { + t.Errorf("LiveOutput() gotW = %v, want %v", gotW, tt.wantW) + } + if got != nil { + t.Errorf("LiveOutput() got unespected %v", got) + } + }) + } +} diff --git a/pkg/state/create.go b/pkg/state/create.go index 2829a678..540f6640 100644 --- a/pkg/state/create.go +++ b/pkg/state/create.go @@ -55,10 +55,12 @@ type StateCreator struct { overrideHelmBinary string + enableLiveOutput bool + remote *remote.Remote } -func NewCreator(logger *zap.SugaredLogger, fs *filesystem.FileSystem, valsRuntime vals.Evaluator, getHelm func(*HelmState) helmexec.Interface, overrideHelmBinary string, remote *remote.Remote) *StateCreator { +func NewCreator(logger *zap.SugaredLogger, fs *filesystem.FileSystem, valsRuntime vals.Evaluator, getHelm func(*HelmState) helmexec.Interface, overrideHelmBinary string, remote *remote.Remote, enableLiveOutput bool) *StateCreator { return &StateCreator{ logger: logger, @@ -68,6 +70,7 @@ func NewCreator(logger *zap.SugaredLogger, fs *filesystem.FileSystem, valsRuntim getHelm: getHelm, overrideHelmBinary: overrideHelmBinary, + enableLiveOutput: enableLiveOutput, remote: remote, } diff --git a/pkg/state/create_test.go b/pkg/state/create_test.go index 83b17a12..18a1ab70 100644 --- a/pkg/state/create_test.go +++ b/pkg/state/create_test.go @@ -65,6 +65,10 @@ type stateTestEnv struct { } func (testEnv stateTestEnv) MustLoadState(t *testing.T, file, envName string) *HelmState { + return testEnv.MustLoadStateWithEnableLiveOutput(t, file, envName, false) +} + +func (testEnv stateTestEnv) MustLoadStateWithEnableLiveOutput(t *testing.T, file, envName string, enableLiveOutput bool) *HelmState { t.Helper() testFs := testhelper.NewTestFs(testEnv.Files) @@ -79,7 +83,7 @@ func (testEnv stateTestEnv) MustLoadState(t *testing.T, file, envName string) *H } r := remote.NewRemote(logger, testFs.Cwd, testFs.ToFileSystem()) - state, err := NewCreator(logger, testFs.ToFileSystem(), nil, nil, "", r). + state, err := NewCreator(logger, testFs.ToFileSystem(), nil, nil, "", r, enableLiveOutput). ParseAndLoad([]byte(yamlContent), filepath.Dir(file), file, envName, true, nil) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -149,7 +153,7 @@ releaseNamespace: mynamespace env := environment.Environment{ Name: "production", } - state, err := NewCreator(logger, testFs.ToFileSystem(), nil, nil, "", r). + state, err := NewCreator(logger, testFs.ToFileSystem(), nil, nil, "", r, false). ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", true, &env) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -236,7 +240,7 @@ overrideNamespace: myns testFs.Cwd = "/example/path/to" r := remote.NewRemote(logger, testFs.Cwd, testFs.ToFileSystem()) - state, err := NewCreator(logger, testFs.ToFileSystem(), nil, nil, "", r). + state, err := NewCreator(logger, testFs.ToFileSystem(), nil, nil, "", r, false). ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", true, nil) if err != nil { t.Fatalf("unexpected error: %v", err) diff --git a/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/config.yaml b/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/config.yaml new file mode 100644 index 00000000..41314ff1 --- /dev/null +++ b/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/config.yaml @@ -0,0 +1,7 @@ +localChartRepoServer: + enabled: true + port: 18080 +chartifyTempDir: temp1 +helmfileArgs: +- --enable-live-output +- template diff --git a/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/input.yaml b/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/input.yaml new file mode 100644 index 00000000..13fd58a0 --- /dev/null +++ b/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/input.yaml @@ -0,0 +1,31 @@ +repositories: +- name: myrepo + url: http://localhost:18080/ + +releases: +- name: foo + chart: ../../charts/raw + values: + - templates: + - | + apiVersion: v1 + kind: ConfigMap + metadata: + name: {{`{{ .Release.Name }}`}}-1 + namespace: {{`{{ .Release.Namespace }}`}} + data: + foo: FOO + dep: + templates: + - | + apiVersion: v1 + kind: ConfigMap + metadata: + name: {{`{{ .Release.Name }}`}}-2 + namespace: {{`{{ .Release.Namespace }}`}} + data: + bar: BAR + dependencies: + - alias: dep + chart: myrepo/raw + version: 0.1.0 diff --git a/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/output.yaml b/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/output.yaml new file mode 100644 index 00000000..e19d662a --- /dev/null +++ b/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/output.yaml @@ -0,0 +1,27 @@ +Live output is enabled +Adding repo myrepo http://localhost:18080/ +"myrepo" has been added to your repositories + +Building dependency release=foo, chart=$WD/temp1/foo +Templating release=foo, chart=$WD/temp1/foo +--- +# Source: raw/templates/charts/dep/templates/resources.yaml +# Source: raw/charts/dep/templates/resources.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: foo-2 + namespace: default +data: + bar: BAR +--- +# Source: raw/templates/resources.yaml +# Source: raw/templates/resources.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: foo-1 + namespace: default +data: + foo: FOO + diff --git a/test/integration/run.sh b/test/integration/run.sh index b1b25799..5c361793 100755 --- a/test/integration/run.sh +++ b/test/integration/run.sh @@ -17,7 +17,7 @@ if [[ ! -d "${dir}" ]]; then dir="${PWD}"; fi # GLOBALS ----------------------------------------------------------------------------------------------------------- test_ns="helmfile-tests-$(date +"%Y%m%d-%H%M%S")" -helmfile="./helmfile --namespace=${test_ns}" +helmfile="./helmfile ${EXTRA_HELMFILE_FLAGS} --namespace=${test_ns}" helm="helm --kube-context=minikube" kubectl="kubectl --context=minikube --namespace=${test_ns}" helm_dir="${PWD}/${dir}/.helm"