diff --git a/cmd/fetch.go b/cmd/fetch.go index 3ff693e5..fb4c6adf 100644 --- a/cmd/fetch.go +++ b/cmd/fetch.go @@ -15,6 +15,14 @@ func NewFetchCmd(globalCfg *config.GlobalImpl) *cobra.Command { cmd := &cobra.Command{ Use: "fetch", Short: "Fetch charts from state file", + Long: `Fetch downloads all charts referenced in the Helmfile state. + +Useful for air-gapped environments: download charts with --output-dir and --write-output, +then transfer the output directory and the generated helmfile.yaml to the air-gapped environment. + +The --write-output flag requires a single helmfile state file specified with -f. +It fails if the input resolves to multiple state files (e.g. a directory or a helmfile +with nested helmfiles: entries).`, RunE: func(cmd *cobra.Command, args []string) error { fetchImpl := config.NewFetchImpl(globalCfg, fetchOptions) err := config.NewCLIConfigImpl(fetchImpl.GlobalImpl) @@ -35,6 +43,7 @@ func NewFetchCmd(globalCfg *config.GlobalImpl) *cobra.Command { f.IntVar(&fetchOptions.Concurrency, "concurrency", 0, "maximum number of concurrent helm processes to run, 0 is unlimited") f.StringVar(&fetchOptions.OutputDir, "output-dir", "", "directory to store charts (default: temporary directory which is deleted when the command terminates)") f.StringVar(&fetchOptions.OutputDirTemplate, "output-dir-template", state.DefaultFetchOutputDirTemplate, "go text template for generating the output directory. Available fields: {{ .OutputDir }}, {{ .ChartName }}, {{ .Release.* }}, {{ .Environment.Name }}, {{ .Environment.KubeContext }}, {{ .Environment.Values.* }}") + f.BoolVar(&fetchOptions.WriteOutput, "write-output", false, "write a helmfile.yaml to stdout with chart references updated to local chart paths; requires --output-dir and a single helmfile (use -f)") return cmd } diff --git a/pkg/app/app.go b/pkg/app/app.go index 29a1ce40..6b9738c6 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -393,7 +393,49 @@ func (a *App) Unittest(c UnittestConfigProvider) error { } func (a *App) Fetch(c FetchConfigProvider) error { - return a.ForEachState(func(run *Run) (ok bool, errs []error) { + if c.WriteOutput() && c.OutputDir() == "" { + return fmt.Errorf("--output-dir is required when --write-output is set") + } + + if c.WriteOutput() { + // Force sequential processing to ensure YAML documents are emitted in order + // without interleaving when multiple helmfile state files are processed. + // Restore the original value when Fetch returns so the App instance is not + // permanently mutated (important for tests and library usage). + prev := a.SequentialHelmfiles + a.SequentialHelmfiles = true + defer func() { a.SequentialHelmfiles = prev }() + } + + // processedStateFileCount tracks how many state files have been processed when + // --write-output is set; used to detect multi-file inputs early and return + // a clear error instead of silently producing semantically incorrect YAML. + var processedStateFileCount int + + // yamlOutput buffers the generated YAML document so that nothing is written to + // stdout until ForEachState completes successfully. This prevents partial/corrupted + // output reaching stdout when a later state file (or chart download error) causes + // the operation to fail. + var yamlOutput strings.Builder + + err := a.ForEachState(func(run *Run) (ok bool, errs []error) { + if c.WriteOutput() { + processedStateFileCount++ + if processedStateFileCount > 1 { + return false, []error{fmt.Errorf( + "--write-output requires a single helmfile state file, but multiple were found; " + + "use -f to specify a single helmfile instead of a directory or a helmfile with nested helmfiles: entries", + )} + } + + // Disable live output to avoid Helm progress/status lines being streamed + // to stdout and corrupting the YAML document emitted by --write-output. + // Restore the original value when this callback returns so the cached helm + // exec instance is not permanently mutated (important for tests and library usage). + run.helm.SetEnableLiveOutput(false) + defer run.helm.SetEnableLiveOutput(a.EnableLiveOutput) + } + prepErr := run.withPreparedCharts("pull", state.ChartPrepareOptions{ ForceDownload: true, SkipRefresh: c.SkipRefresh(), @@ -403,6 +445,27 @@ func (a *App) Fetch(c FetchConfigProvider) error { OutputDirTemplate: c.OutputDirTemplate(), Concurrency: c.Concurrency(), }, func() []error { + if c.WriteOutput() { + for i := range run.state.Releases { + rel := &run.state.Releases[i] + if rel.ChartPath != "" { + rel.Chart = rel.ChartPath + rel.ChartPath = "" + } + } + + stateYaml, yamlErr := run.state.ToYaml() + if yamlErr != nil { + return []error{yamlErr} + } + + sourceFile, pathErr := run.state.FullFilePath() + if pathErr != nil { + return []error{pathErr} + } + fmt.Fprintf(&yamlOutput, "---\n# Source: %s\n\n%s", sourceFile, stateYaml) + } + return nil }) @@ -410,8 +473,14 @@ func (a *App) Fetch(c FetchConfigProvider) error { errs = append(errs, prepErr) } - return + return ok, errs }, false, SetFilter(true)) + + if err == nil && c.WriteOutput() { + fmt.Print(yamlOutput.String()) + } + + return err } func (a *App) Sync(c SyncConfigProvider) error { diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index f979544c..bcc996f3 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -2478,6 +2478,10 @@ func (c configImpl) EnforceNeedsAreInstalled() bool { return c.enforceNeedsAreInstalled } +func (c configImpl) WriteOutput() bool { + return false +} + type applyConfig struct { args string cascade string @@ -2805,8 +2809,9 @@ func MockExecer(logger *zap.SugaredLogger, kubeContext string) (helmexec.Interfa // mocking helmexec.Interface type mockHelmExec struct { - templated []mockTemplates - repos []mockRepo + templated []mockTemplates + repos []mockRepo + enableLiveOutput bool } type mockTemplates struct { @@ -2846,6 +2851,7 @@ func (helm *mockHelmExec) SetHelmBinary(bin string) { } func (helm *mockHelmExec) SetEnableLiveOutput(enableLiveOutput bool) { + helm.enableLiveOutput = enableLiveOutput } func (helm *mockHelmExec) SetDisableForceUpdate(forceUpdate bool) { @@ -4587,6 +4593,168 @@ releases: "state should contain source helmfile name:\n%s\n", out) } +type fetchConfigImpl struct { + configImpl + outputDir string + outputDirTemplate string + writeOutput bool +} + +func (f fetchConfigImpl) OutputDir() string { + return f.outputDir +} + +func (f fetchConfigImpl) OutputDirTemplate() string { + return f.outputDirTemplate +} + +func (f fetchConfigImpl) WriteOutput() bool { + return f.writeOutput +} + +func TestFetch_WriteOutputRequiresOutputDir(t *testing.T) { + files := map[string]string{ + "/path/to/helmfile.yaml": ` +releases: +- name: myrelease1 + chart: mychart1 +`, + } + + var buffer bytes.Buffer + syncWriter := testhelper.NewSyncWriter(&buffer) + logger := helmexec.NewLogger(syncWriter, "debug") + + app := appWithFs(&App{ + OverrideHelmBinary: DefaultHelmBinary, + fs: ffs.DefaultFileSystem(), + OverrideKubeContext: "default", + DisableKubeVersionAutoDetection: true, + Env: "default", + Logger: logger, + Namespace: "testNamespace", + }, files) + + expectNoCallsToHelm(app) + + err := app.Fetch(fetchConfigImpl{ + writeOutput: true, + outputDir: "", + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "--output-dir is required") +} + +func TestFetch_WriteOutput_ErrorsOnMultipleStateFiles(t *testing.T) { + // Two separate helmfile state files in a helmfile.d directory simulate the + // multi-file scenario that --write-output cannot safely handle: the resulting + // multi-document YAML stream would be merged by Helmfile in a way that can + // alter semantics (helmDefaults override, broken relative paths, etc.). + files := map[string]string{ + "/path/to/helmfile.d/first.yaml": ` +releases: +- name: release1 + chart: chart1 +`, + "/path/to/helmfile.d/second.yaml": ` +releases: +- name: release2 + chart: chart2 +`, + } + + var buffer bytes.Buffer + syncWriter := testhelper.NewSyncWriter(&buffer) + logger := helmexec.NewLogger(syncWriter, "debug") + + valsRuntime, err := vals.New(vals.Options{CacheSize: 32}) + if err != nil { + t.Fatalf("unexpected error creating vals runtime: %v", err) + } + + helm := &mockHelmExec{} + + app := appWithFs(&App{ + OverrideHelmBinary: DefaultHelmBinary, + fs: ffs.DefaultFileSystem(), + OverrideKubeContext: "default", + DisableKubeVersionAutoDetection: true, + Env: "default", + Logger: logger, + helms: map[helmKey]helmexec.Interface{ + createHelmKey(DefaultHelmBinary, "default"): helm, + }, + Namespace: "testNamespace", + valsRuntime: valsRuntime, + }, files) + + outputDir := t.TempDir() + + fetchErr := app.Fetch(fetchConfigImpl{ + writeOutput: true, + outputDir: outputDir, + }) + assert.Error(t, fetchErr, "expected error when --write-output is used with multiple state files") + assert.Contains(t, fetchErr.Error(), "--write-output requires a single helmfile state file") +} + +func TestFetch_WriteOutputRestoresSequentialHelmfiles(t *testing.T) { + files := map[string]string{ + "/path/to/helmfile.yaml": ` +releases: +- name: myrelease1 + chart: mychart1 +`, + } + + var buffer bytes.Buffer + syncWriter := testhelper.NewSyncWriter(&buffer) + logger := helmexec.NewLogger(syncWriter, "debug") + + valsRuntime, err := vals.New(vals.Options{CacheSize: 32}) + if err != nil { + t.Fatalf("unexpected error creating vals runtime: %v", err) + } + + // Use a real mock helm exec (not noCallHelmExec) so that Fetch can proceed + // past the validation check and enter the SequentialHelmfiles mutation block. + // Start with enableLiveOutput = true so the restore path is actually exercised: + // Fetch will call SetEnableLiveOutput(false), then the deferred restore call + // SetEnableLiveOutput(true) (a.EnableLiveOutput). If the defer were missing, + // helm.enableLiveOutput would remain false and the assertion below would fail. + helm := &mockHelmExec{enableLiveOutput: true} + + app := appWithFs(&App{ + OverrideHelmBinary: DefaultHelmBinary, + fs: ffs.DefaultFileSystem(), + OverrideKubeContext: "default", + DisableKubeVersionAutoDetection: true, + Env: "default", + Logger: logger, + helms: map[helmKey]helmexec.Interface{ + createHelmKey(DefaultHelmBinary, "default"): helm, + }, + Namespace: "testNamespace", + valsRuntime: valsRuntime, + // Start with SequentialHelmfiles = false; it must be restored after Fetch. + SequentialHelmfiles: false, + // Start with EnableLiveOutput = true; the deferred restore must bring it back. + EnableLiveOutput: true, + }, files) + + outputDir := t.TempDir() + + // Fetch with --write-output + --output-dir enters the mutation block, + // temporarily sets SequentialHelmfiles = true and helm.EnableLiveOutput = false, + // then restores both when it returns. + _ = app.Fetch(fetchConfigImpl{ + writeOutput: true, + outputDir: outputDir, + }) + assert.False(t, app.SequentialHelmfiles, "SequentialHelmfiles should be restored to false after Fetch returns") + assert.True(t, helm.enableLiveOutput, "helm.enableLiveOutput should be restored to true (a.EnableLiveOutput) after Fetch returns") +} + func TestList(t *testing.T) { files := map[string]string{ "/path/to/helmfile.d/first.yaml": ` diff --git a/pkg/app/config.go b/pkg/app/config.go index 9b92e498..8ff2a5cb 100644 --- a/pkg/app/config.go +++ b/pkg/app/config.go @@ -242,6 +242,7 @@ type FetchConfigProvider interface { SkipRefresh() bool OutputDir() string OutputDirTemplate() string + WriteOutput() bool concurrencyConfig } diff --git a/pkg/app/run.go b/pkg/app/run.go index 91e25bb5..4c124df2 100644 --- a/pkg/app/run.go +++ b/pkg/app/run.go @@ -88,7 +88,7 @@ func (r *Run) withPreparedCharts(helmfileCommand string, opts state.ChartPrepare dir = tempDir } else { dir = opts.OutputDir - fmt.Printf("Charts will be downloaded to: %s\n", dir) + fmt.Fprintf(os.Stderr, "Charts will be downloaded to: %s\n", dir) } if _, err := r.state.TriggerGlobalPrepareEvent(helmfileCommand); err != nil { diff --git a/pkg/config/fetch.go b/pkg/config/fetch.go index 462ecdf4..64a53eb4 100644 --- a/pkg/config/fetch.go +++ b/pkg/config/fetch.go @@ -1,6 +1,6 @@ package config -// FetchOptions is the options for the build command +// FetchOptions is the options for the fetch command type FetchOptions struct { // Concurrency is the maximum number of concurrent helm processes to run, 0 is unlimited Concurrency int @@ -8,14 +8,16 @@ type FetchOptions struct { OutputDir string // OutputDirTemplate is the go template to generate the path of output directory OutputDirTemplate string + // WriteOutput writes a helmfile.yaml with chart references updated to point to downloaded local chart paths + WriteOutput bool } -// NewFetchOptions creates a new Apply +// NewFetchOptions creates a new FetchOptions func NewFetchOptions() *FetchOptions { return &FetchOptions{} } -// FetchImpl is impl for applyOptions +// FetchImpl is impl for fetchOptions type FetchImpl struct { *GlobalImpl *FetchOptions @@ -43,3 +45,8 @@ func (c *FetchImpl) OutputDir() string { func (c *FetchImpl) OutputDirTemplate() string { return c.FetchOptions.OutputDirTemplate } + +// WriteOutput returns whether to write a modified helmfile.yaml with local chart paths +func (c *FetchImpl) WriteOutput() bool { + return c.FetchOptions.WriteOutput +} diff --git a/test/integration/run.sh b/test/integration/run.sh index 14d4b435..e6b491ec 100755 --- a/test/integration/run.sh +++ b/test/integration/run.sh @@ -99,6 +99,7 @@ ${kubectl} create namespace ${test_ns} || fail "Could not create namespace ${tes . ${dir}/test-cases/issue-2502-race-condition-local-chart.sh . ${dir}/test-cases/chart-deps-condition.sh . ${dir}/test-cases/fetch-forl-local-chart.sh +. ${dir}/test-cases/fetch-write-output.sh . ${dir}/test-cases/suppress-output-line-regex.sh . ${dir}/test-cases/chartify-jsonPatches-and-strategicMergePatches.sh . ${dir}/test-cases/include-template-func.sh diff --git a/test/integration/test-cases/fetch-write-output.sh b/test/integration/test-cases/fetch-write-output.sh new file mode 100644 index 00000000..53e39321 --- /dev/null +++ b/test/integration/test-cases/fetch-write-output.sh @@ -0,0 +1,39 @@ +fetch_write_output_input_dir="${cases_dir}/fetch-write-output/input" + +fetch_write_output_tmp=$(mktemp -d) + +case_title="fetch with --write-output for air-gapped environments" + +test_start "$case_title" + +info "Testing helmfile fetch --write-output with local chart" +output=$(${helmfile} -f "${fetch_write_output_input_dir}/helmfile.yaml.gotmpl" fetch --output-dir "${fetch_write_output_tmp}" --write-output 2>/dev/null) \ + || fail "\"helmfile fetch --write-output\" shouldn't fail" + +info "Verifying stdout does not contain non-YAML status messages" +echo "${output}" | grep -q "^Charts will be downloaded to:" && fail "stdout should not contain 'Charts will be downloaded to:' (should be on stderr)" || true + +info "Verifying output contains YAML document separator" +echo "${output}" | grep -q "^---" || fail "output should contain YAML document separator" + +info "Verifying output contains source helmfile reference" +echo "${output}" | grep -q "# Source:" || fail "output should contain source helmfile reference" + +info "Verifying output contains release name" +echo "${output}" | grep -q "name: local-chart" || fail "output should contain release name" + +info "Verifying output contains updated chart path pointing to output dir" +echo "${output}" | grep -q "chart:" || fail "output should contain chart field" + +info "Verifying chart files exist in output directory" +cat "${fetch_write_output_tmp}/helmfile-tests/local-chart/raw/latest/Chart.yaml" || fail "Chart.yaml should exist in fetched output directory" + +info "Verifying the chart path in output matches the actual downloaded location" +chart_path=$(echo "${output}" | grep -E "^[[:space:]]+(-[[:space:]]+)?chart:" | head -1 | sed 's/.*chart: *//' | tr -d '"') +if [ ! -f "${chart_path}/Chart.yaml" ]; then + fail "chart path '${chart_path}' from output should point to a directory containing Chart.yaml" +fi + +rm -rf "${fetch_write_output_tmp}" + +test_pass "$case_title" diff --git a/test/integration/test-cases/fetch-write-output/input/helmfile.yaml.gotmpl b/test/integration/test-cases/fetch-write-output/input/helmfile.yaml.gotmpl new file mode 100644 index 00000000..f4561b44 --- /dev/null +++ b/test/integration/test-cases/fetch-write-output/input/helmfile.yaml.gotmpl @@ -0,0 +1,4 @@ +releases: +- name: local-chart + chart: ../../../charts/raw + namespace: local-chart