mirror of
https://github.com/helmfile/helmfile.git
synced 2026-09-30 04:02:49 +02:00
fix: skip failed-prep releases, complete flag wiring, add tests and docs
Review follow-ups for --allow-failed-releases (#2616): - Track per-release chart preparation failures in PrepareCharts (returned as a map keyed by release) and remove those releases from the state in Run.WithPreparedCharts when --allow-failed-releases is set, so a failed release is never executed against its original, un-prepared chart reference (which could either fail again with a duplicate error or, for charts requiring chartify, bypass patches/dependency modifications and produce an unintended result). All failures are still reported at the end via the aggregated MultiError. - Complete the release identity on error results from prepareChartForRelease so failures are attributed to the correct release. - With --allow-failed-releases, continue building dependencies of the remaining charts when 'helm dep build' fails for one release, and skip the affected releases during execution. - Wire --allow-failed-releases into 'helmfile unittest' and 'helmfile status'; remove the dead flag wiring for write-values and list (both never prepare charts, see commandsSkipChartPrep). - Simplify control flow (guard clauses, errors.As, drop dead code and redundant else branches). - Add end-to-end coverage in pkg/app/issue_2616_test.go and extend the state-level tests; document the flag in docs/cli.md and CHANGELOG.md. Signed-off-by: yxxhero <11087727+yxxhero@users.noreply.github.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
### Added
|
||||
|
||||
- Add support for `conditionTemplate` and allow `condition` to be set directly to `true` or `false`.
|
||||
- Add `--allow-failed-releases` global flag to continue preparing charts for the remaining releases when chart preparation fails for a release; failed releases are skipped and all failures are reported at the end (#2616)
|
||||
|
||||
## [1.4.1] - 2026-03-03
|
||||
|
||||
|
||||
+1
-1
@@ -131,7 +131,7 @@ func setGlobalOptionsForRootCmd(fs *pflag.FlagSet, globalOptions *config.GlobalO
|
||||
fs.StringArrayVar(&globalOptions.StateValuesFile, "state-values-file", nil, "specify state values in a YAML file. Used to override .Values within the helmfile template (not values template).")
|
||||
fs.BoolVar(&globalOptions.SkipDeps, "skip-deps", false, `skip running "helm repo update" and "helm dependency build"`)
|
||||
fs.BoolVar(&globalOptions.SkipRefresh, "skip-refresh", false, `skip running "helm repo update"`)
|
||||
fs.BoolVar(&globalOptions.AllowFailedReleases, "allow-failed-releases", false, `continue preparing charts for other releases when chart preparation fails for a release; report failures at the end`)
|
||||
fs.BoolVar(&globalOptions.AllowFailedReleases, "allow-failed-releases", false, `continue preparing charts for other releases when chart preparation fails for a release; failed releases are skipped and all failures are reported at the end`)
|
||||
fs.BoolVar(&globalOptions.StripArgsValuesOnExitError, "strip-args-values-on-exit-error", true, `Strip the potential secret values of the helm command args contained in a helmfile error message`)
|
||||
fs.BoolVar(&globalOptions.DisableForceUpdate, "disable-force-update", false, `do not force helm repos to update when executing "helm repo add" (Helm 3 only)`)
|
||||
fs.BoolVar(&globalOptions.EnforcePluginVerification, "enforce-plugin-verification", false, `fail plugin installation if verification is not supported (for security purposes)`)
|
||||
|
||||
@@ -550,6 +550,7 @@ The following global flags are also available but not shown in the main help out
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--kubeconfig` | `""` | Use a particular kubeconfig file |
|
||||
| `--allow-failed-releases` | false | Continue preparing charts for other releases when chart preparation fails for a release; failed releases are skipped and all failures are reported at the end |
|
||||
| `--skip-refresh` | false | Skip running `helm repo update` (lighter than `--skip-deps` which also skips dependency build) |
|
||||
| `--enforce-plugin-verification` | false | Fail plugin installation if verification is not supported |
|
||||
| `--oci-plain-http` | false | Use plain HTTP for OCI registries (required for local/insecure registries in Helm 4) |
|
||||
|
||||
+23
-21
@@ -276,12 +276,13 @@ func (a *App) Template(c TemplateConfigProvider) error {
|
||||
func (a *App) WriteValues(c WriteValuesConfigProvider) error {
|
||||
return a.ForEachState(func(run *Run) (ok bool, errs []error) {
|
||||
prepErr := run.WithPreparedCharts("write-values", state.ChartPrepareOptions{
|
||||
SkipRepos: c.SkipRefresh() || c.SkipDeps(),
|
||||
SkipRefresh: c.SkipRefresh(),
|
||||
AllowFailedReleases: c.AllowFailedReleases(),
|
||||
SkipDeps: c.SkipDeps(),
|
||||
SkipCleanup: c.SkipCleanup(),
|
||||
Concurrency: c.Concurrency(),
|
||||
// Note: "write-values" never prepares charts (see commandsSkipChartPrep
|
||||
// in run.go), so AllowFailedReleases does not apply here.
|
||||
SkipRepos: c.SkipRefresh() || c.SkipDeps(),
|
||||
SkipRefresh: c.SkipRefresh(),
|
||||
SkipDeps: c.SkipDeps(),
|
||||
SkipCleanup: c.SkipCleanup(),
|
||||
Concurrency: c.Concurrency(),
|
||||
}, func() []error {
|
||||
ok, errs = a.writeValues(run, c)
|
||||
return errs
|
||||
@@ -375,6 +376,7 @@ func (a *App) Unittest(c UnittestConfigProvider) error {
|
||||
ForceDownload: true,
|
||||
SkipRepos: c.SkipRefresh() || c.SkipDeps(),
|
||||
SkipRefresh: c.SkipRefresh(),
|
||||
AllowFailedReleases: c.AllowFailedReleases(),
|
||||
SkipDeps: c.SkipDeps(),
|
||||
SkipCleanup: c.SkipCleanup(),
|
||||
Concurrency: c.Concurrency(),
|
||||
@@ -615,9 +617,10 @@ func (a *App) Apply(c ApplyConfigProvider) error {
|
||||
func (a *App) Status(c StatusesConfigProvider) error {
|
||||
return a.ForEachState(func(run *Run) (ok bool, errs []error) {
|
||||
err := run.WithPreparedCharts("status", state.ChartPrepareOptions{
|
||||
SkipRepos: true,
|
||||
SkipDeps: true,
|
||||
Concurrency: c.Concurrency(),
|
||||
SkipRepos: true,
|
||||
AllowFailedReleases: c.AllowFailedReleases(),
|
||||
SkipDeps: true,
|
||||
Concurrency: c.Concurrency(),
|
||||
}, func() []error {
|
||||
ok, errs = a.status(run, c)
|
||||
return errs
|
||||
@@ -687,10 +690,9 @@ func (a *App) PrintDAGState(c DAGConfigProvider) error {
|
||||
var err error
|
||||
return a.ForEachState(func(run *Run) (ok bool, errs []error) {
|
||||
err = run.WithPreparedCharts("show-dag", state.ChartPrepareOptions{
|
||||
SkipRepos: true,
|
||||
AllowFailedReleases: false,
|
||||
SkipDeps: true,
|
||||
Concurrency: 2,
|
||||
SkipRepos: true,
|
||||
SkipDeps: true,
|
||||
Concurrency: 2,
|
||||
}, func() []error {
|
||||
err = a.dag(run)
|
||||
if err != nil {
|
||||
@@ -705,10 +707,9 @@ func (a *App) PrintDAGState(c DAGConfigProvider) error {
|
||||
func (a *App) PrintState(c StateConfigProvider) error {
|
||||
return a.ForEachState(func(run *Run) (_ bool, errs []error) {
|
||||
err := run.WithPreparedCharts("build", state.ChartPrepareOptions{
|
||||
SkipRepos: true,
|
||||
AllowFailedReleases: false,
|
||||
SkipDeps: true,
|
||||
Concurrency: 2,
|
||||
SkipRepos: true,
|
||||
SkipDeps: true,
|
||||
Concurrency: 2,
|
||||
}, func() []error {
|
||||
if c.EmbedValues() {
|
||||
for i := range run.state.Releases {
|
||||
@@ -779,10 +780,11 @@ func (a *App) ListReleases(c ListConfigProvider) error {
|
||||
|
||||
if !c.SkipCharts() {
|
||||
prepErr := run.WithPreparedCharts("list", state.ChartPrepareOptions{
|
||||
SkipRepos: true,
|
||||
AllowFailedReleases: true,
|
||||
SkipDeps: true,
|
||||
Concurrency: 2,
|
||||
// Note: "list" never prepares charts (see commandsSkipChartPrep in
|
||||
// run.go), so AllowFailedReleases does not apply here.
|
||||
SkipRepos: true,
|
||||
SkipDeps: true,
|
||||
Concurrency: 2,
|
||||
}, func() []error {
|
||||
rel, err := a.list(run)
|
||||
if err != nil {
|
||||
|
||||
+1
-1
@@ -343,7 +343,6 @@ type WriteValuesConfigProvider interface {
|
||||
OutputFileTemplate() string
|
||||
SkipDeps() bool
|
||||
SkipRefresh() bool
|
||||
AllowFailedReleases() bool
|
||||
SkipCleanup() bool
|
||||
IncludeTransitiveNeeds() bool
|
||||
|
||||
@@ -352,6 +351,7 @@ type WriteValuesConfigProvider interface {
|
||||
|
||||
type StatusesConfigProvider interface {
|
||||
Args() string
|
||||
AllowFailedReleases() bool
|
||||
|
||||
concurrencyConfig
|
||||
}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/helmfile/vals"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/helmfile/helmfile/pkg/exectest"
|
||||
ffs "github.com/helmfile/helmfile/pkg/filesystem"
|
||||
"github.com/helmfile/helmfile/pkg/helmexec"
|
||||
)
|
||||
|
||||
// issue2616HelmfileContent defines two releases: one that is templated fine
|
||||
// (logging) and one that always fails chart preparation (error), because OCI
|
||||
// charts do not support the "latest" version tag.
|
||||
const issue2616HelmfileContent = `
|
||||
releases:
|
||||
- name: logging
|
||||
chart: incubator/raw
|
||||
namespace: kube-system
|
||||
|
||||
- name: error
|
||||
chart: oci://example.com/chart/error
|
||||
namespace: failed
|
||||
version: latest
|
||||
`
|
||||
|
||||
// TestTemplateAllowFailedReleases verifies the behavior of the
|
||||
// --allow-failed-releases flag end to end for `helmfile template`:
|
||||
// when chart preparation fails for one release, the remaining releases are
|
||||
// still templated, the failed release is skipped (i.e. never executed against
|
||||
// its un-prepared chart reference) and all failures are reported at the end.
|
||||
func TestTemplateAllowFailedReleases(t *testing.T) {
|
||||
testcases := []struct {
|
||||
name string
|
||||
allowFailedRelease bool
|
||||
|
||||
// wantTemplated contains the releases that must have been passed to
|
||||
// `helm template`; the release failing chart preparation must never
|
||||
// appear here.
|
||||
wantTemplated []exectest.Release
|
||||
}{
|
||||
{
|
||||
name: "default: abort on chart preparation failure",
|
||||
allowFailedRelease: false,
|
||||
wantTemplated: nil,
|
||||
},
|
||||
{
|
||||
name: "allow-failed-releases: skip failed release, template the rest",
|
||||
allowFailedRelease: true,
|
||||
wantTemplated: []exectest.Release{{Name: "logging", Flags: []string{"--kube-context", "default", "--namespace", "kube-system"}}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testcases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var helm = &exectest.Helm{
|
||||
FailOnUnexpectedList: true,
|
||||
FailOnUnexpectedDiff: true,
|
||||
DiffMutex: &sync.Mutex{},
|
||||
ChartsMutex: &sync.Mutex{},
|
||||
ReleasesMutex: &sync.Mutex{},
|
||||
}
|
||||
|
||||
_ = runWithLogCapture(t, "debug", func(t *testing.T, logger *zap.SugaredLogger) {
|
||||
t.Helper()
|
||||
|
||||
valsRuntime, err := vals.New(vals.Options{CacheSize: 32})
|
||||
require.NoError(t, err)
|
||||
|
||||
files := map[string]string{
|
||||
"/path/to/helmfile.yaml": issue2616HelmfileContent,
|
||||
}
|
||||
|
||||
app := appWithFs(&App{
|
||||
OverrideHelmBinary: DefaultHelmBinary,
|
||||
fs: &ffs.FileSystem{Glob: filepath.Glob},
|
||||
OverrideKubeContext: "default",
|
||||
DisableKubeVersionAutoDetection: true,
|
||||
Env: "default",
|
||||
Logger: logger,
|
||||
helms: map[helmKey]helmexec.Interface{
|
||||
createHelmKey("helm", "default"): helm,
|
||||
},
|
||||
valsRuntime: valsRuntime,
|
||||
}, files)
|
||||
|
||||
tmplErr := app.Template(applyConfig{
|
||||
// if we check log output, concurrency must be 1. otherwise the test becomes non-deterministic.
|
||||
concurrency: 1,
|
||||
logger: logger,
|
||||
allowFailedReleases: tc.allowFailedRelease,
|
||||
})
|
||||
|
||||
// The chart preparation failure must be reported in both modes.
|
||||
require.Error(t, tmplErr)
|
||||
assert.Contains(t, tmplErr.Error(), "the version for OCI charts should be semver compliant")
|
||||
|
||||
// The release that failed chart preparation must never be executed.
|
||||
require.Equal(t, tc.wantTemplated, helm.Templated)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
+49
-34
@@ -1,6 +1,7 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
@@ -48,23 +49,23 @@ func (r *Run) askForConfirmation(msg string) bool {
|
||||
// When skipRepos is false, SyncReposOnce still runs normally for all repos.
|
||||
var commandsSkipChartPrep = []string{"write-values", "list"}
|
||||
|
||||
func (r *Run) prepareChartsIfNeeded(helmfileCommand string, dir string, concurrency int, opts state.ChartPrepareOptions) (map[state.PrepareChartKey]string, error) {
|
||||
func (r *Run) prepareChartsIfNeeded(helmfileCommand string, dir string, concurrency int, opts state.ChartPrepareOptions) (map[state.PrepareChartKey]string, map[state.PrepareChartKey]error, error) {
|
||||
// Skip chart preparation for commands that don't need chart pulls
|
||||
if slices.Contains(commandsSkipChartPrep, strings.ToLower(helmfileCommand)) {
|
||||
return nil, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
releaseToChart, errs := r.state.PrepareCharts(r.helm, dir, concurrency, helmfileCommand, opts)
|
||||
releaseToChart, failedReleases, errs := r.state.PrepareCharts(r.helm, dir, concurrency, helmfileCommand, opts)
|
||||
if len(errs) > 0 {
|
||||
if !opts.AllowFailedReleases {
|
||||
// abort on first error
|
||||
return nil, fmt.Errorf("%v", errs)
|
||||
return nil, nil, fmt.Errorf("%v", errs)
|
||||
}
|
||||
// return partial results with errors for the failed ones
|
||||
return releaseToChart, &MultiError{Errors: errs}
|
||||
// return partial results, along with the per-release errors for the failed ones
|
||||
return releaseToChart, failedReleases, &MultiError{Errors: errs}
|
||||
}
|
||||
|
||||
return releaseToChart, nil
|
||||
return releaseToChart, failedReleases, nil
|
||||
}
|
||||
|
||||
func (r *Run) WithPreparedCharts(helmfileCommand string, opts state.ChartPrepareOptions, f func() []error) error {
|
||||
@@ -118,18 +119,25 @@ func (r *Run) WithPreparedCharts(helmfileCommand string, opts state.ChartPrepare
|
||||
// remain available for the entire operation lifecycle. See issue #1799.
|
||||
defer r.state.CleanupChartifyTempDirs()
|
||||
|
||||
releaseToChart, prepareErr := r.prepareChartsIfNeeded(helmfileCommand, dir, opts.Concurrency, opts)
|
||||
// IMPORTANT: on opts.AllowFailedReleases: do not abort only on error here, just forward it to the caller in order to allow for partial results
|
||||
// Only in case prepareCharts failed with a general error and returned to processable release abort anyways
|
||||
if prepareErr != nil && releaseToChart == nil {
|
||||
releaseToChart, failedReleases, prepareErr := r.prepareChartsIfNeeded(helmfileCommand, dir, opts.Concurrency, opts)
|
||||
// A prepare error with no per-release attribution (state parsing, dependency
|
||||
// resolution, ...) is a general failure: always abort, even with
|
||||
// opts.AllowFailedReleases, since there are no processable partial results.
|
||||
// Otherwise, abort only when partial results are not allowed; the failed
|
||||
// releases are skipped below and their errors are reported at the end.
|
||||
if prepareErr != nil && (len(failedReleases) == 0 || !opts.AllowFailedReleases) {
|
||||
return prepareErr
|
||||
}
|
||||
if !opts.AllowFailedReleases {
|
||||
if prepareErr != nil {
|
||||
return prepareErr
|
||||
}
|
||||
}
|
||||
|
||||
// Releases whose chart preparation failed are removed from the state, so that
|
||||
// the operation below never executes them against their original, un-prepared
|
||||
// chart reference. That would either fail again with a duplicate error or,
|
||||
// worse, bypass chartify modifications (patches, dependencies) and produce an
|
||||
// unintended result. Their preparation errors are reported via prepareErr.
|
||||
releases := r.state.Releases
|
||||
if len(failedReleases) > 0 {
|
||||
releases = make([]state.ReleaseSpec, 0, len(r.state.Releases))
|
||||
}
|
||||
for i := range r.state.Releases {
|
||||
rel := &r.state.Releases[i]
|
||||
key := state.PrepareChartKey{
|
||||
@@ -137,6 +145,9 @@ func (r *Run) WithPreparedCharts(helmfileCommand string, opts state.ChartPrepare
|
||||
Namespace: rel.Namespace,
|
||||
KubeContext: rel.KubeContext,
|
||||
}
|
||||
if _, failed := failedReleases[key]; failed {
|
||||
continue
|
||||
}
|
||||
if chart, ok := releaseToChart[key]; ok && chart != rel.Chart {
|
||||
// The chart has been downloaded and modified by Helmfile (and chartify under the hood).
|
||||
// We let the later step use the modified version of the chart, located under the `chart` variable,
|
||||
@@ -145,7 +156,11 @@ func (r *Run) WithPreparedCharts(helmfileCommand string, opts state.ChartPrepare
|
||||
// if it has been modified or not.
|
||||
rel.ChartPath = chart
|
||||
}
|
||||
if len(failedReleases) > 0 {
|
||||
releases = append(releases, *rel)
|
||||
}
|
||||
}
|
||||
r.state.Releases = releases
|
||||
|
||||
r.ReleaseToChart = releaseToChart
|
||||
|
||||
@@ -160,26 +175,26 @@ func (r *Run) WithPreparedCharts(helmfileCommand string, opts state.ChartPrepare
|
||||
|
||||
_, cleanupErr := r.state.TriggerGlobalCleanupEvent(helmfileCommand, firstErr)
|
||||
if !opts.AllowFailedReleases {
|
||||
// return directly on first error
|
||||
return cleanupErr
|
||||
} else {
|
||||
// merge the two errors into a single error output
|
||||
var merged []error
|
||||
if prepareErr != nil {
|
||||
if me, ok := prepareErr.(*MultiError); ok {
|
||||
merged = append(merged, me.Errors...)
|
||||
} else {
|
||||
merged = append(merged, prepareErr)
|
||||
}
|
||||
}
|
||||
if cleanupErr != nil {
|
||||
merged = append(merged, fmt.Errorf("error during global cleanup event: %w", cleanupErr))
|
||||
}
|
||||
if len(merged) > 0 {
|
||||
return &MultiError{Errors: merged}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// merge the preparation and cleanup errors into a single error output
|
||||
var merged []error
|
||||
if prepareErr != nil {
|
||||
var me *MultiError
|
||||
if errors.As(prepareErr, &me) {
|
||||
merged = append(merged, me.Errors...)
|
||||
} else {
|
||||
merged = append(merged, prepareErr)
|
||||
}
|
||||
}
|
||||
if cleanupErr != nil {
|
||||
merged = append(merged, fmt.Errorf("error during global cleanup event: %w", cleanupErr))
|
||||
}
|
||||
if len(merged) > 0 {
|
||||
return &MultiError{Errors: merged}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Run) Deps(c DepsConfigProvider) []error {
|
||||
|
||||
@@ -86,7 +86,7 @@ releases:
|
||||
OutputDirTemplate: "{{ .OutputDir }}/{{ .Release.Name }}",
|
||||
}
|
||||
|
||||
releaseToChart, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
|
||||
releaseToChart, _, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
|
||||
require.Empty(t, errs, "PrepareCharts should not return errors")
|
||||
|
||||
// Verify both releases have prepared charts
|
||||
@@ -152,7 +152,7 @@ releases:
|
||||
OutputDirTemplate: "{{ .OutputDir }}/{{ .Release.Name }}",
|
||||
}
|
||||
|
||||
releaseToChart, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
|
||||
releaseToChart, _, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
|
||||
require.Empty(t, errs)
|
||||
|
||||
// argocd-secrets should NOT have a prepared chart
|
||||
|
||||
@@ -11,13 +11,10 @@ import (
|
||||
"github.com/helmfile/helmfile/pkg/exectest"
|
||||
)
|
||||
|
||||
// TestAllowFailedReleasesFlag_Enabled verifies that the AllowFailedReleases flag
|
||||
// works as intended with one release that should work and another release, that
|
||||
// will fail.
|
||||
// The test makes one release fail by having an oci with latest version, which will always error
|
||||
func TestAllowFailedReleasesFlag_Enabled(t *testing.T) {
|
||||
resetChartCacheForTest()
|
||||
helmfileContent := []byte(`
|
||||
// issue2616HelmfileContent defines two releases: one that prepares fine
|
||||
// (grafana) and one that always fails chart preparation (error), because OCI
|
||||
// charts do not support the "latest" version tag.
|
||||
var issue2616HelmfileContent = []byte(`
|
||||
repositories:
|
||||
- name: stable
|
||||
url: kubernetes-charts.storage.googleapis.com
|
||||
@@ -33,9 +30,15 @@ releases:
|
||||
version: latest
|
||||
`)
|
||||
|
||||
// issue2616PrepareCharts runs PrepareCharts for issue2616HelmfileContent.
|
||||
func issue2616PrepareCharts(t *testing.T, opts ChartPrepareOptions) (map[PrepareChartKey]string, map[PrepareChartKey]error, []error) {
|
||||
t.Helper()
|
||||
|
||||
resetChartCacheForTest()
|
||||
|
||||
logger := zap.NewExample().Sugar()
|
||||
|
||||
st, err := createFromYaml(helmfileContent, "example/path/to/helmfile.yaml", DefaultEnv, logger)
|
||||
st, err := createFromYaml(issue2616HelmfileContent, "example/path/to/helmfile.yaml", DefaultEnv, logger)
|
||||
require.NoError(t, err)
|
||||
|
||||
tempDir := t.TempDir()
|
||||
@@ -46,13 +49,23 @@ releases:
|
||||
|
||||
// OutputDirTemplate includes .OutputDir so charts go into tempDir (auto-cleaned
|
||||
// by t.TempDir), not the global cache or CWD.
|
||||
opts := ChartPrepareOptions{
|
||||
opts.OutputDirTemplate = "{{ .OutputDir }}/{{ .Release.Name }}"
|
||||
|
||||
releaseToChart, failedReleases, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
|
||||
return releaseToChart, failedReleases, errs
|
||||
}
|
||||
|
||||
// TestAllowFailedReleasesFlag_Enabled verifies that the AllowFailedReleases flag
|
||||
// works as intended with one release that should work and another release, that
|
||||
// will fail: the healthy release is prepared and reported in the partial
|
||||
// results, while the failing one is recorded in failedReleases so that callers
|
||||
// can skip it during execution.
|
||||
func TestAllowFailedReleasesFlag_Enabled(t *testing.T) {
|
||||
releaseToChart, failedReleases, errs := issue2616PrepareCharts(t, ChartPrepareOptions{
|
||||
Concurrency: 1,
|
||||
OutputDirTemplate: "{{ .OutputDir }}/{{ .Release.Name }}",
|
||||
AllowFailedReleases: true,
|
||||
}
|
||||
})
|
||||
|
||||
releaseToChart, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
|
||||
require.NotEmpty(t, errs, "PrepareCharts should return an error")
|
||||
|
||||
// Verify only the error chart is not prepared
|
||||
@@ -60,55 +73,26 @@ releases:
|
||||
"grafana chart should be prepared")
|
||||
assert.NotContains(t, releaseToChart, PrepareChartKey{Name: "error", Namespace: "failed"},
|
||||
"error chart should not be prepared")
|
||||
|
||||
// Verify the failed release is attributed per-release, so that the caller
|
||||
// can skip it instead of executing it against the un-prepared chart
|
||||
assert.NotContains(t, failedReleases, PrepareChartKey{Name: "grafana", Namespace: "grafana"},
|
||||
"grafana should not be marked as failed")
|
||||
failedKey := PrepareChartKey{Name: "error", Namespace: "failed"}
|
||||
assert.Contains(t, failedReleases, failedKey, "the failing release should be marked as failed")
|
||||
assert.Error(t, failedReleases[failedKey])
|
||||
}
|
||||
|
||||
// TestAllowFailedReleasesFlag_Disabled verifies that the AllowFailedReleases flag
|
||||
// has no unintended side effects and the default abort on error still works.
|
||||
// The test makes one release fail by having an oci with latest version, which will always error
|
||||
func TestAllowFailedReleasesFlag_Disabled(t *testing.T) {
|
||||
resetChartCacheForTest()
|
||||
cleanupLeakedChartDirs(t, "grafana", "error")
|
||||
helmfileContent := []byte(`
|
||||
repositories:
|
||||
- name: stable
|
||||
url: kubernetes-charts.storage.googleapis.com
|
||||
oci: true
|
||||
releaseToChart, failedReleases, errs := issue2616PrepareCharts(t, ChartPrepareOptions{
|
||||
Concurrency: 1,
|
||||
})
|
||||
|
||||
releases:
|
||||
- name: grafana
|
||||
namespace: grafana
|
||||
chart: stable/grafana
|
||||
- name: error
|
||||
namespace: failed
|
||||
chart: oci://example.com/chart/error
|
||||
version: latest
|
||||
`)
|
||||
|
||||
logger := zap.NewExample().Sugar()
|
||||
|
||||
st, err := createFromYaml(helmfileContent, "example/path/to/helmfile.yaml", DefaultEnv, logger)
|
||||
require.NoError(t, err)
|
||||
|
||||
tempDir := t.TempDir()
|
||||
|
||||
helm := &exectest.Helm{
|
||||
ChartsMutex: &sync.Mutex{},
|
||||
}
|
||||
|
||||
// OutputDirTemplate includes .OutputDir so charts go into tempDir (auto-cleaned
|
||||
// by t.TempDir), not the global cache or CWD.
|
||||
opts := ChartPrepareOptions{
|
||||
Concurrency: 1,
|
||||
OutputDirTemplate: "{{ .OutputDir }}/{{ .Release.Name }}",
|
||||
// PrefetchSharedRemoteCharts: true,
|
||||
}
|
||||
|
||||
releaseToChart, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
|
||||
require.NotEmpty(t, errs, "PrepareCharts should return an error")
|
||||
|
||||
// Verify both releases are not prepared
|
||||
assert.NotContains(t, releaseToChart, PrepareChartKey{Name: "grafana", Namespace: "grafana"},
|
||||
"grafana chart should not be prepared")
|
||||
assert.NotContains(t, releaseToChart, PrepareChartKey{Name: "error", Namespace: "failed"},
|
||||
"error chart should not be prepared")
|
||||
// Verify no partial results are returned on the default abort-on-error path
|
||||
assert.Nil(t, releaseToChart)
|
||||
assert.Nil(t, failedReleases)
|
||||
}
|
||||
|
||||
@@ -116,7 +116,7 @@ func TestPrepareChartsPrefetchesSharedRemoteChart(t *testing.T) {
|
||||
OutputDirTemplate: "{{ .OutputDir }}/{{ .Release.Name }}",
|
||||
}
|
||||
|
||||
releaseToChart, errs := st.PrepareCharts(mockHelm, tempDir, 5, "sync", opts)
|
||||
releaseToChart, _, errs := st.PrepareCharts(mockHelm, tempDir, 5, "sync", opts)
|
||||
require.Empty(t, errs, "PrepareCharts should not return errors")
|
||||
|
||||
assert.Equal(t, int32(1), mockHelm.fetchCount.Load(),
|
||||
@@ -158,7 +158,7 @@ func TestPrepareChartsSkipsPrefetchForUniqueCharts(t *testing.T) {
|
||||
OutputDirTemplate: "{{ .OutputDir }}/{{ .Release.Name }}",
|
||||
}
|
||||
|
||||
releaseToChart, errs := st.PrepareCharts(mockHelm, tempDir, 3, "sync", opts)
|
||||
releaseToChart, _, errs := st.PrepareCharts(mockHelm, tempDir, 3, "sync", opts)
|
||||
require.Empty(t, errs, "PrepareCharts should not return errors")
|
||||
|
||||
assert.Equal(t, int32(0), mockHelm.fetchCount.Load(),
|
||||
@@ -199,7 +199,7 @@ func TestPrepareChartsSkipsPrefetchWhenFetchFlagsDiffer(t *testing.T) {
|
||||
OutputDirTemplate: "{{ .OutputDir }}/{{ .Release.Name }}",
|
||||
}
|
||||
|
||||
releaseToChart, errs := st.PrepareCharts(mockHelm, tempDir, 2, "sync", opts)
|
||||
releaseToChart, _, errs := st.PrepareCharts(mockHelm, tempDir, 2, "sync", opts)
|
||||
require.Empty(t, errs)
|
||||
|
||||
assert.Equal(t, int32(0), mockHelm.fetchCount.Load(),
|
||||
@@ -241,7 +241,7 @@ func TestPrepareChartsSkipsPrefetchWhenVerifyEnabled(t *testing.T) {
|
||||
OutputDirTemplate: "{{ .OutputDir }}/{{ .Release.Name }}",
|
||||
}
|
||||
|
||||
releaseToChart, errs := st.PrepareCharts(mockHelm, tempDir, 2, "sync", opts)
|
||||
releaseToChart, _, errs := st.PrepareCharts(mockHelm, tempDir, 2, "sync", opts)
|
||||
require.Empty(t, errs)
|
||||
|
||||
assert.Equal(t, int32(0), mockHelm.fetchCount.Load(),
|
||||
@@ -294,7 +294,7 @@ func TestPrepareChartsSkipsPrefetchForUnknownRepoChart(t *testing.T) {
|
||||
OutputDirTemplate: "{{ .OutputDir }}/{{ .Release.Name }}",
|
||||
}
|
||||
|
||||
releaseToChart, errs := st.PrepareCharts(mockHelm, tempDir, 2, "diff", opts)
|
||||
releaseToChart, _, errs := st.PrepareCharts(mockHelm, tempDir, 2, "diff", opts)
|
||||
require.Empty(t, errs)
|
||||
|
||||
assert.Equal(t, int32(0), mockHelm.fetchCount.Load(),
|
||||
@@ -367,7 +367,7 @@ func TestPrefetchedSharedChartAllowsConcurrentSyncRelease(t *testing.T) {
|
||||
OutputDirTemplate: "{{ .OutputDir }}/{{ .Release.Name }}",
|
||||
}
|
||||
|
||||
releaseToChart, errs := st.PrepareCharts(fetchHelm, tempDir, 5, "sync", opts)
|
||||
releaseToChart, _, errs := st.PrepareCharts(fetchHelm, tempDir, 5, "sync", opts)
|
||||
require.Empty(t, errs)
|
||||
require.Equal(t, int32(1), fetchHelm.fetchCount.Load())
|
||||
|
||||
@@ -422,7 +422,7 @@ func TestPrefetchedSharedChartAllowsConcurrentDiffRelease(t *testing.T) {
|
||||
OutputDirTemplate: "{{ .OutputDir }}/{{ .Release.Name }}",
|
||||
}
|
||||
|
||||
releaseToChart, errs := st.PrepareCharts(fetchHelm, tempDir, 5, "diff", opts)
|
||||
releaseToChart, _, errs := st.PrepareCharts(fetchHelm, tempDir, 5, "diff", opts)
|
||||
require.Empty(t, errs)
|
||||
require.Equal(t, int32(1), fetchHelm.fetchCount.Load())
|
||||
|
||||
|
||||
@@ -275,7 +275,7 @@ func TestRunHelmDepBuilds_SkipRefreshBehaviors(t *testing.T) {
|
||||
SkipRefresh: tt.optsSkipRefresh,
|
||||
}
|
||||
|
||||
err := st.runHelmDepBuilds(helm, 1, builds, opts)
|
||||
err := st.runHelmDepBuilds(helm, 1, builds, opts, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotNil(t, helm.buildDepsFlags, "BuildDeps should have been called")
|
||||
@@ -336,7 +336,7 @@ func TestRunHelmDepBuilds_MultipleBuilds(t *testing.T) {
|
||||
|
||||
opts := ChartPrepareOptions{SkipRefresh: false}
|
||||
|
||||
err := st.runHelmDepBuilds(helm, 1, builds, opts)
|
||||
err := st.runHelmDepBuilds(helm, 1, builds, opts, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, helm.updateRepoCalled, "UpdateRepo should have been called")
|
||||
|
||||
+64
-18
@@ -2208,22 +2208,24 @@ func (st *HelmState) prepareChartForRelease(release *ReleaseSpec, helm helmexec.
|
||||
}
|
||||
|
||||
// PrepareCharts downloads and prepares all charts for the selected releases.
|
||||
// Returns the chart paths and any errors encountered.
|
||||
// It returns the chart paths for the successfully prepared releases, the set of
|
||||
// releases that failed to prepare (keyed by release, with their errors), and any
|
||||
// errors encountered.
|
||||
//
|
||||
// Note: OCI chart locks are acquired and released during chart download within this function.
|
||||
// The tempDir cleanup is deferred until after helm operations complete in the caller,
|
||||
// so charts remain available during helm commands even though locks are released.
|
||||
func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurrency int, helmfileCommand string, opts ChartPrepareOptions) (map[PrepareChartKey]string, []error) {
|
||||
func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurrency int, helmfileCommand string, opts ChartPrepareOptions) (map[PrepareChartKey]string, map[PrepareChartKey]error, []error) {
|
||||
if !opts.SkipResolve {
|
||||
updated, err := st.ResolveDeps()
|
||||
if err != nil {
|
||||
return nil, []error{err}
|
||||
return nil, nil, []error{err}
|
||||
}
|
||||
*st = *updated
|
||||
}
|
||||
selected, err := st.GetSelectedReleases(opts.IncludeTransitiveNeeds)
|
||||
if err != nil {
|
||||
return nil, []error{err}
|
||||
return nil, nil, []error{err}
|
||||
}
|
||||
|
||||
releases := releasesNeedCharts(selected)
|
||||
@@ -2310,6 +2312,12 @@ func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurre
|
||||
|
||||
prepareChartInfo := make(map[PrepareChartKey]string, len(releases))
|
||||
|
||||
// Releases that failed to prepare. When opts.AllowFailedReleases is set,
|
||||
// these releases are excluded from the returned chart paths so that callers
|
||||
// can skip them instead of executing them against their un-prepared chart
|
||||
// references.
|
||||
failedReleases := make(map[PrepareChartKey]error)
|
||||
|
||||
errs := []error{}
|
||||
|
||||
jobQueue := make(chan *ReleaseSpec, len(releases))
|
||||
@@ -2333,6 +2341,14 @@ func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurre
|
||||
releaseOpts.ForceDownload = true
|
||||
}
|
||||
result := st.prepareChartForRelease(release, helm, dir, helmfileCommand, releaseOpts, workerIndex)
|
||||
if result.err != nil {
|
||||
// Error results returned by prepareChartForRelease may lack the
|
||||
// release identity. Complete it here, so that the failure can be
|
||||
// attributed to the correct release in failedReleases below.
|
||||
result.releaseName = release.Name
|
||||
result.releaseNamespace = release.Namespace
|
||||
result.releaseContext = release.KubeContext
|
||||
}
|
||||
results <- result
|
||||
}
|
||||
},
|
||||
@@ -2345,10 +2361,12 @@ func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurre
|
||||
|
||||
if !opts.AllowFailedReleases {
|
||||
return
|
||||
} else {
|
||||
// continue processing other releases even if one fails
|
||||
continue
|
||||
}
|
||||
|
||||
// Continue processing the other releases and record which one
|
||||
// failed, so that the caller can skip it during execution.
|
||||
failedReleases[chartPrepareResultKey(downloadRes)] = downloadRes.err
|
||||
continue
|
||||
}
|
||||
func() {
|
||||
prepareChartInfoMutex.Lock()
|
||||
@@ -2368,27 +2386,40 @@ func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurre
|
||||
},
|
||||
)
|
||||
|
||||
if len(errs) > 0 {
|
||||
if !opts.AllowFailedReleases {
|
||||
return nil, errs
|
||||
}
|
||||
// else if AllowFailedReleases: return partial results along with errs.
|
||||
if len(errs) > 0 && !opts.AllowFailedReleases {
|
||||
return nil, nil, errs
|
||||
}
|
||||
|
||||
if len(builds) > 0 {
|
||||
if err := st.runHelmDepBuilds(helm, concurrency, builds, opts); err != nil {
|
||||
if err := st.runHelmDepBuilds(helm, concurrency, builds, opts, failedReleases); err != nil {
|
||||
if !opts.AllowFailedReleases {
|
||||
return nil, []error{err}
|
||||
return nil, nil, []error{err}
|
||||
}
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
return prepareChartInfo, errs
|
||||
// Drop the chart paths of releases whose `helm dep build` failed (only
|
||||
// possible with opts.AllowFailedReleases), so that callers skip them too.
|
||||
for key := range failedReleases {
|
||||
delete(prepareChartInfo, key)
|
||||
}
|
||||
|
||||
return prepareChartInfo, failedReleases, errs
|
||||
}
|
||||
|
||||
// chartPrepareResultKey returns the map key identifying the release a chart
|
||||
// preparation result belongs to.
|
||||
func chartPrepareResultKey(r *chartPrepareResult) PrepareChartKey {
|
||||
return PrepareChartKey{
|
||||
Name: r.releaseName,
|
||||
Namespace: r.releaseNamespace,
|
||||
KubeContext: r.releaseContext,
|
||||
}
|
||||
}
|
||||
|
||||
// nolint: unparam
|
||||
func (st *HelmState) runHelmDepBuilds(helm helmexec.Interface, concurrency int, builds []*chartPrepareResult, opts ChartPrepareOptions) error {
|
||||
func (st *HelmState) runHelmDepBuilds(helm helmexec.Interface, concurrency int, builds []*chartPrepareResult, opts ChartPrepareOptions, failedReleases map[PrepareChartKey]error) error {
|
||||
// NOTES:
|
||||
// 1. `helm dep build` fails when it was run concurrency on the same chart.
|
||||
// To avoid that, we run `helm dep build` only once per each local chart.
|
||||
@@ -2412,7 +2443,15 @@ func (st *HelmState) runHelmDepBuilds(helm helmexec.Interface, concurrency int,
|
||||
|
||||
if anySkipRefresh && !opts.SkipRefresh && !st.HelmDefaults.SkipRefresh && st.NeedsRepoUpdate() {
|
||||
if err := helm.UpdateRepo(); err != nil {
|
||||
return fmt.Errorf("updating repo: %w", err)
|
||||
err = fmt.Errorf("updating repo: %w", err)
|
||||
if opts.AllowFailedReleases {
|
||||
// None of the dep builds below can be trusted to produce usable
|
||||
// charts: mark all of them as failed and let the caller skip them.
|
||||
for _, r := range builds {
|
||||
failedReleases[chartPrepareResultKey(r)] = err
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2441,7 +2480,14 @@ func (st *HelmState) runHelmDepBuilds(helm helmexec.Interface, concurrency int,
|
||||
continue
|
||||
}
|
||||
|
||||
return fmt.Errorf("building dependencies of local chart: %w", err)
|
||||
err = fmt.Errorf("building dependencies of local chart %q for release %q: %w", r.chartName, r.releaseName, err)
|
||||
if opts.AllowFailedReleases {
|
||||
// Record the failed release and keep building the remaining charts.
|
||||
failedReleases[chartPrepareResultKey(r)] = err
|
||||
continue
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user