fix: skip failed-prep releases, complete flag wiring, add tests and docs

Review follow-ups for --allow-failed-releases (#2616):

- Track per-release chart preparation failures in PrepareCharts (returned
  as a map keyed by release) and remove those releases from the state in
  Run.WithPreparedCharts when --allow-failed-releases is set, so a failed
  release is never executed against its original, un-prepared chart
  reference (which could either fail again with a duplicate error or, for
  charts requiring chartify, bypass patches/dependency modifications and
  produce an unintended result). All failures are still reported at the
  end via the aggregated MultiError.
- Complete the release identity on error results from
  prepareChartForRelease so failures are attributed to the correct
  release.
- With --allow-failed-releases, continue building dependencies of the
  remaining charts when 'helm dep build' fails for one release, and skip
  the affected releases during execution.
- Wire --allow-failed-releases into 'helmfile unittest' and 'helmfile
  status'; remove the dead flag wiring for write-values and list (both
  never prepare charts, see commandsSkipChartPrep).
- Simplify control flow (guard clauses, errors.As, drop dead code and
  redundant else branches).
- Add end-to-end coverage in pkg/app/issue_2616_test.go and extend the
  state-level tests; document the flag in docs/cli.md and CHANGELOG.md.

Signed-off-by: yxxhero <11087727+yxxhero@users.noreply.github.com>

pkg/state/issue923_test.go

@@ -86,7 +86,7 @@ releases:
 		OutputDirTemplate:      "{{ .OutputDir }}/{{ .Release.Name }}",
 	}
 
-	releaseToChart, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
+	releaseToChart, _, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
 	require.Empty(t, errs, "PrepareCharts should not return errors")
 
 	// Verify both releases have prepared charts
@@ -152,7 +152,7 @@ releases:
 		OutputDirTemplate:      "{{ .OutputDir }}/{{ .Release.Name }}",
 	}
 
-	releaseToChart, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
+	releaseToChart, _, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
 	require.Empty(t, errs)
 
 	// argocd-secrets should NOT have a prepared chart

pkg/state/issue_2616_test.go

@@ -11,13 +11,10 @@ import (
 	"github.com/helmfile/helmfile/pkg/exectest"
 )
 
-// TestAllowFailedReleasesFlag_Enabled verifies that the AllowFailedReleases flag
-// works as intended with one release that should work and another release, that
-// will fail.
-// The test makes one release fail by having an oci with latest version, which will always error
-func TestAllowFailedReleasesFlag_Enabled(t *testing.T) {
-	resetChartCacheForTest()
-	helmfileContent := []byte(`
+// issue2616HelmfileContent defines two releases: one that prepares fine
+// (grafana) and one that always fails chart preparation (error), because OCI
+// charts do not support the "latest" version tag.
+var issue2616HelmfileContent = []byte(`
 repositories:
 - name: stable
   url: kubernetes-charts.storage.googleapis.com
@@ -33,9 +30,15 @@ releases:
     version: latest
 `)
 
+// issue2616PrepareCharts runs PrepareCharts for issue2616HelmfileContent.
+func issue2616PrepareCharts(t *testing.T, opts ChartPrepareOptions) (map[PrepareChartKey]string, map[PrepareChartKey]error, []error) {
+	t.Helper()
+
+	resetChartCacheForTest()
+
 	logger := zap.NewExample().Sugar()
 
-	st, err := createFromYaml(helmfileContent, "example/path/to/helmfile.yaml", DefaultEnv, logger)
+	st, err := createFromYaml(issue2616HelmfileContent, "example/path/to/helmfile.yaml", DefaultEnv, logger)
 	require.NoError(t, err)
 
 	tempDir := t.TempDir()
@@ -46,13 +49,23 @@ releases:
 
 	// OutputDirTemplate includes .OutputDir so charts go into tempDir (auto-cleaned
 	// by t.TempDir), not the global cache or CWD.
-	opts := ChartPrepareOptions{
+	opts.OutputDirTemplate = "{{ .OutputDir }}/{{ .Release.Name }}"
+
+	releaseToChart, failedReleases, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
+	return releaseToChart, failedReleases, errs
+}
+
+// TestAllowFailedReleasesFlag_Enabled verifies that the AllowFailedReleases flag
+// works as intended with one release that should work and another release, that
+// will fail: the healthy release is prepared and reported in the partial
+// results, while the failing one is recorded in failedReleases so that callers
+// can skip it during execution.
+func TestAllowFailedReleasesFlag_Enabled(t *testing.T) {
+	releaseToChart, failedReleases, errs := issue2616PrepareCharts(t, ChartPrepareOptions{
 		Concurrency:         1,
-		OutputDirTemplate:   "{{ .OutputDir }}/{{ .Release.Name }}",
 		AllowFailedReleases: true,
-	}
+	})
 
-	releaseToChart, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
 	require.NotEmpty(t, errs, "PrepareCharts should return an error")
 
 	// Verify only the error chart is not prepared
@@ -60,55 +73,26 @@ releases:
 		"grafana chart should be prepared")
 	assert.NotContains(t, releaseToChart, PrepareChartKey{Name: "error", Namespace: "failed"},
 		"error chart should not be prepared")
+
+	// Verify the failed release is attributed per-release, so that the caller
+	// can skip it instead of executing it against the un-prepared chart
+	assert.NotContains(t, failedReleases, PrepareChartKey{Name: "grafana", Namespace: "grafana"},
+		"grafana should not be marked as failed")
+	failedKey := PrepareChartKey{Name: "error", Namespace: "failed"}
+	assert.Contains(t, failedReleases, failedKey, "the failing release should be marked as failed")
+	assert.Error(t, failedReleases[failedKey])
 }
 
 // TestAllowFailedReleasesFlag_Disabled verifies that the AllowFailedReleases flag
 // has no unintended side effects and the default abort on error still works.
-// The test makes one release fail by having an oci with latest version, which will always error
 func TestAllowFailedReleasesFlag_Disabled(t *testing.T) {
-	resetChartCacheForTest()
-	cleanupLeakedChartDirs(t, "grafana", "error")
-	helmfileContent := []byte(`
-repositories:
-- name: stable
-  url: kubernetes-charts.storage.googleapis.com
-  oci: true
+	releaseToChart, failedReleases, errs := issue2616PrepareCharts(t, ChartPrepareOptions{
+		Concurrency: 1,
+	})
 
-releases:
-  - name: grafana
-    namespace: grafana
-    chart: stable/grafana
-  - name: error
-    namespace: failed
-    chart: oci://example.com/chart/error
-    version: latest
-`)
-
-	logger := zap.NewExample().Sugar()
-
-	st, err := createFromYaml(helmfileContent, "example/path/to/helmfile.yaml", DefaultEnv, logger)
-	require.NoError(t, err)
-
-	tempDir := t.TempDir()
-
-	helm := &exectest.Helm{
-		ChartsMutex: &sync.Mutex{},
-	}
-
-	// OutputDirTemplate includes .OutputDir so charts go into tempDir (auto-cleaned
-	// by t.TempDir), not the global cache or CWD.
-	opts := ChartPrepareOptions{
-		Concurrency:       1,
-		OutputDirTemplate: "{{ .OutputDir }}/{{ .Release.Name }}",
-		// PrefetchSharedRemoteCharts: true,
-	}
-
-	releaseToChart, errs := st.PrepareCharts(helm, tempDir, 1, "apply", opts)
 	require.NotEmpty(t, errs, "PrepareCharts should return an error")
 
-	// Verify both releases are not prepared
-	assert.NotContains(t, releaseToChart, PrepareChartKey{Name: "grafana", Namespace: "grafana"},
-		"grafana chart should not be prepared")
-	assert.NotContains(t, releaseToChart, PrepareChartKey{Name: "error", Namespace: "failed"},
-		"error chart should not be prepared")
+	// Verify no partial results are returned on the default abort-on-error path
+	assert.Nil(t, releaseToChart)
+	assert.Nil(t, failedReleases)
 }

pkg/state/issue_2741_test.go

@@ -116,7 +116,7 @@ func TestPrepareChartsPrefetchesSharedRemoteChart(t *testing.T) {
 		OutputDirTemplate:          "{{ .OutputDir }}/{{ .Release.Name }}",
 	}
 
-	releaseToChart, errs := st.PrepareCharts(mockHelm, tempDir, 5, "sync", opts)
+	releaseToChart, _, errs := st.PrepareCharts(mockHelm, tempDir, 5, "sync", opts)
 	require.Empty(t, errs, "PrepareCharts should not return errors")
 
 	assert.Equal(t, int32(1), mockHelm.fetchCount.Load(),
@@ -158,7 +158,7 @@ func TestPrepareChartsSkipsPrefetchForUniqueCharts(t *testing.T) {
 		OutputDirTemplate:          "{{ .OutputDir }}/{{ .Release.Name }}",
 	}
 
-	releaseToChart, errs := st.PrepareCharts(mockHelm, tempDir, 3, "sync", opts)
+	releaseToChart, _, errs := st.PrepareCharts(mockHelm, tempDir, 3, "sync", opts)
 	require.Empty(t, errs, "PrepareCharts should not return errors")
 
 	assert.Equal(t, int32(0), mockHelm.fetchCount.Load(),
@@ -199,7 +199,7 @@ func TestPrepareChartsSkipsPrefetchWhenFetchFlagsDiffer(t *testing.T) {
 		OutputDirTemplate:          "{{ .OutputDir }}/{{ .Release.Name }}",
 	}
 
-	releaseToChart, errs := st.PrepareCharts(mockHelm, tempDir, 2, "sync", opts)
+	releaseToChart, _, errs := st.PrepareCharts(mockHelm, tempDir, 2, "sync", opts)
 	require.Empty(t, errs)
 
 	assert.Equal(t, int32(0), mockHelm.fetchCount.Load(),
@@ -241,7 +241,7 @@ func TestPrepareChartsSkipsPrefetchWhenVerifyEnabled(t *testing.T) {
 		OutputDirTemplate:          "{{ .OutputDir }}/{{ .Release.Name }}",
 	}
 
-	releaseToChart, errs := st.PrepareCharts(mockHelm, tempDir, 2, "sync", opts)
+	releaseToChart, _, errs := st.PrepareCharts(mockHelm, tempDir, 2, "sync", opts)
 	require.Empty(t, errs)
 
 	assert.Equal(t, int32(0), mockHelm.fetchCount.Load(),
@@ -294,7 +294,7 @@ func TestPrepareChartsSkipsPrefetchForUnknownRepoChart(t *testing.T) {
 		OutputDirTemplate:          "{{ .OutputDir }}/{{ .Release.Name }}",
 	}
 
-	releaseToChart, errs := st.PrepareCharts(mockHelm, tempDir, 2, "diff", opts)
+	releaseToChart, _, errs := st.PrepareCharts(mockHelm, tempDir, 2, "diff", opts)
 	require.Empty(t, errs)
 
 	assert.Equal(t, int32(0), mockHelm.fetchCount.Load(),
@@ -367,7 +367,7 @@ func TestPrefetchedSharedChartAllowsConcurrentSyncRelease(t *testing.T) {
 		OutputDirTemplate:          "{{ .OutputDir }}/{{ .Release.Name }}",
 	}
 
-	releaseToChart, errs := st.PrepareCharts(fetchHelm, tempDir, 5, "sync", opts)
+	releaseToChart, _, errs := st.PrepareCharts(fetchHelm, tempDir, 5, "sync", opts)
 	require.Empty(t, errs)
 	require.Equal(t, int32(1), fetchHelm.fetchCount.Load())
 
@@ -422,7 +422,7 @@ func TestPrefetchedSharedChartAllowsConcurrentDiffRelease(t *testing.T) {
 		OutputDirTemplate:          "{{ .OutputDir }}/{{ .Release.Name }}",
 	}
 
-	releaseToChart, errs := st.PrepareCharts(fetchHelm, tempDir, 5, "diff", opts)
+	releaseToChart, _, errs := st.PrepareCharts(fetchHelm, tempDir, 5, "diff", opts)
 	require.Empty(t, errs)
 	require.Equal(t, int32(1), fetchHelm.fetchCount.Load())
 

pkg/state/run_helm_dep_builds_skip_refresh_test.go

@@ -275,7 +275,7 @@ func TestRunHelmDepBuilds_SkipRefreshBehaviors(t *testing.T) {
 				SkipRefresh: tt.optsSkipRefresh,
 			}
 
-			err := st.runHelmDepBuilds(helm, 1, builds, opts)
+			err := st.runHelmDepBuilds(helm, 1, builds, opts, nil)
 			require.NoError(t, err)
 
 			assert.NotNil(t, helm.buildDepsFlags, "BuildDeps should have been called")
@@ -336,7 +336,7 @@ func TestRunHelmDepBuilds_MultipleBuilds(t *testing.T) {
 
 	opts := ChartPrepareOptions{SkipRefresh: false}
 
-	err := st.runHelmDepBuilds(helm, 1, builds, opts)
+	err := st.runHelmDepBuilds(helm, 1, builds, opts, nil)
 	require.NoError(t, err)
 
 	assert.True(t, helm.updateRepoCalled, "UpdateRepo should have been called")

pkg/state/state.go

@@ -2208,22 +2208,24 @@ func (st *HelmState) prepareChartForRelease(release *ReleaseSpec, helm helmexec.
 }
 
 // PrepareCharts downloads and prepares all charts for the selected releases.
-// Returns the chart paths and any errors encountered.
+// It returns the chart paths for the successfully prepared releases, the set of
+// releases that failed to prepare (keyed by release, with their errors), and any
+// errors encountered.
 //
 // Note: OCI chart locks are acquired and released during chart download within this function.
 // The tempDir cleanup is deferred until after helm operations complete in the caller,
 // so charts remain available during helm commands even though locks are released.
-func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurrency int, helmfileCommand string, opts ChartPrepareOptions) (map[PrepareChartKey]string, []error) {
+func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurrency int, helmfileCommand string, opts ChartPrepareOptions) (map[PrepareChartKey]string, map[PrepareChartKey]error, []error) {
 	if !opts.SkipResolve {
 		updated, err := st.ResolveDeps()
 		if err != nil {
-			return nil, []error{err}
+			return nil, nil, []error{err}
 		}
 		*st = *updated
 	}
 	selected, err := st.GetSelectedReleases(opts.IncludeTransitiveNeeds)
 	if err != nil {
-		return nil, []error{err}
+		return nil, nil, []error{err}
 	}
 
 	releases := releasesNeedCharts(selected)
@@ -2310,6 +2312,12 @@ func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurre
 
 	prepareChartInfo := make(map[PrepareChartKey]string, len(releases))
 
+	// Releases that failed to prepare. When opts.AllowFailedReleases is set,
+	// these releases are excluded from the returned chart paths so that callers
+	// can skip them instead of executing them against their un-prepared chart
+	// references.
+	failedReleases := make(map[PrepareChartKey]error)
+
 	errs := []error{}
 
 	jobQueue := make(chan *ReleaseSpec, len(releases))
@@ -2333,6 +2341,14 @@ func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurre
 					releaseOpts.ForceDownload = true
 				}
 				result := st.prepareChartForRelease(release, helm, dir, helmfileCommand, releaseOpts, workerIndex)
+				if result.err != nil {
+					// Error results returned by prepareChartForRelease may lack the
+					// release identity. Complete it here, so that the failure can be
+					// attributed to the correct release in failedReleases below.
+					result.releaseName = release.Name
+					result.releaseNamespace = release.Namespace
+					result.releaseContext = release.KubeContext
+				}
 				results <- result
 			}
 		},
@@ -2345,10 +2361,12 @@ func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurre
 
 					if !opts.AllowFailedReleases {
 						return
-					} else {
-						// continue processing other releases even if one fails
-						continue
 					}
+
+					// Continue processing the other releases and record which one
+					// failed, so that the caller can skip it during execution.
+					failedReleases[chartPrepareResultKey(downloadRes)] = downloadRes.err
+					continue
 				}
 				func() {
 					prepareChartInfoMutex.Lock()
@@ -2368,27 +2386,40 @@ func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurre
 		},
 	)
 
-	if len(errs) > 0 {
-		if !opts.AllowFailedReleases {
-			return nil, errs
-		}
-		// else if AllowFailedReleases: return partial results along with errs.
+	if len(errs) > 0 && !opts.AllowFailedReleases {
+		return nil, nil, errs
 	}
 
 	if len(builds) > 0 {
-		if err := st.runHelmDepBuilds(helm, concurrency, builds, opts); err != nil {
+		if err := st.runHelmDepBuilds(helm, concurrency, builds, opts, failedReleases); err != nil {
 			if !opts.AllowFailedReleases {
-				return nil, []error{err}
+				return nil, nil, []error{err}
 			}
 			errs = append(errs, err)
 		}
 	}
 
-	return prepareChartInfo, errs
+	// Drop the chart paths of releases whose `helm dep build` failed (only
+	// possible with opts.AllowFailedReleases), so that callers skip them too.
+	for key := range failedReleases {
+		delete(prepareChartInfo, key)
+	}
+
+	return prepareChartInfo, failedReleases, errs
+}
+
+// chartPrepareResultKey returns the map key identifying the release a chart
+// preparation result belongs to.
+func chartPrepareResultKey(r *chartPrepareResult) PrepareChartKey {
+	return PrepareChartKey{
+		Name:        r.releaseName,
+		Namespace:   r.releaseNamespace,
+		KubeContext: r.releaseContext,
+	}
 }
 
 // nolint: unparam
-func (st *HelmState) runHelmDepBuilds(helm helmexec.Interface, concurrency int, builds []*chartPrepareResult, opts ChartPrepareOptions) error {
+func (st *HelmState) runHelmDepBuilds(helm helmexec.Interface, concurrency int, builds []*chartPrepareResult, opts ChartPrepareOptions, failedReleases map[PrepareChartKey]error) error {
 	// NOTES:
 	// 1. `helm dep build` fails when it was run concurrency on the same chart.
 	//    To avoid that, we run `helm dep build` only once per each local chart.
@@ -2412,7 +2443,15 @@ func (st *HelmState) runHelmDepBuilds(helm helmexec.Interface, concurrency int,
 
 	if anySkipRefresh && !opts.SkipRefresh && !st.HelmDefaults.SkipRefresh && st.NeedsRepoUpdate() {
 		if err := helm.UpdateRepo(); err != nil {
-			return fmt.Errorf("updating repo: %w", err)
+			err = fmt.Errorf("updating repo: %w", err)
+			if opts.AllowFailedReleases {
+				// None of the dep builds below can be trusted to produce usable
+				// charts: mark all of them as failed and let the caller skip them.
+				for _, r := range builds {
+					failedReleases[chartPrepareResultKey(r)] = err
+				}
+			}
+			return err
 		}
 	}
 
@@ -2441,7 +2480,14 @@ func (st *HelmState) runHelmDepBuilds(helm helmexec.Interface, concurrency int,
 				continue
 			}
 
-			return fmt.Errorf("building dependencies of local chart: %w", err)
+			err = fmt.Errorf("building dependencies of local chart %q for release %q: %w", r.chartName, r.releaseName, err)
+			if opts.AllowFailedReleases {
+				// Record the failed release and keep building the remaining charts.
+				failedReleases[chartPrepareResultKey(r)] = err
+				continue
+			}
+
+			return err
 		}
 	}