Update DAG with dependencies (#1477)

* Add show-dag command

Signed-off-by: vlpav030 <vpav.030@gmail.com>

cmd/root.go

@@ -97,6 +97,7 @@ func NewRootCmd(globalConfig *config.GlobalOptions) (*cobra.Command, error) {
 		NewSyncCmd(globalImpl),
 		NewDiffCmd(globalImpl),
 		NewStatusCmd(globalImpl),
+		NewShowDAGCmd(globalImpl),
 		extension.NewVersionCobraCmd(
 			versionOpts...,
 		),

cmd/show-dag.go

@@ -0,0 +1,32 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/helmfile/helmfile/pkg/app"
+	"github.com/helmfile/helmfile/pkg/config"
+)
+
+func NewShowDAGCmd(globalCfg *config.GlobalImpl) *cobra.Command {
+	showDAGOptions := config.NewShowDAGOptions()
+
+	cmd := &cobra.Command{
+		Use:   "show-dag",
+		Short: "It prints a table with 3 columns, GROUP, RELEASE, and DEPENDENCIES. GROUP is the unsigned, monotonically increasing integer starting from 1. All the releases with the same GROUP are deployed concurrently. Everything in GROUP 2 starts being deployed only after everything in GROUP 1 got successfully deployed. RELEASE is the release that belongs to the GROUP. DEPENDENCIES is the list of releases that the RELEASE depends on. It should always be empty for releases in GROUP 1. DEPENDENCIES for a release in GROUP 2 should have some or all dependencies appeared in GROUP 1. It can be \"some\" because Helmfile simplifies the DAGs of releases into a DAG of groups, so that Helmfile always produce a single DAG for everything written in helmfile.yaml, even when there are technically two or more independent DAGs of releases in it.",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			showDAGImpl := config.NewShowDAGImpl(globalCfg, showDAGOptions)
+			err := config.NewCLIConfigImpl(showDAGImpl.GlobalImpl)
+			if err != nil {
+				return err
+			}
+
+			if err := showDAGImpl.ValidateConfig(); err != nil {
+				return err
+			}
+
+			a := app.New(showDAGImpl)
+			return toCLIError(showDAGImpl.GlobalImpl, a.PrintDAGState(showDAGImpl))
+		},
+	}
+	return cmd
+}

docs/index.md

@@ -569,6 +569,7 @@ Available Commands:
   lint         Lint charts from state file (helm lint)
   list         List releases defined in state file
   repos        Add chart repositories defined in state file
+  show-dag     It prints a table with 3 columns, GROUP, RELEASE, and DEPENDENCIES. GROUP is the unsigned, monotonically increasing integer starting from 1. All the releases with the same GROUP are deployed concurrently. Everything in GROUP 2 starts being deployed only after everything in GROUP 1 got successfully deployed. RELEASE is the release that belongs to the GROUP. DEPENDENCIES is the list of releases that the RELEASE depends on. It should always be empty for releases in GROUP 1. DEPENDENCIES for a release in GROUP 2 should have some or all dependencies appeared in GROUP 1. It can be "some" because Helmfile simplifies the DAGs of releases into a DAG of groups, so that Helmfile always produce a single DAG for everything written in helmfile.yaml, even when there are technically two or more independent DAGs of releases in it.
   status       Retrieve status of releases in state file
   sync         Sync releases defined in state file
   template     Template releases defined in state file
@@ -710,6 +711,16 @@ The `helmfile version` sub-command prints the version of Helmfile.Optional `-o`
 
 default it will check for the latest version of Helmfile and print a tip if the current version is not the latest. To disable this behavior, set environment variable `HELMFILE_UPGRADE_NOTICE_DISABLED` to any non-empty value.
 
+### show-dag
+
+It prints a table with 3 columns, GROUP, RELEASE, and DEPENDENCIES.
+
+GROUP is the unsigned, monotonically increasing integer starting from 1. All the releases with the same GROUP are deployed concurrently. Everything in GROUP 2 starts being deployed only after everything in GROUP 1 got successfully deployed.
+
+RELEASE is the release that belongs to the GROUP.
+
+DEPENDENCIES is the list of releases that the RELEASE depends on. It should always be empty for releases in GROUP 1. DEPENDENCIES for a release in GROUP 2 should have some or all dependencies appeared in GROUP 1. It can be "some" because Helmfile simplifies the DAGs of releases into a DAG of groups, so that Helmfile always produce a single DAG for everything written in helmfile.yaml, even when there are technically two or more independent DAGs of releases in it.
+
 ## Paths Overview
 
 Using manifest files in conjunction with command line argument can be a bit confusing.

pkg/app/app.go

@@ -530,6 +530,23 @@ func (a *App) Test(c TestConfigProvider) error {
 	}, false, SetFilter(true))
 }
 
+func (a *App) PrintDAGState(c DAGConfigProvider) error {
+	var err error
+	return a.ForEachState(func(run *Run) (ok bool, errs []error) {
+		err = run.withPreparedCharts("show-dag", state.ChartPrepareOptions{
+			SkipRepos:   true,
+			SkipDeps:    true,
+			Concurrency: 2,
+		}, func() {
+			err = a.dag(run)
+			if err != nil {
+				errs = append(errs, err)
+			}
+		})
+		return ok, errs
+	}, false, SetFilter(true))
+}
+
 func (a *App) PrintState(c StateConfigProvider) error {
 	return a.ForEachState(func(run *Run) (_ bool, errs []error) {
 		err := run.withPreparedCharts("build", state.ChartPrepareOptions{
@@ -583,6 +600,19 @@ func (a *App) PrintState(c StateConfigProvider) error {
 	}, false, SetFilter(true))
 }
 
+func (a *App) dag(r *Run) error {
+	st := r.state
+
+	batches, err := st.PlanReleases(state.PlanOptions{SelectedReleases: st.Releases, Reverse: false, SkipNeeds: false, IncludeNeeds: true, IncludeTransitiveNeeds: true})
+	if err != nil {
+		return err
+	}
+
+	fmt.Print(printDAG(batches))
+
+	return nil
+}
+
 func (a *App) ListReleases(c ListConfigProvider) error {
 	var releases []*HelmRelease
 
@@ -990,6 +1020,28 @@ func printBatches(batches [][]state.Release) string {
 	return buf.String()
 }
 
+func printDAG(batches [][]state.Release) string {
+	buf := &bytes.Buffer{}
+
+	w := new(tabwriter.Writer)
+
+	w.Init(buf, 0, 1, 1, ' ', 0)
+
+	fmt.Fprintln(w, "GROUP\tRELEASE\tDEPENDENCIES")
+
+	for i, batch := range batches {
+		for _, r := range batch {
+			id := state.ReleaseToID(&r.ReleaseSpec)
+			needs := r.ReleaseSpec.Needs
+			fmt.Fprintf(w, "%d\t%s\t%s\n", i+1, id, strings.Join(needs, ", "))
+		}
+	}
+
+	_ = w.Flush()
+
+	return buf.String()
+}
+
 // nolint: unparam
 func withDAG(templated *state.HelmState, helm helmexec.Interface, logger *zap.SugaredLogger, opts state.PlanOptions, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) (bool, []error) {
 	batches, err := templated.PlanReleases(opts)

pkg/app/config.go

@@ -266,6 +266,8 @@ type StateConfigProvider interface {
 	EmbedValues() bool
 }
 
+type DAGConfigProvider any
+
 type concurrencyConfig interface {
 	Concurrency() int
 }

pkg/app/dag_test.go

@@ -0,0 +1,147 @@
+package app
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/helmfile/vals"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+
+	ffs "github.com/helmfile/helmfile/pkg/filesystem"
+	"github.com/helmfile/helmfile/pkg/testhelper"
+	"github.com/helmfile/helmfile/pkg/testutil"
+)
+
+func testDAG(t *testing.T, cfg configImpl) {
+	type testcase struct {
+		environment string
+		ns          string
+		error       string
+		selectors   []string
+		expected    string
+	}
+
+	check := func(t *testing.T, tc testcase, cfg configImpl) {
+		t.Helper()
+
+		bs := runWithLogCapture(t, "debug", func(t *testing.T, logger *zap.SugaredLogger) {
+			t.Helper()
+
+			valsRuntime, err := vals.New(vals.Options{CacheSize: 32})
+			if err != nil {
+				t.Errorf("unexpected error creating vals runtime: %v", err)
+			}
+
+			files := map[string]string{
+				"/path/to/helmfile.yaml": `
+environments:
+  development: {}
+  shared: {}
+---
+releases:
+- name: logging
+  chart: incubator/raw
+  namespace: kube-system
+- name: kubernetes-external-secrets
+  chart: incubator/raw
+  namespace: kube-system
+  needs:
+  - kube-system/logging
+- name: external-secrets
+  chart: incubator/raw
+  namespace: default
+  labels:
+    app: test
+  needs:
+  - kube-system/kubernetes-external-secrets
+- name: my-release
+  chart: incubator/raw
+  namespace: default
+  labels:
+    app: test
+  needs:
+  - default/external-secrets
+# Disabled releases are treated as missing
+- name: disabled
+  chart: incubator/raw
+  namespace: kube-system
+  installed: false
+- name: test2
+  chart: incubator/raw
+  needs:
+  - kube-system/disabled
+- name: test3
+  chart: incubator/raw
+  needs:
+  - test2
+- name: test4
+  chart: incubator/raw
+  needs:
+  - test2
+  - test3
+`,
+			}
+
+			app := appWithFs(&App{
+				OverrideHelmBinary:  DefaultHelmBinary,
+				fs:                  ffs.DefaultFileSystem(),
+				OverrideKubeContext: "default",
+				Env:                 tc.environment,
+				Logger:              logger,
+				valsRuntime:         valsRuntime,
+			}, files)
+
+			expectNoCallsToHelm(app)
+
+			if tc.ns != "" {
+				app.Namespace = tc.ns
+			}
+
+			if tc.selectors != nil {
+				app.Selectors = tc.selectors
+			}
+
+			var dagErr error
+			out, err := testutil.CaptureStdout(func() {
+				dagErr = app.PrintDAGState(cfg)
+			})
+			assert.NoError(t, err)
+
+			var gotErr string
+			if dagErr != nil {
+				gotErr = dagErr.Error()
+			}
+
+			if d := cmp.Diff(tc.error, gotErr); d != "" {
+				t.Fatalf("unexpected error: want (-), got (+): %s", d)
+			}
+
+			assert.Equal(t, tc.expected, out)
+		})
+
+		testhelper.RequireLog(t, "dag_test", bs)
+	}
+
+	t.Run("DAG lists dependencies in order", func(t *testing.T) {
+		check(t, testcase{
+			environment: "default",
+			expected: `GROUP RELEASE                                         DEPENDENCIES
+1     default/kube-system/logging                     
+1     default/kube-system/disabled                    
+2     default/kube-system/kubernetes-external-secrets default/kube-system/logging
+2     default//test2                                  default/kube-system/disabled
+3     default/default/external-secrets                default/kube-system/kubernetes-external-secrets
+3     default//test3                                  default//test2
+4     default/default/my-release                      default/default/external-secrets
+4     default//test4                                  default//test2, default//test3
+`,
+		}, cfg)
+	})
+}
+
+func TestDAG(t *testing.T) {
+	t.Run("DAG", func(t *testing.T) {
+		testDAG(t, configImpl{})
+	})
+}

pkg/app/testdata/dag_test/dag_lists_dependencies_in_order

@@ -0,0 +1,120 @@
+processing file "helmfile.yaml" in directory "."
+changing working directory to "/path/to"
+first-pass rendering starting for "helmfile.yaml.part.0": inherited=&{default  map[] map[]}, overrode=<nil>
+first-pass uses: &{default  map[] map[]}
+first-pass rendering output of "helmfile.yaml.part.0":
+ 0: 
+ 1: environments:
+ 2:   development: {}
+ 3:   shared: {}
+
+first-pass produced: &{default  map[] map[]}
+first-pass rendering result of "helmfile.yaml.part.0": {default  map[] map[]}
+vals:
+map[]
+defaultVals:[]
+second-pass rendering result of "helmfile.yaml.part.0":
+ 0: 
+ 1: environments:
+ 2:   development: {}
+ 3:   shared: {}
+
+merged environment: &{default  map[] map[]}
+first-pass rendering starting for "helmfile.yaml.part.1": inherited=&{default  map[] map[]}, overrode=<nil>
+first-pass uses: &{default  map[] map[]}
+first-pass rendering output of "helmfile.yaml.part.1":
+ 0: releases:
+ 1: - name: logging
+ 2:   chart: incubator/raw
+ 3:   namespace: kube-system
+ 4: - name: kubernetes-external-secrets
+ 5:   chart: incubator/raw
+ 6:   namespace: kube-system
+ 7:   needs:
+ 8:   - kube-system/logging
+ 9: - name: external-secrets
+10:   chart: incubator/raw
+11:   namespace: default
+12:   labels:
+13:     app: test
+14:   needs:
+15:   - kube-system/kubernetes-external-secrets
+16: - name: my-release
+17:   chart: incubator/raw
+18:   namespace: default
+19:   labels:
+20:     app: test
+21:   needs:
+22:   - default/external-secrets
+23: # Disabled releases are treated as missing
+24: - name: disabled
+25:   chart: incubator/raw
+26:   namespace: kube-system
+27:   installed: false
+28: - name: test2
+29:   chart: incubator/raw
+30:   needs:
+31:   - kube-system/disabled
+32: - name: test3
+33:   chart: incubator/raw
+34:   needs:
+35:   - test2
+36: - name: test4
+37:   chart: incubator/raw
+38:   needs:
+39:   - test2
+40:   - test3
+41: 
+
+first-pass produced: &{default  map[] map[]}
+first-pass rendering result of "helmfile.yaml.part.1": {default  map[] map[]}
+vals:
+map[]
+defaultVals:[]
+second-pass rendering result of "helmfile.yaml.part.1":
+ 0: releases:
+ 1: - name: logging
+ 2:   chart: incubator/raw
+ 3:   namespace: kube-system
+ 4: - name: kubernetes-external-secrets
+ 5:   chart: incubator/raw
+ 6:   namespace: kube-system
+ 7:   needs:
+ 8:   - kube-system/logging
+ 9: - name: external-secrets
+10:   chart: incubator/raw
+11:   namespace: default
+12:   labels:
+13:     app: test
+14:   needs:
+15:   - kube-system/kubernetes-external-secrets
+16: - name: my-release
+17:   chart: incubator/raw
+18:   namespace: default
+19:   labels:
+20:     app: test
+21:   needs:
+22:   - default/external-secrets
+23: # Disabled releases are treated as missing
+24: - name: disabled
+25:   chart: incubator/raw
+26:   namespace: kube-system
+27:   installed: false
+28: - name: test2
+29:   chart: incubator/raw
+30:   needs:
+31:   - kube-system/disabled
+32: - name: test3
+33:   chart: incubator/raw
+34:   needs:
+35:   - test2
+36: - name: test4
+37:   chart: incubator/raw
+38:   needs:
+39:   - test2
+40:   - test3
+41: 
+
+merged environment: &{default  map[] map[]}
+WARNING: release test2 needs disabled, but disabled is not installed due to installed: false. Either mark disabled as installed or remove disabled from test2's needs
+changing working directory back to "/path/to"

pkg/config/show-dag.go

@@ -0,0 +1,24 @@
+package config
+
+// ShowDAGOptions is the options for the build command
+type ShowDAGOptions struct {
+}
+
+// NewShowDAGOptions creates a new ShowDAGOptions
+func NewShowDAGOptions() *ShowDAGOptions {
+	return &ShowDAGOptions{}
+}
+
+// ShowDAGImpl is impl for applyOptions
+type ShowDAGImpl struct {
+	*GlobalImpl
+	*ShowDAGOptions
+}
+
+// NewShowDAGImpl creates a new ShowDAGImpl
+func NewShowDAGImpl(g *GlobalImpl, b *ShowDAGOptions) *ShowDAGImpl {
+	return &ShowDAGImpl{
+		GlobalImpl:     g,
+		ShowDAGOptions: b,
+	}
+}