feat: Add `needs: [NS/NAME]` for controlling installation/deletion order declaratively (#914)

Introduces DAG-aware installation/deletion ordering to Helmfile.

`needs` controls the order of the installation/deletion of the release:

```yaml
relesaes:
- name: somerelease
  needs:
  - [TILLER_NAMESPACE/][NAMESPACE/]anotherelease
```

All the releases listed under `needs` are installed before(or deleted after) the release itself.

For the following example, `helmfile [sync|apply]` installs releases in this order:

1. logging
2. servicemesh
3. myapp1 and myapp2

```yaml
  - name: myapp1
    chart: charts/myapp
    needs:
    - servicemesh
    - logging
  - name: myapp2
    chart: charts/myapp
    needs:
    - servicemesh
    - logging
  - name: servicemesh
    chart: charts/istio
    needs:
    - logging
  - name: logging
    chart: charts/fluentd
```

Note that all the releases in a same group is installed concurrently. That is, myapp1 and myapp2 are installed concurrently.

On `helmdile [delete|destroy]`, deleations happen in the reverse order.

That is, `myapp1` and `myapp2` are deleted first, then `servicemesh`, and finally `logging`.

Resolves #715

README.md

@@ -723,6 +723,50 @@ With the [helm-tiller](https://github.com/rimusz/helm-tiller) plugin installed,
 To enable this mode, you need to define `tillerless: true` and set the `tillerNamespace` in the `helmDefaults` section
 or in the `releases` entries.
 
+## DAG-aware installation/deletion ordering
+
+`needs` controls the order of the installation/deletion of the release:
+
+```yaml
+relesaes:
+- name: somerelease
+  needs:
+  - [TILLER_NAMESPACE/][NAMESPACE/]anotherelease
+```
+
+All the releases listed under `needs` are installed before(or deleted after) the release itself.
+
+For the following example, `helmfile [sync|apply]` installs releases in this order:
+
+1. logging
+2. servicemesh
+3. myapp1 and myapp2
+
+```yaml
+  - name: myapp1
+    chart: charts/myapp
+    needs:
+    - servicemesh
+    - logging
+  - name: myapp2
+    chart: charts/myapp
+    needs:
+    - servicemesh
+    - logging
+  - name: servicemesh
+    chart: charts/istio
+    needs:
+    - logging
+  - name: logging
+    chart: charts/fluentd
+```
+
+Note that all the releases in a same group is installed concurrently. That is, myapp1 and myapp2 are installed concurrently.
+
+On `helmdile [delete|destroy]`, deleations happen in the reverse order.
+
+That is, `myapp1` and `myapp2` are deleted first, then `servicemesh`, and finally `logging`.
+
 ## Separating helmfile.yaml into multiple independent files
 
 Once your `helmfile.yaml` got to contain too many releases,

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/r3labs/diff v0.0.0-20190801153147-a71de73c46ad
 	github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939
 	github.com/urfave/cli v1.20.0
+	github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363
 	github.com/variantdev/vals v0.0.0-20191026125821-5d18b16cf30a
 	go.mozilla.org/sops v0.0.0-20190912205235-14a22d7a7060 // indirect
 	go.opencensus.io v0.22.1 // indirect

go.sum

@@ -18,6 +18,7 @@ cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFx
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0 h1:VV2nUM3wwLLGh9lSABFgZMjInyUbJeaRSE64WuAIQ+4=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f/go.mod h1:sk5LnIjB/nIEU7yP5sDQExVm62wu0pBh3yrElngUisI=
 contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA=
@@ -85,6 +86,7 @@ github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RP
 github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver v1.4.1 h1:CaDA1wAoM3rj9sAFyyZP37LloExUzxFGYt+DqJ870JA=
 github.com/Masterminds/semver v1.4.1/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60=
 github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
@@ -228,6 +230,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -240,6 +243,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
 github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c=
 github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU=
 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
@@ -313,6 +317,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0=
 github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -577,7 +582,10 @@ github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
 github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28 h1:Yf7/DcGM+61oLBGXQV2Q+ztEGBcOe3EUnbKSOn4fQuE=
 github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363 h1:KrfQBEUn+wEOQ/6UIfoqRDvn+Q/wZridQ7t0G1vQqKE=
+github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363/go.mod h1:pH1TQsNSLj2uxMo9NNl9zdGy01Wtn+/2MT96BrKmVyE=
 github.com/variantdev/vals v0.0.0-20191025124021-e86de6f8cd7d h1:od9EUts72ELxOSrqJxtbxAU/c30XCaA+xnq0jDWMQrA=
 github.com/variantdev/vals v0.0.0-20191025124021-e86de6f8cd7d/go.mod h1:7Gb8avEj7D9rMIqn4Nn9RBKXo/0rZ7Z0e+3bmHIs420=
 github.com/variantdev/vals v0.0.0-20191026125821-5d18b16cf30a h1:zrV+XXPXniLy9ZVHUN7DLVVKnMjActASxwsMKIohfLA=
@@ -608,6 +616,7 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.8.0 h1:r6Za1Rii8+EGOYRDLvpooNOF6kP3iyDnkpzbw67gCQ8=
 go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
 go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=

pkg/state/state.go

@@ -20,6 +20,7 @@ import (
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/remote"
 	"github.com/roboll/helmfile/pkg/tmpl"
+	"github.com/variantdev/dag/pkg/dag"
 
 	"regexp"
 
@@ -144,6 +145,8 @@ type ReleaseSpec struct {
 	// MissingFileHandler is set to either "Error" or "Warn". "Error" instructs helmfile to fail when unable to find a values or secrets file. When "Warn", it prints the file and continues.
 	// The default value for MissingFileHandler is "Error".
 	MissingFileHandler *string `yaml:"missingFileHandler,omitempty"`
+	// Needs is the [TILLER_NS/][NS/]NAME representations of releases that this release depends on.
+	Needs []string `yaml:"needs,omitempty"`
 
 	// Hooks is a list of extension points paired with operations, that are executed in specific points of the lifecycle of releases defined in helmfile
 	Hooks []event.Hook `yaml:"hooks,omitempty"`
@@ -408,12 +411,89 @@ func (st *HelmState) SyncReleases(affectedReleases *AffectedReleases, helm helme
 		return prepErrs
 	}
 
+	availableIds := make([]string, len(preps))
+	idToPrep := map[string]syncPrepareResult{}
+
+	d := dag.New()
+	for i, p := range preps {
+		r := p.release
+
+		id := releaseToID(r)
+
+		idToPrep[id] = p
+
+		availableIds[i] = id
+
+		d.Add(id, dag.Dependencies(r.Needs))
+	}
+
+	plan, err := d.Plan()
+	if err != nil {
+		return []error{err}
+	}
+
+	groupsTotal := len(plan)
+
+	st.logger.Debugf("syncing %d groups of releases in this order: %s", groupsTotal, plan)
+
+	for id, prep := range idToPrep {
+		for _, need := range prep.release.Needs {
+			if _, ok := idToPrep[need]; !ok {
+				return []error{fmt.Errorf("%q needs %q, but it must be one of %s", id, need, strings.Join(availableIds, ", "))}
+			}
+		}
+	}
+
+	for groupIndex, dagNodesInGroup := range plan {
+		var idsInGroup []string
+		var prepsInGroup []syncPrepareResult
+
+		for _, node := range dagNodesInGroup {
+			prepareResult, ok := idToPrep[node.Id]
+			if !ok {
+				panic(fmt.Sprintf("[bug] no release found for dag node id %q", node.Id))
+			}
+			prepsInGroup = append(prepsInGroup, prepareResult)
+			idsInGroup = append(idsInGroup, node.Id)
+		}
+
+		st.logger.Debugf("syncing releases in group %d/%d: %s", groupIndex+1, groupsTotal, strings.Join(idsInGroup, ", "))
+
+		errs := st.syncReleaseGroup(affectedReleases, helm, prepsInGroup)
+		if len(errs) > 0 {
+			return errs
+		}
+	}
+
+	return nil
+}
+
+func releaseToID(r *ReleaseSpec) string {
+	var id string
+
+	tns := r.TillerNamespace
+	if tns != "" {
+		id += tns + "/"
+	}
+
+	ns := r.Namespace
+	if ns != "" {
+		id += ns + "/"
+	}
+
+	id += r.Name
+
+	return id
+}
+
+func (st *HelmState) syncReleaseGroup(affectedReleases *AffectedReleases, helm helmexec.Interface, preps []syncPrepareResult) []error {
 	errs := []error{}
 	jobQueue := make(chan *syncPrepareResult, len(preps))
 	results := make(chan syncResult, len(preps))
+	concurrency := len(preps)
 
 	st.scatterGather(
-		workerLimit,
+		concurrency,
 		len(preps),
 		func() {
 			for i := 0; i < len(preps); i++ {
@@ -1010,8 +1090,9 @@ func (st *HelmState) ReleaseStatuses(helm helmexec.Interface, workerLimit int) [
 }
 
 // DeleteReleases wrapper for executing helm delete on the releases
+// This function traverses the DAG of the releases in the reverse order, so that the releases that are NOT depended by any others are deleted first.
 func (st *HelmState) DeleteReleases(affectedReleases *AffectedReleases, helm helmexec.Interface, concurrency int, purge bool) []error {
-	return st.scatterGatherReleases(helm, concurrency, func(release ReleaseSpec, workerIndex int) error {
+	return st.dagAwareReverseIterateOnReleases(helm, concurrency, func(release ReleaseSpec, workerIndex int) error {
 		if !release.Desired() {
 			return nil
 		}

pkg/state/state_run.go

@@ -2,9 +2,11 @@ package state
 
 import (
 	"fmt"
+	"strings"
 	"sync"
 
 	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/variantdev/dag/pkg/dag"
 )
 
 type result struct {
@@ -51,9 +53,14 @@ func (st *HelmState) scatterGather(concurrency int, items int, produceInputs fun
 
 func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency int,
 	do func(ReleaseSpec, int) error) []error {
+
+	return st.iterateOnReleases(helm, concurrency, st.Releases, do)
+}
+
+func (st *HelmState) iterateOnReleases(helm helmexec.Interface, concurrency int, inputs []ReleaseSpec,
+	do func(ReleaseSpec, int) error) []error {
 	var errs []error
 
-	inputs := st.Releases
 	inputsSize := len(inputs)
 
 	releases := make(chan ReleaseSpec)
@@ -96,3 +103,52 @@ func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency
 
 	return nil
 }
+
+func (st *HelmState) dagAwareReverseIterateOnReleases(helm helmexec.Interface, concurrency int,
+	do func(ReleaseSpec, int) error) []error {
+
+	idToRelease := map[string]ReleaseSpec{}
+
+	preps := st.Releases
+
+	d := dag.New()
+	for _, r := range preps {
+
+		id := releaseToID(&r)
+
+		idToRelease[id] = r
+
+		d.Add(id, dag.Dependencies(r.Needs))
+	}
+
+	plan, err := d.Plan()
+	if err != nil {
+		return []error{err}
+	}
+
+	groupsTotal := len(plan)
+
+	st.logger.Debugf("processing %d groups of releases in this order: %s", groupsTotal, plan)
+
+	for groupIndex := len(plan) - 1; groupIndex >= 0; groupIndex-- {
+		dagNodesInGroup := plan[groupIndex]
+
+		var idsInGroup []string
+		var releasesInGroup []ReleaseSpec
+
+		for _, node := range dagNodesInGroup {
+			releasesInGroup = append(releasesInGroup, idToRelease[node.Id])
+			idsInGroup = append(idsInGroup, node.Id)
+		}
+
+		st.logger.Debugf("processing releases in group %d/%d: %s", groupIndex+1, groupsTotal, strings.Join(idsInGroup, ", "))
+
+		errs := st.iterateOnReleases(helm, concurrency, releasesInGroup, do)
+
+		if len(errs) > 0 {
+			return errs
+		}
+	}
+
+	return nil
+}

pkg/state/state_test.go

@@ -871,6 +871,7 @@ func TestHelmState_SyncReleases(t *testing.T) {
 		releases      []ReleaseSpec
 		helm          *mockHelmExec
 		wantReleases  []mockRelease
+		wantErrorMsgs []string
 	}{
 		{
 			name: "normal release",
@@ -961,6 +962,106 @@ func TestHelmState_SyncReleases(t *testing.T) {
 			helm:         &mockHelmExec{},
 			wantReleases: []mockRelease{{"releaseName", []string{"--set", "foo.bar[0]={A,B}"}}},
 		},
+		{
+			name: "foo needs bar",
+			releases: []ReleaseSpec{
+				{
+					Name:  "foo",
+					Chart: "charts/foo",
+					Needs: []string{
+						"bar",
+					},
+				},
+				{
+					Name:  "bar",
+					Chart: "charts/bar",
+				},
+			},
+			helm:         &mockHelmExec{},
+			wantReleases: []mockRelease{{"bar", []string{}}, {"foo", []string{}}},
+		},
+		{
+			name: "bar needs foo",
+			releases: []ReleaseSpec{
+				{
+					Name:  "foo",
+					Chart: "charts/foo",
+				},
+				{
+					Name:  "bar",
+					Chart: "charts/bar",
+					Needs: []string{
+						"foo",
+					},
+				},
+			},
+			helm:         &mockHelmExec{},
+			wantReleases: []mockRelease{{"foo", []string{}}, {"bar", []string{}}},
+		},
+		{
+			name: "ns2/bar needs ns1/foo",
+			releases: []ReleaseSpec{
+				{
+					Name:      "foo",
+					Namespace: "ns1",
+					Chart:     "charts/foo",
+				},
+				{
+					Name:      "bar",
+					Namespace: "ns2",
+					Chart:     "charts/bar",
+					Needs: []string{
+						"ns1/foo",
+					},
+				},
+			},
+			helm:         &mockHelmExec{},
+			wantReleases: []mockRelease{{"foo", []string{"--namespace", "ns1"}}, {"bar", []string{"--namespace", "ns2"}}},
+		},
+		{
+			name: "tillerns1/ns1/foo needs tillerns2/ns2/bar",
+			releases: []ReleaseSpec{
+				{
+					Name:            "foo",
+					Chart:           "charts/foo",
+					Namespace:       "ns1",
+					TillerNamespace: "tillerns1",
+					Needs: []string{
+						"tillerns2/ns2/bar",
+					},
+				},
+				{
+					Name:            "bar",
+					Namespace:       "ns2",
+					TillerNamespace: "tillerns2",
+					Chart:           "charts/bar",
+				},
+			},
+			helm:         &mockHelmExec{},
+			wantReleases: []mockRelease{{"bar", []string{"--tiller-namespace", "tillerns2", "--namespace", "ns2"}}, {"foo", []string{"--tiller-namespace", "tillerns1", "--namespace", "ns1"}}},
+		},
+		{
+			name: "tillerns1/ns1/foo needs tillerns2/ns2/bar",
+			releases: []ReleaseSpec{
+				{
+					Name:            "foo",
+					Chart:           "charts/foo",
+					Namespace:       "ns1",
+					TillerNamespace: "tillerns1",
+					Needs: []string{
+						"bar",
+					},
+				},
+				{
+					Name:            "bar",
+					Namespace:       "ns2",
+					TillerNamespace: "tillerns2",
+					Chart:           "charts/bar",
+				},
+			},
+			helm:          &mockHelmExec{},
+			wantErrorMsgs: []string{`"tillerns1/ns1/foo" needs "bar", but it must be one of tillerns1/ns1/foo, tillerns2/ns2/bar`},
+		},
 	}
 	for i := range tests {
 		tt := tests[i]
@@ -970,7 +1071,23 @@ func TestHelmState_SyncReleases(t *testing.T) {
 				logger:      logger,
 				valsRuntime: valsRuntime,
 			}
-			if _ = state.SyncReleases(&AffectedReleases{}, tt.helm, []string{}, 1); !reflect.DeepEqual(tt.helm.releases, tt.wantReleases) {
+			if errs := state.SyncReleases(&AffectedReleases{}, tt.helm, []string{}, 1); errs != nil && len(errs) > 0 {
+				if len(errs) != len(tt.wantErrorMsgs) {
+					t.Fatalf("Unexpected errors: %v\nExpected: %v", errs, tt.wantErrorMsgs)
+				}
+				var mismatch int
+				for i := range tt.wantErrorMsgs {
+					expected := tt.wantErrorMsgs[i]
+					actual := errs[i].Error()
+					if !reflect.DeepEqual(actual, expected) {
+						t.Errorf("Unexpected error: expected=%v, got=%v", expected, actual)
+					}
+				}
+				if mismatch > 0 {
+					t.Fatalf("%d unexpected errors detected", mismatch)
+				}
+			}
+			if !reflect.DeepEqual(tt.helm.releases, tt.wantReleases) {
 				t.Errorf("HelmState.SyncReleases() for [%s] = %v, want %v", tt.name, tt.helm.releases, tt.wantReleases)
 			}
 		})