From 7666e95690d92a32f67a05c75fc954da893d4c43 Mon Sep 17 00:00:00 2001 From: KUOKA Yusuke Date: Mon, 28 Oct 2019 12:57:25 +0900 Subject: [PATCH] feat: Add `needs: [NS/NAME]` for controlling installation/deletion order declaratively (#914) Introduces DAG-aware installation/deletion ordering to Helmfile. `needs` controls the order of the installation/deletion of the release: ```yaml relesaes: - name: somerelease needs: - [TILLER_NAMESPACE/][NAMESPACE/]anotherelease ``` All the releases listed under `needs` are installed before(or deleted after) the release itself. For the following example, `helmfile [sync|apply]` installs releases in this order: 1. logging 2. servicemesh 3. myapp1 and myapp2 ```yaml - name: myapp1 chart: charts/myapp needs: - servicemesh - logging - name: myapp2 chart: charts/myapp needs: - servicemesh - logging - name: servicemesh chart: charts/istio needs: - logging - name: logging chart: charts/fluentd ``` Note that all the releases in a same group is installed concurrently. That is, myapp1 and myapp2 are installed concurrently. On `helmdile [delete|destroy]`, deleations happen in the reverse order. That is, `myapp1` and `myapp2` are deleted first, then `servicemesh`, and finally `logging`. Resolves #715 --- README.md | 44 ++++++++++++++ go.mod | 1 + go.sum | 9 +++ pkg/state/state.go | 85 ++++++++++++++++++++++++++- pkg/state/state_run.go | 58 +++++++++++++++++- pkg/state/state_test.go | 127 ++++++++++++++++++++++++++++++++++++++-- 6 files changed, 316 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 75df68b2..2c6373c6 100644 --- a/README.md +++ b/README.md @@ -723,6 +723,50 @@ With the [helm-tiller](https://github.com/rimusz/helm-tiller) plugin installed, To enable this mode, you need to define `tillerless: true` and set the `tillerNamespace` in the `helmDefaults` section or in the `releases` entries. +## DAG-aware installation/deletion ordering + +`needs` controls the order of the installation/deletion of the release: + +```yaml +relesaes: +- name: somerelease + needs: + - [TILLER_NAMESPACE/][NAMESPACE/]anotherelease +``` + +All the releases listed under `needs` are installed before(or deleted after) the release itself. + +For the following example, `helmfile [sync|apply]` installs releases in this order: + +1. logging +2. servicemesh +3. myapp1 and myapp2 + +```yaml + - name: myapp1 + chart: charts/myapp + needs: + - servicemesh + - logging + - name: myapp2 + chart: charts/myapp + needs: + - servicemesh + - logging + - name: servicemesh + chart: charts/istio + needs: + - logging + - name: logging + chart: charts/fluentd +``` + +Note that all the releases in a same group is installed concurrently. That is, myapp1 and myapp2 are installed concurrently. + +On `helmdile [delete|destroy]`, deleations happen in the reverse order. + +That is, `myapp1` and `myapp2` are deleted first, then `servicemesh`, and finally `logging`. + ## Separating helmfile.yaml into multiple independent files Once your `helmfile.yaml` got to contain too many releases, diff --git a/go.mod b/go.mod index 2c301346..31c30378 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/r3labs/diff v0.0.0-20190801153147-a71de73c46ad github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939 github.com/urfave/cli v1.20.0 + github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363 github.com/variantdev/vals v0.0.0-20191026125821-5d18b16cf30a go.mozilla.org/sops v0.0.0-20190912205235-14a22d7a7060 // indirect go.opencensus.io v0.22.1 // indirect diff --git a/go.sum b/go.sum index da54458b..29de08ef 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,7 @@ cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFx cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0 h1:VV2nUM3wwLLGh9lSABFgZMjInyUbJeaRSE64WuAIQ+4= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f/go.mod h1:sk5LnIjB/nIEU7yP5sDQExVm62wu0pBh3yrElngUisI= contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= @@ -85,6 +86,7 @@ github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RP github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.4.1 h1:CaDA1wAoM3rj9sAFyyZP37LloExUzxFGYt+DqJ870JA= github.com/Masterminds/semver v1.4.1/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= @@ -228,6 +230,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -240,6 +243,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c= github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= @@ -313,6 +317,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -577,7 +582,10 @@ github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok= github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28 h1:Yf7/DcGM+61oLBGXQV2Q+ztEGBcOe3EUnbKSOn4fQuE= github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363 h1:KrfQBEUn+wEOQ/6UIfoqRDvn+Q/wZridQ7t0G1vQqKE= +github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363/go.mod h1:pH1TQsNSLj2uxMo9NNl9zdGy01Wtn+/2MT96BrKmVyE= github.com/variantdev/vals v0.0.0-20191025124021-e86de6f8cd7d h1:od9EUts72ELxOSrqJxtbxAU/c30XCaA+xnq0jDWMQrA= github.com/variantdev/vals v0.0.0-20191025124021-e86de6f8cd7d/go.mod h1:7Gb8avEj7D9rMIqn4Nn9RBKXo/0rZ7Z0e+3bmHIs420= github.com/variantdev/vals v0.0.0-20191026125821-5d18b16cf30a h1:zrV+XXPXniLy9ZVHUN7DLVVKnMjActASxwsMKIohfLA= @@ -608,6 +616,7 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.8.0 h1:r6Za1Rii8+EGOYRDLvpooNOF6kP3iyDnkpzbw67gCQ8= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= diff --git a/pkg/state/state.go b/pkg/state/state.go index 84604373..f9b56dbe 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -20,6 +20,7 @@ import ( "github.com/roboll/helmfile/pkg/helmexec" "github.com/roboll/helmfile/pkg/remote" "github.com/roboll/helmfile/pkg/tmpl" + "github.com/variantdev/dag/pkg/dag" "regexp" @@ -144,6 +145,8 @@ type ReleaseSpec struct { // MissingFileHandler is set to either "Error" or "Warn". "Error" instructs helmfile to fail when unable to find a values or secrets file. When "Warn", it prints the file and continues. // The default value for MissingFileHandler is "Error". MissingFileHandler *string `yaml:"missingFileHandler,omitempty"` + // Needs is the [TILLER_NS/][NS/]NAME representations of releases that this release depends on. + Needs []string `yaml:"needs,omitempty"` // Hooks is a list of extension points paired with operations, that are executed in specific points of the lifecycle of releases defined in helmfile Hooks []event.Hook `yaml:"hooks,omitempty"` @@ -408,12 +411,89 @@ func (st *HelmState) SyncReleases(affectedReleases *AffectedReleases, helm helme return prepErrs } + availableIds := make([]string, len(preps)) + idToPrep := map[string]syncPrepareResult{} + + d := dag.New() + for i, p := range preps { + r := p.release + + id := releaseToID(r) + + idToPrep[id] = p + + availableIds[i] = id + + d.Add(id, dag.Dependencies(r.Needs)) + } + + plan, err := d.Plan() + if err != nil { + return []error{err} + } + + groupsTotal := len(plan) + + st.logger.Debugf("syncing %d groups of releases in this order: %s", groupsTotal, plan) + + for id, prep := range idToPrep { + for _, need := range prep.release.Needs { + if _, ok := idToPrep[need]; !ok { + return []error{fmt.Errorf("%q needs %q, but it must be one of %s", id, need, strings.Join(availableIds, ", "))} + } + } + } + + for groupIndex, dagNodesInGroup := range plan { + var idsInGroup []string + var prepsInGroup []syncPrepareResult + + for _, node := range dagNodesInGroup { + prepareResult, ok := idToPrep[node.Id] + if !ok { + panic(fmt.Sprintf("[bug] no release found for dag node id %q", node.Id)) + } + prepsInGroup = append(prepsInGroup, prepareResult) + idsInGroup = append(idsInGroup, node.Id) + } + + st.logger.Debugf("syncing releases in group %d/%d: %s", groupIndex+1, groupsTotal, strings.Join(idsInGroup, ", ")) + + errs := st.syncReleaseGroup(affectedReleases, helm, prepsInGroup) + if len(errs) > 0 { + return errs + } + } + + return nil +} + +func releaseToID(r *ReleaseSpec) string { + var id string + + tns := r.TillerNamespace + if tns != "" { + id += tns + "/" + } + + ns := r.Namespace + if ns != "" { + id += ns + "/" + } + + id += r.Name + + return id +} + +func (st *HelmState) syncReleaseGroup(affectedReleases *AffectedReleases, helm helmexec.Interface, preps []syncPrepareResult) []error { errs := []error{} jobQueue := make(chan *syncPrepareResult, len(preps)) results := make(chan syncResult, len(preps)) + concurrency := len(preps) st.scatterGather( - workerLimit, + concurrency, len(preps), func() { for i := 0; i < len(preps); i++ { @@ -1010,8 +1090,9 @@ func (st *HelmState) ReleaseStatuses(helm helmexec.Interface, workerLimit int) [ } // DeleteReleases wrapper for executing helm delete on the releases +// This function traverses the DAG of the releases in the reverse order, so that the releases that are NOT depended by any others are deleted first. func (st *HelmState) DeleteReleases(affectedReleases *AffectedReleases, helm helmexec.Interface, concurrency int, purge bool) []error { - return st.scatterGatherReleases(helm, concurrency, func(release ReleaseSpec, workerIndex int) error { + return st.dagAwareReverseIterateOnReleases(helm, concurrency, func(release ReleaseSpec, workerIndex int) error { if !release.Desired() { return nil } diff --git a/pkg/state/state_run.go b/pkg/state/state_run.go index be2cbe68..9cbea83f 100644 --- a/pkg/state/state_run.go +++ b/pkg/state/state_run.go @@ -2,9 +2,11 @@ package state import ( "fmt" + "strings" "sync" "github.com/roboll/helmfile/pkg/helmexec" + "github.com/variantdev/dag/pkg/dag" ) type result struct { @@ -51,9 +53,14 @@ func (st *HelmState) scatterGather(concurrency int, items int, produceInputs fun func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency int, do func(ReleaseSpec, int) error) []error { + + return st.iterateOnReleases(helm, concurrency, st.Releases, do) +} + +func (st *HelmState) iterateOnReleases(helm helmexec.Interface, concurrency int, inputs []ReleaseSpec, + do func(ReleaseSpec, int) error) []error { var errs []error - inputs := st.Releases inputsSize := len(inputs) releases := make(chan ReleaseSpec) @@ -96,3 +103,52 @@ func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency return nil } + +func (st *HelmState) dagAwareReverseIterateOnReleases(helm helmexec.Interface, concurrency int, + do func(ReleaseSpec, int) error) []error { + + idToRelease := map[string]ReleaseSpec{} + + preps := st.Releases + + d := dag.New() + for _, r := range preps { + + id := releaseToID(&r) + + idToRelease[id] = r + + d.Add(id, dag.Dependencies(r.Needs)) + } + + plan, err := d.Plan() + if err != nil { + return []error{err} + } + + groupsTotal := len(plan) + + st.logger.Debugf("processing %d groups of releases in this order: %s", groupsTotal, plan) + + for groupIndex := len(plan) - 1; groupIndex >= 0; groupIndex-- { + dagNodesInGroup := plan[groupIndex] + + var idsInGroup []string + var releasesInGroup []ReleaseSpec + + for _, node := range dagNodesInGroup { + releasesInGroup = append(releasesInGroup, idToRelease[node.Id]) + idsInGroup = append(idsInGroup, node.Id) + } + + st.logger.Debugf("processing releases in group %d/%d: %s", groupIndex+1, groupsTotal, strings.Join(idsInGroup, ", ")) + + errs := st.iterateOnReleases(helm, concurrency, releasesInGroup, do) + + if len(errs) > 0 { + return errs + } + } + + return nil +} diff --git a/pkg/state/state_test.go b/pkg/state/state_test.go index 1ea5e39b..786acb03 100644 --- a/pkg/state/state_test.go +++ b/pkg/state/state_test.go @@ -867,10 +867,11 @@ func TestHelmState_SyncRepos(t *testing.T) { func TestHelmState_SyncReleases(t *testing.T) { tests := []struct { - name string - releases []ReleaseSpec - helm *mockHelmExec - wantReleases []mockRelease + name string + releases []ReleaseSpec + helm *mockHelmExec + wantReleases []mockRelease + wantErrorMsgs []string }{ { name: "normal release", @@ -961,6 +962,106 @@ func TestHelmState_SyncReleases(t *testing.T) { helm: &mockHelmExec{}, wantReleases: []mockRelease{{"releaseName", []string{"--set", "foo.bar[0]={A,B}"}}}, }, + { + name: "foo needs bar", + releases: []ReleaseSpec{ + { + Name: "foo", + Chart: "charts/foo", + Needs: []string{ + "bar", + }, + }, + { + Name: "bar", + Chart: "charts/bar", + }, + }, + helm: &mockHelmExec{}, + wantReleases: []mockRelease{{"bar", []string{}}, {"foo", []string{}}}, + }, + { + name: "bar needs foo", + releases: []ReleaseSpec{ + { + Name: "foo", + Chart: "charts/foo", + }, + { + Name: "bar", + Chart: "charts/bar", + Needs: []string{ + "foo", + }, + }, + }, + helm: &mockHelmExec{}, + wantReleases: []mockRelease{{"foo", []string{}}, {"bar", []string{}}}, + }, + { + name: "ns2/bar needs ns1/foo", + releases: []ReleaseSpec{ + { + Name: "foo", + Namespace: "ns1", + Chart: "charts/foo", + }, + { + Name: "bar", + Namespace: "ns2", + Chart: "charts/bar", + Needs: []string{ + "ns1/foo", + }, + }, + }, + helm: &mockHelmExec{}, + wantReleases: []mockRelease{{"foo", []string{"--namespace", "ns1"}}, {"bar", []string{"--namespace", "ns2"}}}, + }, + { + name: "tillerns1/ns1/foo needs tillerns2/ns2/bar", + releases: []ReleaseSpec{ + { + Name: "foo", + Chart: "charts/foo", + Namespace: "ns1", + TillerNamespace: "tillerns1", + Needs: []string{ + "tillerns2/ns2/bar", + }, + }, + { + Name: "bar", + Namespace: "ns2", + TillerNamespace: "tillerns2", + Chart: "charts/bar", + }, + }, + helm: &mockHelmExec{}, + wantReleases: []mockRelease{{"bar", []string{"--tiller-namespace", "tillerns2", "--namespace", "ns2"}}, {"foo", []string{"--tiller-namespace", "tillerns1", "--namespace", "ns1"}}}, + }, + { + name: "tillerns1/ns1/foo needs tillerns2/ns2/bar", + releases: []ReleaseSpec{ + { + Name: "foo", + Chart: "charts/foo", + Namespace: "ns1", + TillerNamespace: "tillerns1", + Needs: []string{ + "bar", + }, + }, + { + Name: "bar", + Namespace: "ns2", + TillerNamespace: "tillerns2", + Chart: "charts/bar", + }, + }, + helm: &mockHelmExec{}, + wantErrorMsgs: []string{`"tillerns1/ns1/foo" needs "bar", but it must be one of tillerns1/ns1/foo, tillerns2/ns2/bar`}, + }, } for i := range tests { tt := tests[i] @@ -970,7 +1071,23 @@ func TestHelmState_SyncReleases(t *testing.T) { logger: logger, valsRuntime: valsRuntime, } - if _ = state.SyncReleases(&AffectedReleases{}, tt.helm, []string{}, 1); !reflect.DeepEqual(tt.helm.releases, tt.wantReleases) { + if errs := state.SyncReleases(&AffectedReleases{}, tt.helm, []string{}, 1); errs != nil && len(errs) > 0 { + if len(errs) != len(tt.wantErrorMsgs) { + t.Fatalf("Unexpected errors: %v\nExpected: %v", errs, tt.wantErrorMsgs) + } + var mismatch int + for i := range tt.wantErrorMsgs { + expected := tt.wantErrorMsgs[i] + actual := errs[i].Error() + if !reflect.DeepEqual(actual, expected) { + t.Errorf("Unexpected error: expected=%v, got=%v", expected, actual) + } + } + if mismatch > 0 { + t.Fatalf("%d unexpected errors detected", mismatch) + } + } + if !reflect.DeepEqual(tt.helm.releases, tt.wantReleases) { t.Errorf("HelmState.SyncReleases() for [%s] = %v, want %v", tt.name, tt.helm.releases, tt.wantReleases) } })