fix: Fix needs to work for upgrades and when selectors are provided (#922)

* fix: Fix `needs` to work for upgrades and when selectors are provided

Fixes #919

* Add test framework for `helmfile apply`

* Various enhancements and fixes to the DAG support

- Make the order of upgrades/deletes more deterministic for testability
- Fix the test framework so that we can validate log outputs and errors
- Add more test cases for `helmfile apply`, along with bug fixes.
- Make sure it fails with an intuitive error when you have non-existent releases referenced from witin "needs"

pkg/state/state.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -20,9 +21,6 @@ import (
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/remote"
 	"github.com/roboll/helmfile/pkg/tmpl"
-	"github.com/variantdev/dag/pkg/dag"
-
-	"regexp"
 
 	"github.com/tatsushid/go-prettytable"
 	"github.com/variantdev/vals"
@@ -45,7 +43,7 @@ type HelmState struct {
 	Helmfiles          []SubHelmfileSpec `yaml:"helmfiles,omitempty"`
 	DeprecatedContext  string            `yaml:"context,omitempty"`
 	DeprecatedReleases []ReleaseSpec     `yaml:"charts,omitempty"`
-	Namespace          string            `yaml:"namespace,omitempty"`
+	OverrideNamespace  string            `yaml:"namespace,omitempty"`
 	Repositories       []RepositorySpec  `yaml:"repositories,omitempty"`
 	Releases           []ReleaseSpec     `yaml:"releases,omitempty"`
 	Selectors          []string          `yaml:"-"`
@@ -195,6 +193,12 @@ type ReleaseSpec struct {
 	installedVersion string
 }
 
+type Release struct {
+	ReleaseSpec
+
+	Filtered bool
+}
+
 // SetValue are the key values to set on a helm release
 type SetValue struct {
 	Name   string   `yaml:"name,omitempty"`
@@ -217,9 +221,16 @@ const MissingFileHandlerInfo = "Info"
 const MissingFileHandlerWarn = "Warn"
 const MissingFileHandlerDebug = "Debug"
 
-func (st *HelmState) applyDefaultsTo(spec *ReleaseSpec) {
-	if st.Namespace != "" {
-		spec.Namespace = st.Namespace
+func (st *HelmState) ApplyOverrides(spec *ReleaseSpec) {
+	if st.OverrideNamespace != "" {
+		spec.Namespace = st.OverrideNamespace
+
+		for i := 0; i < len(spec.Needs); i++ {
+			n := spec.Needs[i]
+			if len(strings.Split(n, "/")) == 1 {
+				spec.Needs[i] = st.OverrideNamespace + "/" + n
+			}
+		}
 	}
 }
 
@@ -290,7 +301,7 @@ func (st *HelmState) prepareSyncReleases(helm helmexec.Interface, additionalValu
 		},
 		func(workerIndex int) {
 			for release := range jobs {
-				st.applyDefaultsTo(release)
+				st.ApplyOverrides(release)
 
 				// If `installed: false`, the only potential operation on this release would be uninstalling.
 				// We skip generating values files in that case, because for an uninstall with `helm delete`, we don't need to those.
@@ -370,10 +381,10 @@ func (st *HelmState) isReleaseInstalled(context helmexec.HelmContext, helm helme
 	return false, nil
 }
 
-func (st *HelmState) DetectReleasesToBeDeleted(helm helmexec.Interface) ([]*ReleaseSpec, error) {
-	detected := []*ReleaseSpec{}
-	for i := range st.Releases {
-		release := st.Releases[i]
+func (st *HelmState) DetectReleasesToBeDeleted(helm helmexec.Interface, releases []ReleaseSpec) ([]ReleaseSpec, error) {
+	detected := []ReleaseSpec{}
+	for i := range releases {
+		release := releases[i]
 
 		if !release.Desired() {
 			installed, err := st.isReleaseInstalled(st.createHelmContext(&release, 0), helm, release)
@@ -382,7 +393,7 @@ func (st *HelmState) DetectReleasesToBeDeleted(helm helmexec.Interface) ([]*Rele
 			} else if installed {
 				// Otherwise `release` messed up(https://github.com/roboll/helmfile/issues/554)
 				r := release
-				detected = append(detected, &r)
+				detected = append(detected, r)
 			}
 		}
 	}
@@ -399,76 +410,7 @@ func (o *SyncOpts) Apply(opts *SyncOpts) {
 	*opts = *o
 }
 
-// SyncReleases wrapper for executing helm upgrade on the releases
-func (st *HelmState) SyncReleases(affectedReleases *AffectedReleases, helm helmexec.Interface, additionalValues []string, workerLimit int, opt ...SyncOpt) []error {
-	opts := &SyncOpts{}
-	for _, o := range opt {
-		o.Apply(opts)
-	}
-
-	preps, prepErrs := st.prepareSyncReleases(helm, additionalValues, workerLimit, opts)
-	if len(prepErrs) > 0 {
-		return prepErrs
-	}
-
-	availableIds := make([]string, len(preps))
-	idToPrep := map[string]syncPrepareResult{}
-
-	d := dag.New()
-	for i, p := range preps {
-		r := p.release
-
-		id := releaseToID(r)
-
-		idToPrep[id] = p
-
-		availableIds[i] = id
-
-		d.Add(id, dag.Dependencies(r.Needs))
-	}
-
-	plan, err := d.Plan()
-	if err != nil {
-		return []error{err}
-	}
-
-	groupsTotal := len(plan)
-
-	st.logger.Debugf("syncing %d groups of releases in this order: %s", groupsTotal, plan)
-
-	for id, prep := range idToPrep {
-		for _, need := range prep.release.Needs {
-			if _, ok := idToPrep[need]; !ok {
-				return []error{fmt.Errorf("%q needs %q, but it must be one of %s", id, need, strings.Join(availableIds, ", "))}
-			}
-		}
-	}
-
-	for groupIndex, dagNodesInGroup := range plan {
-		var idsInGroup []string
-		var prepsInGroup []syncPrepareResult
-
-		for _, node := range dagNodesInGroup {
-			prepareResult, ok := idToPrep[node.Id]
-			if !ok {
-				panic(fmt.Sprintf("[bug] no release found for dag node id %q", node.Id))
-			}
-			prepsInGroup = append(prepsInGroup, prepareResult)
-			idsInGroup = append(idsInGroup, node.Id)
-		}
-
-		st.logger.Debugf("syncing releases in group %d/%d: %s", groupIndex+1, groupsTotal, strings.Join(idsInGroup, ", "))
-
-		errs := st.syncReleaseGroup(affectedReleases, helm, workerLimit, prepsInGroup)
-		if len(errs) > 0 {
-			return errs
-		}
-	}
-
-	return nil
-}
-
-func releaseToID(r *ReleaseSpec) string {
+func ReleaseToID(r *ReleaseSpec) string {
 	var id string
 
 	tns := r.TillerNamespace
@@ -486,18 +428,112 @@ func releaseToID(r *ReleaseSpec) string {
 	return id
 }
 
-func (st *HelmState) syncReleaseGroup(affectedReleases *AffectedReleases, helm helmexec.Interface, concurrency int, preps []syncPrepareResult) []error {
+// DeleteReleasesForSync deletes releases that are marked for deletion
+func (st *HelmState) DeleteReleasesForSync(affectedReleases *AffectedReleases, helm helmexec.Interface, workerLimit int) []error {
 	errs := []error{}
-	jobQueue := make(chan *syncPrepareResult, len(preps))
-	results := make(chan syncResult, len(preps))
-	if concurrency == 0 {
-		concurrency = len(preps)
+
+	releases := st.Releases
+
+	jobQueue := make(chan *ReleaseSpec, len(releases))
+	results := make(chan syncResult, len(releases))
+	if workerLimit == 0 {
+		workerLimit = len(releases)
 	}
 
 	m := new(sync.Mutex)
 
 	st.scatterGather(
-		concurrency,
+		workerLimit,
+		len(releases),
+		func() {
+			for i := 0; i < len(releases); i++ {
+				jobQueue <- &releases[i]
+			}
+			close(jobQueue)
+		},
+		func(workerIndex int) {
+			for release := range jobQueue {
+				var relErr *ReleaseError
+				context := st.createHelmContext(release, workerIndex)
+
+				if _, err := st.triggerPresyncEvent(release, "sync"); err != nil {
+					relErr = newReleaseError(release, err)
+				} else {
+					var args []string
+					if isHelm3() {
+						args = []string{}
+					} else {
+						args = []string{"--purge"}
+					}
+					deletionFlags := st.appendConnectionFlags(args, release)
+					m.Lock()
+					if err := helm.DeleteRelease(context, release.Name, deletionFlags...); err != nil {
+						affectedReleases.Failed = append(affectedReleases.Failed, release)
+						relErr = newReleaseError(release, err)
+					} else {
+						affectedReleases.Deleted = append(affectedReleases.Deleted, release)
+					}
+					m.Unlock()
+				}
+
+				if relErr == nil {
+					results <- syncResult{}
+				} else {
+					results <- syncResult{errors: []*ReleaseError{relErr}}
+				}
+
+				if _, err := st.triggerPostsyncEvent(release, relErr, "sync"); err != nil {
+					st.logger.Warnf("warn: %v\n", err)
+				}
+
+				if _, err := st.triggerCleanupEvent(release, "sync"); err != nil {
+					st.logger.Warnf("warn: %v\n", err)
+				}
+			}
+		},
+		func() {
+			for i := 0; i < len(releases); {
+				select {
+				case res := <-results:
+					if len(res.errors) > 0 {
+						for _, e := range res.errors {
+							errs = append(errs, e)
+						}
+					}
+				}
+				i++
+			}
+		},
+	)
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}
+
+// SyncReleases wrapper for executing helm upgrade on the releases
+func (st *HelmState) SyncReleases(affectedReleases *AffectedReleases, helm helmexec.Interface, additionalValues []string, workerLimit int, opt ...SyncOpt) []error {
+	opts := &SyncOpts{}
+	for _, o := range opt {
+		o.Apply(opts)
+	}
+
+	preps, prepErrs := st.prepareSyncReleases(helm, additionalValues, workerLimit, opts)
+	if len(prepErrs) > 0 {
+		return prepErrs
+	}
+
+	errs := []error{}
+	jobQueue := make(chan *syncPrepareResult, len(preps))
+	results := make(chan syncResult, len(preps))
+	if workerLimit == 0 {
+		workerLimit = len(preps)
+	}
+
+	m := new(sync.Mutex)
+
+	st.scatterGather(
+		workerLimit,
 		len(preps),
 		func() {
 			for i := 0; i < len(preps); i++ {
@@ -730,7 +766,7 @@ func (st *HelmState) TemplateReleases(helm helmexec.Interface, outputDir string,
 			continue
 		}
 
-		st.applyDefaultsTo(&release)
+		st.ApplyOverrides(&release)
 
 		flags, err := st.flagsForTemplate(helm, &release, 0)
 		if err != nil {
@@ -916,7 +952,7 @@ func (st *HelmState) prepareDiffReleases(helm helmexec.Interface, additionalValu
 			for release := range jobs {
 				errs := []error{}
 
-				st.applyDefaultsTo(release)
+				st.ApplyOverrides(release)
 
 				// TODO We need a long-term fix for this :)
 				// See https://github.com/roboll/helmfile/issues/737
@@ -1020,7 +1056,7 @@ type DiffOpt interface{ Apply(*DiffOpts) }
 
 // DiffReleases wrapper for executing helm diff on the releases
 // It returns releases that had any changes
-func (st *HelmState) DiffReleases(helm helmexec.Interface, additionalValues []string, workerLimit int, detailedExitCode, suppressSecrets bool, triggerCleanupEvents bool, opt ...DiffOpt) ([]*ReleaseSpec, []error) {
+func (st *HelmState) DiffReleases(helm helmexec.Interface, additionalValues []string, workerLimit int, detailedExitCode, suppressSecrets bool, triggerCleanupEvents bool, opt ...DiffOpt) ([]ReleaseSpec, []error) {
 	opts := &DiffOpts{}
 	for _, o := range opt {
 		o.Apply(opts)
@@ -1028,13 +1064,13 @@ func (st *HelmState) DiffReleases(helm helmexec.Interface, additionalValues []st
 
 	preps, prepErrs := st.prepareDiffReleases(helm, additionalValues, workerLimit, detailedExitCode, suppressSecrets, opts)
 	if len(prepErrs) > 0 {
-		return []*ReleaseSpec{}, prepErrs
+		return []ReleaseSpec{}, prepErrs
 	}
 
 	jobQueue := make(chan *diffPrepareResult, len(preps))
 	results := make(chan diffResult, len(preps))
 
-	rs := []*ReleaseSpec{}
+	rs := []ReleaseSpec{}
 	errs := []error{}
 
 	st.scatterGather(
@@ -1076,7 +1112,7 @@ func (st *HelmState) DiffReleases(helm helmexec.Interface, additionalValues []st
 				if res.err != nil {
 					errs = append(errs, res.err)
 					if res.err.Code == 2 {
-						rs = append(rs, res.err.ReleaseSpec)
+						rs = append(rs, *res.err.ReleaseSpec)
 					}
 				}
 			}
@@ -1102,7 +1138,7 @@ func (st *HelmState) ReleaseStatuses(helm helmexec.Interface, workerLimit int) [
 // DeleteReleases wrapper for executing helm delete on the releases
 // This function traverses the DAG of the releases in the reverse order, so that the releases that are NOT depended by any others are deleted first.
 func (st *HelmState) DeleteReleases(affectedReleases *AffectedReleases, helm helmexec.Interface, concurrency int, purge bool) []error {
-	return st.dagAwareReverseIterateOnReleases(helm, concurrency, func(release ReleaseSpec, workerIndex int) error {
+	return st.scatterGatherReleases(helm, concurrency, func(release ReleaseSpec, workerIndex int) error {
 		if !release.Desired() {
 			return nil
 		}
@@ -1180,19 +1216,17 @@ func (st *HelmState) Clean() []error {
 	return nil
 }
 
-// FilterReleases allows for the execution of helm commands against a subset of the releases in the helmfile.
-func (st *HelmState) FilterReleases() error {
-	var filteredReleases []ReleaseSpec
-	releaseSet := map[string][]ReleaseSpec{}
+func MarkFilteredReleases(releases []ReleaseSpec, selectors []string) ([]Release, error) {
+	var filteredReleases []Release
 	filters := []ReleaseFilter{}
-	for _, label := range st.Selectors {
+	for _, label := range selectors {
 		f, err := ParseLabels(label)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		filters = append(filters, f)
 	}
-	for _, r := range st.Releases {
+	for _, r := range releases {
 		if r.Labels == nil {
 			r.Labels = map[string]string{}
 		}
@@ -1202,22 +1236,40 @@ func (st *HelmState) FilterReleases() error {
 		// Strip off just the last portion for the name stable/newrelic would give newrelic
 		chartSplit := strings.Split(r.Chart, "/")
 		r.Labels["chart"] = chartSplit[len(chartSplit)-1]
+		var matched bool
 		for _, f := range filters {
 			if r.Labels == nil {
 				r.Labels = map[string]string{}
 			}
 			if f.Match(r) {
-				releaseSet[r.Name] = append(releaseSet[r.Name], r)
-				continue
+				matched = true
+				break
 			}
 		}
+		res := Release{
+			ReleaseSpec: r,
+			Filtered:    len(filters) > 0 && !matched,
+		}
+		filteredReleases = append(filteredReleases, res)
 	}
-	for _, r := range releaseSet {
-		filteredReleases = append(filteredReleases, r...)
+
+	return filteredReleases, nil
+}
+
+// FilterReleases allows for the execution of helm commands against a subset of the releases in the helmfile.
+func (st *HelmState) FilterReleases() error {
+	filteredReleases, err := MarkFilteredReleases(st.Releases, st.Selectors)
+	if err != nil {
+		return err
 	}
-	st.Releases = filteredReleases
-	numFound := len(filteredReleases)
-	st.logger.Debugf("%d release(s) matching %s found in %s\n", numFound, strings.Join(st.Selectors, ","), st.FilePath)
+	var releases []ReleaseSpec
+	for _, r := range filteredReleases {
+		if !r.Filtered {
+			releases = append(releases, r.ReleaseSpec)
+		}
+	}
+	st.Releases = releases
+	st.logger.Debugf("%d release(s) matching %s found in %s\n", len(filteredReleases), strings.Join(st.Selectors, ","), st.FilePath)
 	return nil
 }
 
@@ -1267,7 +1319,7 @@ func (st *HelmState) triggerReleaseEvent(evt string, evtErr error, r *ReleaseSpe
 		Hooks:         r.Hooks,
 		StateFilePath: st.FilePath,
 		BasePath:      st.basePath,
-		Namespace:     st.Namespace,
+		Namespace:     st.OverrideNamespace,
 		Env:           st.Env,
 		Logger:        st.logger,
 		ReadFile:      st.readFile,

pkg/state/state_exec_tmpl.go

@@ -39,7 +39,7 @@ func (st *HelmState) mustLoadVals() map[string]interface{} {
 func (st *HelmState) valuesFileTemplateData() EnvironmentTemplateData {
 	return EnvironmentTemplateData{
 		Environment: st.Env,
-		Namespace:   st.Namespace,
+		Namespace:   st.OverrideNamespace,
 		Values:      st.mustLoadVals(),
 	}
 }

pkg/state/state_exec_tmpl_test.go

@@ -138,9 +138,9 @@ func TestHelmState_executeTemplates(t *testing.T) {
 				HelmDefaults: HelmSpec{
 					KubeContext: "test_context",
 				},
-				Env:          environment.Environment{Name: "test_env"},
-				Namespace:    "test-namespace_",
-				Repositories: nil,
+				Env:               environment.Environment{Name: "test_env"},
+				OverrideNamespace: "test-namespace_",
+				Repositories:      nil,
 				Releases: []ReleaseSpec{
 					tt.input,
 				},
@@ -238,9 +238,9 @@ func TestHelmState_recursiveRefsTemplates(t *testing.T) {
 				HelmDefaults: HelmSpec{
 					KubeContext: "test_context",
 				},
-				Env:          environment.Environment{Name: "test_env"},
-				Namespace:    "test-namespace_",
-				Repositories: nil,
+				Env:               environment.Environment{Name: "test_env"},
+				OverrideNamespace: "test-namespace_",
+				Repositories:      nil,
 				Releases: []ReleaseSpec{
 					tt.input,
 				},

pkg/state/state_run.go

@@ -2,7 +2,7 @@ package state
 
 import (
 	"fmt"
-	"strings"
+	"sort"
 	"sync"
 
 	"github.com/roboll/helmfile/pkg/helmexec"
@@ -104,51 +104,96 @@ func (st *HelmState) iterateOnReleases(helm helmexec.Interface, concurrency int,
 	return nil
 }
 
-func (st *HelmState) dagAwareReverseIterateOnReleases(helm helmexec.Interface, concurrency int,
-	do func(ReleaseSpec, int) error) []error {
+type PlanOpts struct {
+	Reverse bool
+}
 
-	idToRelease := map[string]ReleaseSpec{}
+func PlanReleases(releases []ReleaseSpec, selectors []string, reverse bool) ([][]Release, error) {
+	marked, err := MarkFilteredReleases(releases, selectors)
+	if err != nil {
+		return nil, err
+	}
 
-	preps := st.Releases
+	groups, err := SortedReleaseGroups(marked, reverse)
+	if err != nil {
+		return nil, err
+	}
+
+	return groups, nil
+}
+
+func SortedReleaseGroups(releases []Release, reverse bool) ([][]Release, error) {
+	groups, err := GroupReleasesByDependency(releases)
+	if err != nil {
+		return nil, err
+	}
+
+	if reverse {
+		sort.Slice(groups, func(i, j int) bool {
+			return j < i
+		})
+	}
+
+	return groups, nil
+}
+
+func GroupReleasesByDependency(releases []Release) ([][]Release, error) {
+	idToReleases := map[string][]Release{}
 
 	d := dag.New()
-	for _, r := range preps {
+	for _, r := range releases {
 
-		id := releaseToID(&r)
+		id := ReleaseToID(&r.ReleaseSpec)
 
-		idToRelease[id] = r
+		idToReleases[id] = append(idToReleases[id], r)
 
-		d.Add(id, dag.Dependencies(r.Needs))
+		// Only compute dependencies from non-filtered releases
+		if !r.Filtered {
+			d.Add(id, dag.Dependencies(r.Needs))
+		}
+	}
+
+	for _, r := range releases {
+		if !r.Filtered {
+			for _, n := range r.Needs {
+				if _, ok := idToReleases[n]; !ok {
+					id := ReleaseToID(&r.ReleaseSpec)
+					return nil, fmt.Errorf("%q has dependency to inexistent release %q", id, n)
+				}
+			}
+		}
 	}
 
 	plan, err := d.Plan()
 	if err != nil {
-		return []error{err}
+		return nil, err
 	}
 
-	groupsTotal := len(plan)
+	var result [][]Release
 
-	st.logger.Debugf("processing %d groups of releases in this order: %s", groupsTotal, plan)
-
-	for groupIndex := len(plan) - 1; groupIndex >= 0; groupIndex-- {
+	for groupIndex := 0; groupIndex < len(plan); groupIndex++ {
 		dagNodesInGroup := plan[groupIndex]
 
 		var idsInGroup []string
-		var releasesInGroup []ReleaseSpec
+		var releasesInGroup []Release
 
 		for _, node := range dagNodesInGroup {
-			releasesInGroup = append(releasesInGroup, idToRelease[node.Id])
 			idsInGroup = append(idsInGroup, node.Id)
 		}
 
-		st.logger.Debugf("processing releases in group %d/%d: %s", groupIndex+1, groupsTotal, strings.Join(idsInGroup, ", "))
+		// Make the helmfile behavior deterministic for reproducibility and ease of testing
+		sort.Strings(idsInGroup)
 
-		errs := st.iterateOnReleases(helm, concurrency, releasesInGroup, do)
-
-		if len(errs) > 0 {
-			return errs
+		for _, id := range idsInGroup {
+			releases, ok := idToReleases[id]
+			if !ok {
+				panic(fmt.Errorf("bug: unexpectedly failed to get releases for id %q", id))
+			}
+			releasesInGroup = append(releasesInGroup, releases...)
 		}
+
+		result = append(result, releasesInGroup)
 	}
 
-	return nil
+	return result, nil
 }

pkg/state/state_test.go

@@ -1,20 +1,17 @@
 package state
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"reflect"
 	"testing"
 
+	"github.com/roboll/helmfile/pkg/exectest"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/testhelper"
 	"github.com/variantdev/vals"
-
-	"errors"
-	"strings"
-
-	"fmt"
 )
 
 var logger = helmexec.NewLogger(os.Stdout, "warn")
@@ -133,12 +130,12 @@ func TestHelmState_applyDefaultsTo(t *testing.T) {
 				basePath:           tt.fields.BaseChartPath,
 				DeprecatedContext:  tt.fields.Context,
 				DeprecatedReleases: tt.fields.DeprecatedReleases,
-				Namespace:          tt.fields.Namespace,
+				OverrideNamespace:  tt.fields.Namespace,
 				Repositories:       tt.fields.Repositories,
 				Releases:           tt.fields.Releases,
 			}
-			if state.applyDefaultsTo(&tt.args.spec); !reflect.DeepEqual(tt.args.spec, tt.want) {
-				t.Errorf("HelmState.applyDefaultsTo() = %v, want %v", tt.args.spec, tt.want)
+			if state.ApplyOverrides(&tt.args.spec); !reflect.DeepEqual(tt.args.spec, tt.want) {
+				t.Errorf("HelmState.ApplyOverrides() = %v, want %v", tt.args.spec, tt.want)
 			}
 		})
 	}
@@ -665,124 +662,11 @@ func Test_normalizeChart(t *testing.T) {
 
 // mocking helmexec.Interface
 
-type listKey struct {
-	filter string
-	flags  string
-}
-
-type mockHelmExec struct {
-	charts   []string
-	repo     []string
-	releases []mockRelease
-	deleted  []mockRelease
-	lists    map[listKey]string
-	diffed   []mockRelease
-
-	updateDepsCallbacks map[string]func(string) error
-}
-
-type mockRelease struct {
-	name  string
-	flags []string
-}
-
-type mockAffected struct {
-	upgraded []*mockRelease
-	deleted  []*mockRelease
-	failed   []*mockRelease
-}
-
-func (helm *mockHelmExec) UpdateDeps(chart string) error {
-	if strings.Contains(chart, "error") {
-		return fmt.Errorf("simulated UpdateDeps failure for chart: %s", chart)
-	}
-	helm.charts = append(helm.charts, chart)
-
-	if helm.updateDepsCallbacks != nil {
-		callback, exists := helm.updateDepsCallbacks[chart]
-		if exists {
-			if err := callback(chart); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func (helm *mockHelmExec) BuildDeps(name, chart string) error {
-	if strings.Contains(chart, "error") {
-		return errors.New("error")
-	}
-	helm.charts = append(helm.charts, chart)
-	return nil
-}
-
-func (helm *mockHelmExec) SetExtraArgs(args ...string) {
-	return
-}
-func (helm *mockHelmExec) SetHelmBinary(bin string) {
-	return
-}
-func (helm *mockHelmExec) AddRepo(name, repository, cafile, certfile, keyfile, username, password string) error {
-	helm.repo = []string{name, repository, cafile, certfile, keyfile, username, password}
-	return nil
-}
-func (helm *mockHelmExec) UpdateRepo() error {
-	return nil
-}
-func (helm *mockHelmExec) SyncRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
-	if strings.Contains(name, "error") {
-		return errors.New("error")
-	}
-	helm.releases = append(helm.releases, mockRelease{name: name, flags: flags})
-	helm.charts = append(helm.charts, chart)
-	return nil
-}
-func (helm *mockHelmExec) DiffRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
-	helm.diffed = append(helm.diffed, mockRelease{name: name, flags: flags})
-	return nil
-}
-func (helm *mockHelmExec) ReleaseStatus(context helmexec.HelmContext, release string, flags ...string) error {
-	if strings.Contains(release, "error") {
-		return errors.New("error")
-	}
-	helm.releases = append(helm.releases, mockRelease{name: release, flags: flags})
-	return nil
-}
-func (helm *mockHelmExec) DeleteRelease(context helmexec.HelmContext, name string, flags ...string) error {
-	if strings.Contains(name, "error") {
-		return errors.New("error")
-	}
-	helm.deleted = append(helm.deleted, mockRelease{name: name, flags: flags})
-	return nil
-}
-func (helm *mockHelmExec) List(context helmexec.HelmContext, filter string, flags ...string) (string, error) {
-	return helm.lists[listKey{filter: filter, flags: strings.Join(flags, "")}], nil
-}
-func (helm *mockHelmExec) DecryptSecret(context helmexec.HelmContext, name string, flags ...string) (string, error) {
-	return "", nil
-}
-func (helm *mockHelmExec) TestRelease(context helmexec.HelmContext, name string, flags ...string) error {
-	if strings.Contains(name, "error") {
-		return errors.New("error")
-	}
-	helm.releases = append(helm.releases, mockRelease{name: name, flags: flags})
-	return nil
-}
-func (helm *mockHelmExec) Fetch(chart string, flags ...string) error {
-	return nil
-}
-func (helm *mockHelmExec) Lint(name, chart string, flags ...string) error {
-	return nil
-}
-func (helm *mockHelmExec) TemplateRelease(name, chart string, flags ...string) error {
-	return nil
-}
 func TestHelmState_SyncRepos(t *testing.T) {
 	tests := []struct {
 		name  string
 		repos []RepositorySpec
-		helm  *mockHelmExec
+		helm  *exectest.Helm
 		envs  map[string]string
 		want  []string
 	}{
@@ -798,7 +682,7 @@ func TestHelmState_SyncRepos(t *testing.T) {
 					Password: "",
 				},
 			},
-			helm: &mockHelmExec{},
+			helm: &exectest.Helm{},
 			want: []string{"name", "http://example.com/", "", "", "", "", ""},
 		},
 		{
@@ -813,7 +697,7 @@ func TestHelmState_SyncRepos(t *testing.T) {
 					Password: "",
 				},
 			},
-			helm: &mockHelmExec{},
+			helm: &exectest.Helm{},
 			want: []string{"name", "http://example.com/", "", "certfile", "keyfile", "", ""},
 		},
 		{
@@ -827,7 +711,7 @@ func TestHelmState_SyncRepos(t *testing.T) {
 					Password: "",
 				},
 			},
-			helm: &mockHelmExec{},
+			helm: &exectest.Helm{},
 			want: []string{"name", "http://example.com/", "cafile", "", "", "", ""},
 		},
 		{
@@ -842,7 +726,7 @@ func TestHelmState_SyncRepos(t *testing.T) {
 					Password: "example_password",
 				},
 			},
-			helm: &mockHelmExec{},
+			helm: &exectest.Helm{},
 			want: []string{"name", "http://example.com/", "", "", "", "example_user", "example_password"},
 		},
 	}
@@ -858,8 +742,8 @@ func TestHelmState_SyncRepos(t *testing.T) {
 			state := &HelmState{
 				Repositories: tt.repos,
 			}
-			if _ = state.SyncRepos(tt.helm); !reflect.DeepEqual(tt.helm.repo, tt.want) {
-				t.Errorf("HelmState.SyncRepos() for [%s] = %v, want %v", tt.name, tt.helm.repo, tt.want)
+			if _ = state.SyncRepos(tt.helm); !reflect.DeepEqual(tt.helm.Repo, tt.want) {
+				t.Errorf("HelmState.SyncRepos() for [%s] = %v, want %v", tt.name, tt.helm.Repo, tt.want)
 			}
 		})
 	}
@@ -869,8 +753,8 @@ func TestHelmState_SyncReleases(t *testing.T) {
 	tests := []struct {
 		name          string
 		releases      []ReleaseSpec
-		helm          *mockHelmExec
-		wantReleases  []mockRelease
+		helm          *exectest.Helm
+		wantReleases  []exectest.Release
 		wantErrorMsgs []string
 	}{
 		{
@@ -881,8 +765,8 @@ func TestHelmState_SyncReleases(t *testing.T) {
 					Chart: "foo",
 				},
 			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"releaseName", []string{}}},
+			helm:         &exectest.Helm{},
+			wantReleases: []exectest.Release{{Name: "releaseName", Flags: []string{}}},
 		},
 		{
 			name: "with tiller args",
@@ -893,8 +777,8 @@ func TestHelmState_SyncReleases(t *testing.T) {
 					TillerNamespace: "tillerns",
 				},
 			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"releaseName", []string{"--tiller-namespace", "tillerns"}}},
+			helm:         &exectest.Helm{},
+			wantReleases: []exectest.Release{{Name: "releaseName", Flags: []string{"--tiller-namespace", "tillerns"}}},
 		},
 		{
 			name: "escaped values",
@@ -914,8 +798,8 @@ func TestHelmState_SyncReleases(t *testing.T) {
 					},
 				},
 			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"releaseName", []string{"--set", "someList=a\\,b\\,c", "--set", "json=\\{\"name\": \"john\"\\}"}}},
+			helm:         &exectest.Helm{},
+			wantReleases: []exectest.Release{{Name: "releaseName", Flags: []string{"--set", "someList=a\\,b\\,c", "--set", "json=\\{\"name\": \"john\"\\}"}}},
 		},
 		{
 			name: "set single value from file",
@@ -939,8 +823,8 @@ func TestHelmState_SyncReleases(t *testing.T) {
 					},
 				},
 			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"releaseName", []string{"--set", "foo=FOO", "--set-file", "bar=path/to/bar", "--set", "baz=BAZ"}}},
+			helm:         &exectest.Helm{},
+			wantReleases: []exectest.Release{{Name: "releaseName", Flags: []string{"--set", "foo=FOO", "--set-file", "bar=path/to/bar", "--set", "baz=BAZ"}}},
 		},
 		{
 			name: "set single array value in an array",
@@ -959,108 +843,8 @@ func TestHelmState_SyncReleases(t *testing.T) {
 					},
 				},
 			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"releaseName", []string{"--set", "foo.bar[0]={A,B}"}}},
-		},
-		{
-			name: "foo needs bar",
-			releases: []ReleaseSpec{
-				{
-					Name:  "foo",
-					Chart: "charts/foo",
-					Needs: []string{
-						"bar",
-					},
-				},
-				{
-					Name:  "bar",
-					Chart: "charts/bar",
-				},
-			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"bar", []string{}}, {"foo", []string{}}},
-		},
-		{
-			name: "bar needs foo",
-			releases: []ReleaseSpec{
-				{
-					Name:  "foo",
-					Chart: "charts/foo",
-				},
-				{
-					Name:  "bar",
-					Chart: "charts/bar",
-					Needs: []string{
-						"foo",
-					},
-				},
-			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"foo", []string{}}, {"bar", []string{}}},
-		},
-		{
-			name: "ns2/bar needs ns1/foo",
-			releases: []ReleaseSpec{
-				{
-					Name:      "foo",
-					Namespace: "ns1",
-					Chart:     "charts/foo",
-				},
-				{
-					Name:      "bar",
-					Namespace: "ns2",
-					Chart:     "charts/bar",
-					Needs: []string{
-						"ns1/foo",
-					},
-				},
-			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"foo", []string{"--namespace", "ns1"}}, {"bar", []string{"--namespace", "ns2"}}},
-		},
-		{
-			name: "tillerns1/ns1/foo needs tillerns2/ns2/bar",
-			releases: []ReleaseSpec{
-				{
-					Name:            "foo",
-					Chart:           "charts/foo",
-					Namespace:       "ns1",
-					TillerNamespace: "tillerns1",
-					Needs: []string{
-						"tillerns2/ns2/bar",
-					},
-				},
-				{
-					Name:            "bar",
-					Namespace:       "ns2",
-					TillerNamespace: "tillerns2",
-					Chart:           "charts/bar",
-				},
-			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"bar", []string{"--tiller-namespace", "tillerns2", "--namespace", "ns2"}}, {"foo", []string{"--tiller-namespace", "tillerns1", "--namespace", "ns1"}}},
-		},
-		{
-			name: "tillerns1/ns1/foo needs tillerns2/ns2/bar",
-			releases: []ReleaseSpec{
-				{
-					Name:            "foo",
-					Chart:           "charts/foo",
-					Namespace:       "ns1",
-					TillerNamespace: "tillerns1",
-					Needs: []string{
-						"bar",
-					},
-				},
-				{
-					Name:            "bar",
-					Namespace:       "ns2",
-					TillerNamespace: "tillerns2",
-					Chart:           "charts/bar",
-				},
-			},
-			helm:          &mockHelmExec{},
-			wantErrorMsgs: []string{`"tillerns1/ns1/foo" needs "bar", but it must be one of tillerns1/ns1/foo, tillerns2/ns2/bar`},
+			helm:         &exectest.Helm{},
+			wantReleases: []exectest.Release{{Name: "releaseName", Flags: []string{"--set", "foo.bar[0]={A,B}"}}},
 		},
 	}
 	for i := range tests {
@@ -1087,8 +871,8 @@ func TestHelmState_SyncReleases(t *testing.T) {
 					t.Fatalf("%d unexpected errors detected", mismatch)
 				}
 			}
-			if !reflect.DeepEqual(tt.helm.releases, tt.wantReleases) {
-				t.Errorf("HelmState.SyncReleases() for [%s] = %v, want %v", tt.name, tt.helm.releases, tt.wantReleases)
+			if !reflect.DeepEqual(tt.helm.Releases, tt.wantReleases) {
+				t.Errorf("HelmState.SyncReleases() for [%s] = %v, want %v", tt.name, tt.helm.Releases, tt.wantReleases)
 			}
 		})
 	}
@@ -1177,11 +961,11 @@ func TestHelmState_SyncReleases_MissingValuesFileForUndesiredRelease(t *testing.
 			}
 			fs := testhelper.NewTestFs(map[string]string{})
 			state = injectFs(state, fs)
-			helm := &mockHelmExec{
-				lists: map[listKey]string{},
+			helm := &exectest.Helm{
+				Lists: map[exectest.ListKey]string{},
 			}
 			//simulate the helm.list call result
-			helm.lists[listKey{filter: "^" + tt.release.Name + "$"}] = tt.listResult
+			helm.Lists[exectest.ListKey{Filter: "^" + tt.release.Name + "$"}] = tt.listResult
 
 			affectedReleases := AffectedReleases{}
 			errs := state.SyncReleases(&affectedReleases, helm, []string{}, 1)
@@ -1213,7 +997,7 @@ func TestHelmState_SyncReleasesAffectedRealeases(t *testing.T) {
 		name         string
 		releases     []ReleaseSpec
 		installed    []bool
-		wantAffected mockAffected
+		wantAffected exectest.Affected
 	}{
 		{
 			name: "2 release",
@@ -1227,7 +1011,14 @@ func TestHelmState_SyncReleasesAffectedRealeases(t *testing.T) {
 					Chart: "bar",
 				},
 			},
-			wantAffected: mockAffected{[]*mockRelease{{"releaseNameFoo", []string{}}, {"releaseNameBar", []string{}}}, nil, nil},
+			wantAffected: exectest.Affected{
+				Upgraded: []*exectest.Release{
+					{Name: "releaseNameFoo", Flags: []string{}},
+					{Name: "releaseNameBar", Flags: []string{}},
+				},
+				Deleted: nil,
+				Failed:  nil,
+			},
 		},
 		{
 			name: "2 removed",
@@ -1243,8 +1034,15 @@ func TestHelmState_SyncReleasesAffectedRealeases(t *testing.T) {
 					Installed: &no,
 				},
 			},
-			installed:    []bool{true, true},
-			wantAffected: mockAffected{nil, []*mockRelease{{"releaseNameFoo", []string{}}, {"releaseNameBar", []string{}}}, nil},
+			installed: []bool{true, true},
+			wantAffected: exectest.Affected{
+				Upgraded: nil,
+				Deleted: []*exectest.Release{
+					{Name: "releaseNameFoo", Flags: []string{}},
+					{Name: "releaseNameBar", Flags: []string{}},
+				},
+				Failed: nil,
+			},
 		},
 		{
 			name: "2 errors",
@@ -1258,7 +1056,14 @@ func TestHelmState_SyncReleasesAffectedRealeases(t *testing.T) {
 					Chart: "foo",
 				},
 			},
-			wantAffected: mockAffected{nil, nil, []*mockRelease{{"releaseNameFoo-error", []string{}}, {"releaseNameBar-error", []string{}}}},
+			wantAffected: exectest.Affected{
+				Upgraded: nil,
+				Deleted:  nil,
+				Failed: []*exectest.Release{
+					{Name: "releaseNameFoo-error", Flags: []string{}},
+					{Name: "releaseNameBar-error", Flags: []string{}},
+				},
+			},
 		},
 		{
 			name: "1 removed, 1 new, 1 error",
@@ -1277,8 +1082,18 @@ func TestHelmState_SyncReleasesAffectedRealeases(t *testing.T) {
 					Chart: "foo",
 				},
 			},
-			installed:    []bool{true, true, true},
-			wantAffected: mockAffected{[]*mockRelease{{"releaseNameFoo", []string{}}}, []*mockRelease{{"releaseNameBar", []string{}}}, []*mockRelease{{"releaseNameFoo-error", []string{}}}},
+			installed: []bool{true, true, true},
+			wantAffected: exectest.Affected{
+				Upgraded: []*exectest.Release{
+					{Name: "releaseNameFoo", Flags: []string{}},
+				},
+				Deleted: []*exectest.Release{
+					{Name: "releaseNameBar", Flags: []string{}},
+				},
+				Failed: []*exectest.Release{
+					{Name: "releaseNameFoo-error", Flags: []string{}},
+				},
+			},
 		},
 	}
 	for i := range tests {
@@ -1289,33 +1104,33 @@ func TestHelmState_SyncReleasesAffectedRealeases(t *testing.T) {
 				logger:      logger,
 				valsRuntime: valsRuntime,
 			}
-			helm := &mockHelmExec{
-				lists: map[listKey]string{},
+			helm := &exectest.Helm{
+				Lists: map[exectest.ListKey]string{},
 			}
 			//simulate the release is already installed
 			for i, release := range tt.releases {
 				if tt.installed != nil && tt.installed[i] {
-					helm.lists[listKey{filter: "^" + release.Name + "$"}] = release.Name
+					helm.Lists[exectest.ListKey{Filter: "^" + release.Name + "$"}] = release.Name
 				}
 			}
 
 			affectedReleases := AffectedReleases{}
 			if err := state.SyncReleases(&affectedReleases, helm, []string{}, 1); err != nil {
-				if !testEq(affectedReleases.Failed, tt.wantAffected.failed) {
-					t.Errorf("HelmState.SynchAffectedRelease() error failed for [%s] = %v, want %v", tt.name, affectedReleases.Failed, tt.wantAffected.failed)
+				if !testEq(affectedReleases.Failed, tt.wantAffected.Failed) {
+					t.Errorf("HelmState.SynchAffectedRelease() error failed for [%s] = %v, want %v", tt.name, affectedReleases.Failed, tt.wantAffected.Failed)
 				} //else expected error
 			}
-			if !testEq(affectedReleases.Upgraded, tt.wantAffected.upgraded) {
-				t.Errorf("HelmState.SynchAffectedRelease() upgrade failed for [%s] = %v, want %v", tt.name, affectedReleases.Upgraded, tt.wantAffected.upgraded)
+			if !testEq(affectedReleases.Upgraded, tt.wantAffected.Upgraded) {
+				t.Errorf("HelmState.SynchAffectedRelease() upgrade failed for [%s] = %v, want %v", tt.name, affectedReleases.Upgraded, tt.wantAffected.Upgraded)
 			}
-			if !testEq(affectedReleases.Deleted, tt.wantAffected.deleted) {
-				t.Errorf("HelmState.SynchAffectedRelease() deleted failed for [%s] = %v, want %v", tt.name, affectedReleases.Deleted, tt.wantAffected.deleted)
+			if !testEq(affectedReleases.Deleted, tt.wantAffected.Deleted) {
+				t.Errorf("HelmState.SynchAffectedRelease() deleted failed for [%s] = %v, want %v", tt.name, affectedReleases.Deleted, tt.wantAffected.Deleted)
 			}
 		})
 	}
 }
 
-func testEq(a []*ReleaseSpec, b []*mockRelease) bool {
+func testEq(a []*ReleaseSpec, b []*exectest.Release) bool {
 
 	// If one is nil, the other must also be nil.
 	if (a == nil) != (b == nil) {
@@ -1327,7 +1142,7 @@ func testEq(a []*ReleaseSpec, b []*mockRelease) bool {
 	}
 
 	for i := range a {
-		if a[i].Name != b[i].name {
+		if a[i].Name != b[i].Name {
 			return false
 		}
 	}
@@ -1391,11 +1206,11 @@ func TestGetDeployedVersion(t *testing.T) {
 				logger:      logger,
 				valsRuntime: valsRuntime,
 			}
-			helm := &mockHelmExec{
-				lists: map[listKey]string{},
+			helm := &exectest.Helm{
+				Lists: map[exectest.ListKey]string{},
 			}
 			//simulate the helm.list call result
-			helm.lists[listKey{filter: "^" + tt.release.Name + "$"}] = tt.listResult
+			helm.Lists[exectest.ListKey{Filter: "^" + tt.release.Name + "$"}] = tt.listResult
 
 			affectedReleases := AffectedReleases{}
 			state.SyncReleases(&affectedReleases, helm, []string{}, 1)
@@ -1411,8 +1226,8 @@ func TestHelmState_DiffReleases(t *testing.T) {
 	tests := []struct {
 		name         string
 		releases     []ReleaseSpec
-		helm         *mockHelmExec
-		wantReleases []mockRelease
+		helm         *exectest.Helm
+		wantReleases []exectest.Release
 	}{
 		{
 			name: "normal release",
@@ -1422,8 +1237,8 @@ func TestHelmState_DiffReleases(t *testing.T) {
 					Chart: "foo",
 				},
 			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"releaseName", []string{}}},
+			helm:         &exectest.Helm{},
+			wantReleases: []exectest.Release{{Name: "releaseName", Flags: []string{}}},
 		},
 		{
 			name: "with tiller args",
@@ -1434,8 +1249,8 @@ func TestHelmState_DiffReleases(t *testing.T) {
 					TillerNamespace: "tillerns",
 				},
 			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"releaseName", []string{"--tiller-namespace", "tillerns"}}},
+			helm:         &exectest.Helm{},
+			wantReleases: []exectest.Release{{Name: "releaseName", Flags: []string{"--tiller-namespace", "tillerns"}}},
 		},
 		{
 			name: "escaped values",
@@ -1455,8 +1270,10 @@ func TestHelmState_DiffReleases(t *testing.T) {
 					},
 				},
 			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"releaseName", []string{"--set", "someList=a\\,b\\,c", "--set", "json=\\{\"name\": \"john\"\\}"}}},
+			helm: &exectest.Helm{},
+			wantReleases: []exectest.Release{
+				{Name: "releaseName", Flags: []string{"--set", "someList=a\\,b\\,c", "--set", "json=\\{\"name\": \"john\"\\}"}},
+			},
 		},
 		{
 			name: "set single value from file",
@@ -1480,8 +1297,10 @@ func TestHelmState_DiffReleases(t *testing.T) {
 					},
 				},
 			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"releaseName", []string{"--set", "foo=FOO", "--set-file", "bar=path/to/bar", "--set", "baz=BAZ"}}},
+			helm: &exectest.Helm{},
+			wantReleases: []exectest.Release{
+				{Name: "releaseName", Flags: []string{"--set", "foo=FOO", "--set-file", "bar=path/to/bar", "--set", "baz=BAZ"}},
+			},
 		},
 		{
 			name: "set single array value in an array",
@@ -1500,8 +1319,10 @@ func TestHelmState_DiffReleases(t *testing.T) {
 					},
 				},
 			},
-			helm:         &mockHelmExec{},
-			wantReleases: []mockRelease{{"releaseName", []string{"--set", "foo.bar[0]={A,B}"}}},
+			helm: &exectest.Helm{},
+			wantReleases: []exectest.Release{
+				{Name: "releaseName", Flags: []string{"--set", "foo.bar[0]={A,B}"}},
+			},
 		},
 	}
 	for i := range tests {
@@ -1516,8 +1337,8 @@ func TestHelmState_DiffReleases(t *testing.T) {
 			if errs != nil && len(errs) > 0 {
 				t.Errorf("unexpected error: %v", errs)
 			}
-			if !reflect.DeepEqual(tt.helm.diffed, tt.wantReleases) {
-				t.Errorf("HelmState.DiffReleases() for [%s] = %v, want %v", tt.name, tt.helm.releases, tt.wantReleases)
+			if !reflect.DeepEqual(tt.helm.Diffed, tt.wantReleases) {
+				t.Errorf("HelmState.DiffReleases() for [%s] = %v, want %v", tt.name, tt.helm.Releases, tt.wantReleases)
 			}
 		})
 	}
@@ -1527,7 +1348,7 @@ func TestHelmState_SyncReleasesCleanup(t *testing.T) {
 	tests := []struct {
 		name                    string
 		releases                []ReleaseSpec
-		helm                    *mockHelmExec
+		helm                    *exectest.Helm
 		expectedNumRemovedFiles int
 	}{
 		{
@@ -1538,7 +1359,7 @@ func TestHelmState_SyncReleasesCleanup(t *testing.T) {
 					Chart: "foo",
 				},
 			},
-			helm:                    &mockHelmExec{},
+			helm:                    &exectest.Helm{},
 			expectedNumRemovedFiles: 0,
 		},
 		{
@@ -1554,7 +1375,7 @@ func TestHelmState_SyncReleasesCleanup(t *testing.T) {
 					},
 				},
 			},
-			helm:                    &mockHelmExec{},
+			helm:                    &exectest.Helm{},
 			expectedNumRemovedFiles: 1,
 		},
 		{
@@ -1571,7 +1392,7 @@ func TestHelmState_SyncReleasesCleanup(t *testing.T) {
 					},
 				},
 			},
-			helm:                    &mockHelmExec{},
+			helm:                    &exectest.Helm{},
 			expectedNumRemovedFiles: 2,
 		},
 	}
@@ -1611,7 +1432,7 @@ func TestHelmState_DiffReleasesCleanup(t *testing.T) {
 	tests := []struct {
 		name                    string
 		releases                []ReleaseSpec
-		helm                    *mockHelmExec
+		helm                    *exectest.Helm
 		expectedNumRemovedFiles int
 	}{
 		{
@@ -1622,7 +1443,7 @@ func TestHelmState_DiffReleasesCleanup(t *testing.T) {
 					Chart: "foo",
 				},
 			},
-			helm:                    &mockHelmExec{},
+			helm:                    &exectest.Helm{},
 			expectedNumRemovedFiles: 0,
 		},
 		{
@@ -1638,7 +1459,7 @@ func TestHelmState_DiffReleasesCleanup(t *testing.T) {
 					},
 				},
 			},
-			helm:                    &mockHelmExec{},
+			helm:                    &exectest.Helm{},
 			expectedNumRemovedFiles: 1,
 		},
 		{
@@ -1655,7 +1476,7 @@ func TestHelmState_DiffReleasesCleanup(t *testing.T) {
 					},
 				},
 			},
-			helm:                    &mockHelmExec{},
+			helm:                    &exectest.Helm{},
 			expectedNumRemovedFiles: 2,
 		},
 	}
@@ -1693,8 +1514,8 @@ func TestHelmState_DiffReleasesCleanup(t *testing.T) {
 }
 
 func TestHelmState_UpdateDeps(t *testing.T) {
-	helm := &mockHelmExec{
-		updateDepsCallbacks: map[string]func(string) error{},
+	helm := &exectest.Helm{
+		UpdateDepsCallbacks: map[string]func(string) error{},
 	}
 
 	var generatedDir string
@@ -1704,7 +1525,7 @@ func TestHelmState_UpdateDeps(t *testing.T) {
 		if err != nil {
 			return "", err
 		}
-		helm.updateDepsCallbacks[generatedDir] = func(chart string) error {
+		helm.UpdateDepsCallbacks[generatedDir] = func(chart string) error {
 			content := []byte(`dependencies:
 - name: envoy
   repository: https://kubernetes-charts.storage.googleapis.com
@@ -1763,8 +1584,8 @@ generated: 2019-05-16T15:42:45.50486+09:00
 
 	errs := state.UpdateDeps(helm)
 	want := []string{"/", "/examples", "/helmfile", "/src/published", generatedDir}
-	if !reflect.DeepEqual(helm.charts, want) {
-		t.Errorf("HelmState.UpdateDeps() = %v, want %v", helm.charts, want)
+	if !reflect.DeepEqual(helm.Charts, want) {
+		t.Errorf("HelmState.UpdateDeps() = %v, want %v", helm.Charts, want)
 	}
 	if len(errs) != 0 {
 		t.Errorf("HelmState.UpdateDeps() - no errors, but got %d: %v", len(errs), errs)
@@ -1833,8 +1654,8 @@ func TestHelmState_ReleaseStatuses(t *testing.T) {
 	tests := []struct {
 		name     string
 		releases []ReleaseSpec
-		helm     *mockHelmExec
-		want     []mockRelease
+		helm     *exectest.Helm
+		want     []exectest.Release
 		wantErr  bool
 	}{
 		{
@@ -1844,8 +1665,10 @@ func TestHelmState_ReleaseStatuses(t *testing.T) {
 					Name: "releaseA",
 				},
 			},
-			helm: &mockHelmExec{},
-			want: []mockRelease{{"releaseA", []string{}}},
+			helm: &exectest.Helm{},
+			want: []exectest.Release{
+				{Name: "releaseA", Flags: []string{}},
+			},
 		},
 		{
 			name: "happy path",
@@ -1854,7 +1677,7 @@ func TestHelmState_ReleaseStatuses(t *testing.T) {
 					Name: "error",
 				},
 			},
-			helm:    &mockHelmExec{},
+			helm:    &exectest.Helm{},
 			wantErr: true,
 		},
 		{
@@ -1867,7 +1690,7 @@ func TestHelmState_ReleaseStatuses(t *testing.T) {
 					},
 				},
 			},
-			helm:    &mockHelmExec{},
+			helm:    &exectest.Helm{},
 			wantErr: true,
 		},
 		{
@@ -1881,7 +1704,7 @@ func TestHelmState_ReleaseStatuses(t *testing.T) {
 					Installed: boolValue(false),
 				},
 			},
-			helm:    &mockHelmExec{},
+			helm:    &exectest.Helm{},
 			wantErr: false,
 		},
 		{
@@ -1892,8 +1715,10 @@ func TestHelmState_ReleaseStatuses(t *testing.T) {
 					TillerNamespace: "tillerns",
 				},
 			},
-			helm: &mockHelmExec{},
-			want: []mockRelease{{"releaseA", []string{"--tiller-namespace", "tillerns"}}},
+			helm: &exectest.Helm{},
+			want: []exectest.Release{
+				{Name: "releaseA", Flags: []string{"--tiller-namespace", "tillerns"}},
+			},
 		},
 	}
 	for i := range tests {
@@ -1920,8 +1745,8 @@ func TestHelmState_ReleaseStatuses(t *testing.T) {
 				t.Errorf("ReleaseStatuses() for %s error = %v, wantErr %v", tt.name, errs, tt.wantErr)
 				return
 			}
-			if !reflect.DeepEqual(tt.helm.releases, tt.want) {
-				t.Errorf("HelmState.ReleaseStatuses() for [%s] = %v, want %v", tt.name, tt.helm.releases, tt.want)
+			if !reflect.DeepEqual(tt.helm.Releases, tt.want) {
+				t.Errorf("HelmState.ReleaseStatuses() for [%s] = %v, want %v", tt.name, tt.helm.Releases, tt.want)
 			}
 		}
 		t.Run(tt.name, f)
@@ -1933,8 +1758,8 @@ func TestHelmState_TestReleasesNoCleanUp(t *testing.T) {
 		name            string
 		cleanup         bool
 		releases        []ReleaseSpec
-		helm            *mockHelmExec
-		want            []mockRelease
+		helm            *exectest.Helm
+		want            []exectest.Release
 		wantErr         bool
 		tillerNamespace string
 	}{
@@ -1945,8 +1770,8 @@ func TestHelmState_TestReleasesNoCleanUp(t *testing.T) {
 					Name: "releaseA",
 				},
 			},
-			helm: &mockHelmExec{},
-			want: []mockRelease{{"releaseA", []string{"--timeout", "1"}}},
+			helm: &exectest.Helm{},
+			want: []exectest.Release{{Name: "releaseA", Flags: []string{"--timeout", "1"}}},
 		},
 		{
 			name:    "do cleanup",
@@ -1956,8 +1781,8 @@ func TestHelmState_TestReleasesNoCleanUp(t *testing.T) {
 					Name: "releaseB",
 				},
 			},
-			helm: &mockHelmExec{},
-			want: []mockRelease{{"releaseB", []string{"--cleanup", "--timeout", "1"}}},
+			helm: &exectest.Helm{},
+			want: []exectest.Release{{Name: "releaseB", Flags: []string{"--cleanup", "--timeout", "1"}}},
 		},
 		{
 			name: "happy path",
@@ -1966,7 +1791,7 @@ func TestHelmState_TestReleasesNoCleanUp(t *testing.T) {
 					Name: "error",
 				},
 			},
-			helm:    &mockHelmExec{},
+			helm:    &exectest.Helm{},
 			wantErr: true,
 		},
 		{
@@ -1977,8 +1802,8 @@ func TestHelmState_TestReleasesNoCleanUp(t *testing.T) {
 					TillerNamespace: "tillerns",
 				},
 			},
-			helm: &mockHelmExec{},
-			want: []mockRelease{{"releaseA", []string{"--timeout", "1", "--tiller-namespace", "tillerns"}}},
+			helm: &exectest.Helm{},
+			want: []exectest.Release{{Name: "releaseA", Flags: []string{"--timeout", "1", "--tiller-namespace", "tillerns"}}},
 		},
 	}
 	for i := range tests {
@@ -1993,8 +1818,8 @@ func TestHelmState_TestReleasesNoCleanUp(t *testing.T) {
 				t.Errorf("TestReleases() for %s error = %v, wantErr %v", tt.name, errs, tt.wantErr)
 				return
 			}
-			if !reflect.DeepEqual(tt.helm.releases, tt.want) {
-				t.Errorf("HelmState.TestReleases() for [%s] = %v, want %v", tt.name, tt.helm.releases, tt.want)
+			if !reflect.DeepEqual(tt.helm.Releases, tt.want) {
+				t.Errorf("HelmState.TestReleases() for [%s] = %v, want %v", tt.name, tt.helm.Releases, tt.want)
 			}
 		}
 		t.Run(tt.name, f)
@@ -2053,7 +1878,7 @@ func TestHelmState_NoReleaseMatched(t *testing.T) {
 func TestHelmState_Delete(t *testing.T) {
 	tests := []struct {
 		name            string
-		deleted         []mockRelease
+		deleted         []exectest.Release
 		wantErr         bool
 		desired         *bool
 		installed       bool
@@ -2069,7 +1894,7 @@ func TestHelmState_Delete(t *testing.T) {
 			desired:   boolValue(true),
 			installed: true,
 			purge:     false,
-			deleted:   []mockRelease{{"releaseA", []string{}}},
+			deleted:   []exectest.Release{{Name: "releaseA", Flags: []string{}}},
 		},
 		{
 			name:      "desired(default) and installed (purge=false)",
@@ -2077,7 +1902,7 @@ func TestHelmState_Delete(t *testing.T) {
 			desired:   nil,
 			installed: true,
 			purge:     false,
-			deleted:   []mockRelease{{"releaseA", []string{}}},
+			deleted:   []exectest.Release{{Name: "releaseA", Flags: []string{}}},
 		},
 		{
 			name:      "desired(default) and installed (purge=false) but error",
@@ -2085,7 +1910,7 @@ func TestHelmState_Delete(t *testing.T) {
 			desired:   nil,
 			installed: true,
 			purge:     false,
-			deleted:   []mockRelease{{"releaseA", []string{}}},
+			deleted:   []exectest.Release{{Name: "releaseA", Flags: []string{}}},
 		},
 		{
 			name:      "desired and installed (purge=true)",
@@ -2093,7 +1918,7 @@ func TestHelmState_Delete(t *testing.T) {
 			desired:   boolValue(true),
 			installed: true,
 			purge:     true,
-			deleted:   []mockRelease{{"releaseA", []string{"--purge"}}},
+			deleted:   []exectest.Release{{Name: "releaseA", Flags: []string{"--purge"}}},
 		},
 		{
 			name:      "desired but not installed (purge=false)",
@@ -2101,7 +1926,7 @@ func TestHelmState_Delete(t *testing.T) {
 			desired:   boolValue(true),
 			installed: false,
 			purge:     false,
-			deleted:   []mockRelease{},
+			deleted:   []exectest.Release{},
 		},
 		{
 			name:      "desired but not installed (purge=true)",
@@ -2109,7 +1934,7 @@ func TestHelmState_Delete(t *testing.T) {
 			desired:   boolValue(true),
 			installed: false,
 			purge:     true,
-			deleted:   []mockRelease{},
+			deleted:   []exectest.Release{},
 		},
 		{
 			name:      "installed but filtered (purge=false)",
@@ -2117,7 +1942,7 @@ func TestHelmState_Delete(t *testing.T) {
 			desired:   boolValue(false),
 			installed: true,
 			purge:     false,
-			deleted:   []mockRelease{},
+			deleted:   []exectest.Release{},
 		},
 		{
 			name:      "installed but filtered (purge=true)",
@@ -2125,7 +1950,7 @@ func TestHelmState_Delete(t *testing.T) {
 			desired:   boolValue(false),
 			installed: true,
 			purge:     true,
-			deleted:   []mockRelease{},
+			deleted:   []exectest.Release{},
 		},
 		{
 			name:      "not installed, and filtered (purge=false)",
@@ -2133,7 +1958,7 @@ func TestHelmState_Delete(t *testing.T) {
 			desired:   boolValue(false),
 			installed: false,
 			purge:     false,
-			deleted:   []mockRelease{},
+			deleted:   []exectest.Release{},
 		},
 		{
 			name:      "not installed, and filtered (purge=true)",
@@ -2141,7 +1966,7 @@ func TestHelmState_Delete(t *testing.T) {
 			desired:   boolValue(false),
 			installed: false,
 			purge:     true,
-			deleted:   []mockRelease{},
+			deleted:   []exectest.Release{},
 		},
 		{
 			name:            "with tiller args",
@@ -2151,7 +1976,7 @@ func TestHelmState_Delete(t *testing.T) {
 			purge:           true,
 			tillerNamespace: "tillerns",
 			flags:           "--tiller-namespacetillerns",
-			deleted:         []mockRelease{{"releaseA", []string{"--purge", "--tiller-namespace", "tillerns"}}},
+			deleted:         []exectest.Release{{Name: "releaseA", Flags: []string{"--purge", "--tiller-namespace", "tillerns"}}},
 		},
 		{
 			name:        "with kubecontext",
@@ -2161,7 +1986,7 @@ func TestHelmState_Delete(t *testing.T) {
 			purge:       true,
 			kubeContext: "ctx",
 			flags:       "--kube-contextctx",
-			deleted:     []mockRelease{{"releaseA", []string{"--purge", "--kube-context", "ctx"}}},
+			deleted:     []exectest.Release{{Name: "releaseA", Flags: []string{"--purge", "--kube-context", "ctx"}}},
 		},
 		{
 			name:           "with default kubecontext",
@@ -2171,7 +1996,7 @@ func TestHelmState_Delete(t *testing.T) {
 			purge:          true,
 			defKubeContext: "defctx",
 			flags:          "--kube-contextdefctx",
-			deleted:        []mockRelease{{"releaseA", []string{"--purge", "--kube-context", "defctx"}}},
+			deleted:        []exectest.Release{{Name: "releaseA", Flags: []string{"--purge", "--kube-context", "defctx"}}},
 		},
 		{
 			name:           "with non-default and default kubecontexts",
@@ -2182,7 +2007,7 @@ func TestHelmState_Delete(t *testing.T) {
 			kubeContext:    "ctx",
 			defKubeContext: "defctx",
 			flags:          "--kube-contextctx",
-			deleted:        []mockRelease{{"releaseA", []string{"--purge", "--kube-context", "ctx"}}},
+			deleted:        []exectest.Release{{Name: "releaseA", Flags: []string{"--purge", "--kube-context", "ctx"}}},
 		},
 	}
 	for i := range tests {
@@ -2208,12 +2033,12 @@ func TestHelmState_Delete(t *testing.T) {
 				Releases: releases,
 				logger:   logger,
 			}
-			helm := &mockHelmExec{
-				lists:   map[listKey]string{},
-				deleted: []mockRelease{},
+			helm := &exectest.Helm{
+				Lists:   map[exectest.ListKey]string{},
+				Deleted: []exectest.Release{},
 			}
 			if tt.installed {
-				helm.lists[listKey{filter: "^" + name + "$", flags: tt.flags}] = name
+				helm.Lists[exectest.ListKey{Filter: "^" + name + "$", Flags: tt.flags}] = name
 			}
 			affectedReleases := AffectedReleases{}
 			errs := state.DeleteReleases(&affectedReleases, helm, 1, tt.purge)
@@ -2222,8 +2047,8 @@ func TestHelmState_Delete(t *testing.T) {
 					t.Errorf("DeleteReleases() for %s error = %v, wantErr %v", tt.name, errs, tt.wantErr)
 					return
 				}
-			} else if !(reflect.DeepEqual(tt.deleted, helm.deleted) && (len(affectedReleases.Deleted) == len(tt.deleted))) {
-				t.Errorf("unexpected deletions happened: expected %v, got %v", tt.deleted, helm.deleted)
+			} else if !(reflect.DeepEqual(tt.deleted, helm.Deleted) && (len(affectedReleases.Deleted) == len(tt.deleted))) {
+				t.Errorf("unexpected deletions happened: expected %v, got %v", tt.deleted, helm.Deleted)
 			}
 		}
 		t.Run(tt.name, f)