Share helmexec from State Creation (#804)

Closes #444 and #782 

This is the final PR to fully cache and parallelize helm secret decryption.  It threads the shared helmexec.Interface into the StateCreator and HelmState structs to be used during environment secret decryption.  This should effectively cache secrets for the duration of a helmfile run, regardless of where they are first decrypted.

pkg/app/app.go

@@ -2,9 +2,6 @@ package app
 
 import (
 	"fmt"
-	"github.com/roboll/helmfile/pkg/helmexec"
-	"github.com/roboll/helmfile/pkg/remote"
-	"github.com/roboll/helmfile/pkg/state"
 	"io/ioutil"
 	"log"
 	"os"
@@ -12,6 +9,10 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/remote"
+	"github.com/roboll/helmfile/pkg/state"
+
 	"go.uber.org/zap"
 
 	"path/filepath"
@@ -239,6 +240,7 @@ func (a *App) loadDesiredStateFromYaml(file string, opts ...LoadOpts) (*state.He
 		Reverse:     a.Reverse,
 		KubeContext: a.KubeContext,
 		glob:        a.glob,
+		helm:        a.helmExecer,
 	}
 
 	var op LoadOpts

pkg/app/desired_state_file_loader.go

@@ -4,12 +4,14 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"github.com/imdario/mergo"
-	"github.com/roboll/helmfile/pkg/environment"
-	"github.com/roboll/helmfile/pkg/state"
-	"go.uber.org/zap"
 	"path/filepath"
 	"sort"
+
+	"github.com/imdario/mergo"
+	"github.com/roboll/helmfile/pkg/environment"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/state"
+	"go.uber.org/zap"
 )
 
 type desiredStateLoader struct {
@@ -25,6 +27,7 @@ type desiredStateLoader struct {
 	glob       func(string) ([]string, error)
 
 	logger *zap.SugaredLogger
+	helm   helmexec.Interface
 }
 
 func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, error) {
@@ -125,7 +128,7 @@ func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *e
 }
 
 func (a *desiredStateLoader) underlying() *state.StateCreator {
-	c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob)
+	c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.helm)
 	c.LoadFile = a.loadFile
 	return c
 }

pkg/state/create.go

@@ -38,13 +38,14 @@ type StateCreator struct {
 	fileExists func(string) (bool, error)
 	abs        func(string) (string, error)
 	glob       func(string) ([]string, error)
+	helm       helmexec.Interface
 
 	Strict bool
 
 	LoadFile func(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*HelmState, error)
 }
 
-func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error)) *StateCreator {
+func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), helm helmexec.Interface) *StateCreator {
 	return &StateCreator{
 		logger:     logger,
 		readFile:   readFile,
@@ -52,6 +53,7 @@ func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error)
 		abs:        abs,
 		glob:       glob,
 		Strict:     true,
+		helm:       helm,
 	}
 }
 
@@ -61,6 +63,7 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState,
 
 	state.FilePath = file
 	state.basePath = baseDir
+	state.helm = c.helm
 
 	decoder := yaml.NewDecoder(bytes.NewReader(content))
 	if !c.Strict {
@@ -186,9 +189,6 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 		}
 
 		if len(envSpec.Secrets) > 0 {
-			helm := helmexec.New(st.logger, "", &helmexec.ShellRunner{
-				Logger: st.logger,
-			})
 
 			var envSecretFiles []string
 			for _, urlOrPath := range envSpec.Secrets {
@@ -202,7 +202,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 
 				envSecretFiles = append(envSecretFiles, resolved...)
 			}
-			if err = st.scatterGatherEnvSecretFiles(envSecretFiles, helm, envVals, readFile); err != nil {
+			if err = st.scatterGatherEnvSecretFiles(envSecretFiles, envVals, readFile); err != nil {
 				return nil, err
 			}
 		}
@@ -225,7 +225,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 	return newEnv, nil
 }
 
-func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, helm helmexec.Interface, envVals map[string]interface{}, readFile func(string) ([]byte, error)) error {
+func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, envVals map[string]interface{}, readFile func(string) ([]byte, error)) error {
 	var errs []error
 
 	inputs := envSecretFiles
@@ -239,6 +239,7 @@ func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, helm h
 
 	secrets := make(chan string, inputsSize)
 	results := make(chan secretResult, inputsSize)
+	helm := st.helm
 
 	st.scatterGather(0, inputsSize,
 		func() {

pkg/state/create_test.go

@@ -1,13 +1,14 @@
 package state
 
 import (
-	"github.com/roboll/helmfile/pkg/testhelper"
-	"go.uber.org/zap"
 	"io/ioutil"
 	"path/filepath"
 	"reflect"
 	"testing"
 
+	"github.com/roboll/helmfile/pkg/testhelper"
+	"go.uber.org/zap"
+
 	. "gotest.tools/assert"
 	"gotest.tools/assert/cmp"
 )
@@ -107,7 +108,7 @@ bar: {{ readFile "bar.txt" }}
 	})
 	testFs.Cwd = "/example/path/to"
 
-	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", false, nil)
+	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", false, nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}

pkg/state/state.go

@@ -62,6 +62,7 @@ type HelmState struct {
 	tempDir    func(string, string) (string, error)
 
 	runner helmexec.Runner
+	helm   helmexec.Interface
 }
 
 // SubHelmfileSpec defines the subhelmfile path and options