Add integration with Hashicorp Vault, AWS SSM, SecretsManager (#906)

* feat: Add integration with Hashicorp Vault, AWS SSM, SecretsManager

Fields which are rendered: Release.Values, Release.SetValues.Value, Release.SetValues.Values

Example:
```
values:
- foo: ref+vault://mykv/foo?address=http://127.0.0.1:8200#/mykey
set:
- name: xyz
  values:
  - ref+vault://mykv/foo?address=http://127.0.0.1:8200#/mykey3
```

Resolves #881

* feat: Update integration with variantdev/vals

New ref+.\* secret formats are used:
https://github.com/variantdev/vals/tree/6565695a031f407a1066bdd31ae313db6a33ec0a#suported-backends

Resolves #881

pkg/app/app.go

@@ -13,6 +13,7 @@ import (
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/remote"
 	"github.com/roboll/helmfile/pkg/state"
+	"github.com/variantdev/vals"
 
 	"go.uber.org/zap"
 
@@ -20,6 +21,11 @@ import (
 	"sort"
 )
 
+const (
+	// cache size for improving performance of ref+.* secrets rendering
+	valsCacheSize = 512
+)
+
 type App struct {
 	KubeContext string
 	Logger      *zap.SugaredLogger
@@ -49,6 +55,8 @@ type App struct {
 	remote *remote.Remote
 
 	helmExecer helmexec.Interface
+
+	valsRuntime vals.Evaluator
 }
 
 func New(conf ConfigProvider) *App {
@@ -78,6 +86,13 @@ func Init(app *App) *App {
 	app.fileExistsAt = fileExistsAt
 	app.fileExists = fileExists
 	app.directoryExistsAt = directoryExistsAt
+
+	var err error
+	app.valsRuntime, err = vals.New(valsCacheSize)
+	if err != nil {
+		panic(fmt.Sprintf("Failed to initialize vals runtime: %v", err))
+	}
+
 	return app
 }
 
@@ -274,6 +289,7 @@ func (a *App) loadDesiredStateFromYaml(file string, opts ...LoadOpts) (*state.He
 		KubeContext: a.KubeContext,
 		glob:        a.glob,
 		helm:        a.helmExecer,
+		valsRuntime: a.valsRuntime,
 	}
 
 	var op LoadOpts

pkg/app/app_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/state"
 	"github.com/roboll/helmfile/pkg/testhelper"
+	"github.com/variantdev/vals"
 
 	"go.uber.org/zap"
 	"gotest.tools/env"
@@ -440,8 +441,8 @@ func TestVisitDesiredStatesWithReleasesFiltered_EmbeddedSelectors(t *testing.T)
 helmfiles:
 - path: helmfile.d/a*.yaml
   selectors:
-  - name=prometheus      
-  - name=zipkin      
+  - name=prometheus
+  - name=zipkin
 - helmfile.d/b*.yaml
 - path: helmfile.d/c*.yaml
   selectors: []
@@ -1944,6 +1945,11 @@ releases:
 	var buffer bytes.Buffer
 	logger := helmexec.NewLogger(&buffer, "debug")
 
+	valsRuntime, err := vals.New(32)
+	if err != nil {
+		t.Errorf("unexpected error creating vals runtime: %v", err)
+	}
+
 	app := appWithFs(&App{
 		glob:        filepath.Glob,
 		abs:         filepath.Abs,
@@ -1952,6 +1958,7 @@ releases:
 		Logger:      logger,
 		helmExecer:  helm,
 		Namespace:   "testNamespace",
+		valsRuntime: valsRuntime,
 	}, files)
 	app.Template(configImpl{set: []string{"foo=a", "bar=b"}})
 

pkg/app/desired_state_file_loader.go

@@ -11,6 +11,7 @@ import (
 	"github.com/roboll/helmfile/pkg/environment"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/state"
+	"github.com/variantdev/vals"
 	"go.uber.org/zap"
 )
 
@@ -26,8 +27,9 @@ type desiredStateLoader struct {
 	abs        func(string) (string, error)
 	glob       func(string) ([]string, error)
 
-	logger *zap.SugaredLogger
-	helm   helmexec.Interface
+	logger      *zap.SugaredLogger
+	helm        helmexec.Interface
+	valsRuntime vals.Evaluator
 }
 
 func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, error) {
@@ -128,7 +130,7 @@ func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *e
 }
 
 func (a *desiredStateLoader) underlying() *state.StateCreator {
-	c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.helm)
+	c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.helm, a.valsRuntime)
 	c.LoadFile = a.loadFile
 	return c
 }

pkg/state/create.go

@@ -11,6 +11,7 @@ import (
 	"github.com/roboll/helmfile/pkg/environment"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/maputil"
+	"github.com/variantdev/vals"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
 )
@@ -33,27 +34,29 @@ func (e *UndefinedEnvError) Error() string {
 }
 
 type StateCreator struct {
-	logger     *zap.SugaredLogger
-	readFile   func(string) ([]byte, error)
-	fileExists func(string) (bool, error)
-	abs        func(string) (string, error)
-	glob       func(string) ([]string, error)
-	helm       helmexec.Interface
+	logger      *zap.SugaredLogger
+	readFile    func(string) ([]byte, error)
+	fileExists  func(string) (bool, error)
+	abs         func(string) (string, error)
+	glob        func(string) ([]string, error)
+	helm        helmexec.Interface
+	valsRuntime vals.Evaluator
 
 	Strict bool
 
 	LoadFile func(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*HelmState, error)
 }
 
-func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), helm helmexec.Interface) *StateCreator {
+func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), helm helmexec.Interface, valsRuntime vals.Evaluator) *StateCreator {
 	return &StateCreator{
-		logger:     logger,
-		readFile:   readFile,
-		fileExists: fileExists,
-		abs:        abs,
-		glob:       glob,
-		Strict:     true,
-		helm:       helm,
+		logger:      logger,
+		readFile:    readFile,
+		fileExists:  fileExists,
+		abs:         abs,
+		glob:        glob,
+		Strict:      true,
+		helm:        helm,
+		valsRuntime: valsRuntime,
 	}
 }
 
@@ -107,6 +110,7 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState,
 	state.removeFile = os.Remove
 	state.fileExists = c.fileExists
 	state.glob = c.glob
+	state.valsRuntime = c.valsRuntime
 
 	return &state, nil
 }

pkg/state/create_test.go

@@ -108,7 +108,7 @@ bar: {{ readFile "bar.txt" }}
 	})
 	testFs.Cwd = "/example/path/to"
 
-	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", false, nil)
+	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil, nil).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", false, nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}

pkg/state/state.go

@@ -24,6 +24,7 @@ import (
 	"regexp"
 
 	"github.com/tatsushid/go-prettytable"
+	"github.com/variantdev/vals"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
 )
@@ -61,8 +62,9 @@ type HelmState struct {
 	glob       func(string) ([]string, error)
 	tempDir    func(string, string) (string, error)
 
-	runner helmexec.Runner
-	helm   helmexec.Interface
+	runner      helmexec.Runner
+	helm        helmexec.Interface
+	valsRuntime vals.Evaluator
 }
 
 // SubHelmfileSpec defines the subhelmfile path and options
@@ -1521,7 +1523,7 @@ func (st *HelmState) generateTemporaryValuesFiles(values []interface{}, missingF
 			}
 			st.logger.Debugf("successfully generated the value file at %s. produced:\n%s", path, string(yamlBytes))
 			generatedFiles = append(generatedFiles, valfile.Name())
-		case map[interface{}]interface{}:
+		case map[interface{}]interface{}, map[string]interface{}:
 			valfile, err := ioutil.TempFile("", "values")
 			if err != nil {
 				return nil, err
@@ -1557,7 +1559,17 @@ func (st *HelmState) namespaceAndValuesFlags(helm helmexec.Interface, release *R
 		}
 	}
 
-	generatedFiles, err := st.generateTemporaryValuesFiles(values, release.MissingFileHandler)
+	valuesMapSecretsRendered, err := st.valsRuntime.Eval(map[string]interface{}{"values": values})
+	if err != nil {
+		return nil, err
+	}
+
+	valuesSecretsRendered, ok := valuesMapSecretsRendered["values"].([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("Failed to render values in %s for release %s: type %T isn't supported", st.FilePath, release.Name, valuesMapSecretsRendered["values"])
+	}
+
+	generatedFiles, err := st.generateTemporaryValuesFiles(valuesSecretsRendered, release.MissingFileHandler)
 	if err != nil {
 		return nil, err
 	}
@@ -1594,12 +1606,20 @@ func (st *HelmState) namespaceAndValuesFlags(helm helmexec.Interface, release *R
 	if len(release.SetValues) > 0 {
 		for _, set := range release.SetValues {
 			if set.Value != "" {
-				flags = append(flags, "--set", fmt.Sprintf("%s=%s", escape(set.Name), escape(set.Value)))
+				renderedValue, err := renderValsSecrets(st.valsRuntime, set.Value)
+				if err != nil {
+					return nil, fmt.Errorf("Failed to render set value entry in %s for release %s: %v", st.FilePath, release.Name, err)
+				}
+				flags = append(flags, "--set", fmt.Sprintf("%s=%s", escape(set.Name), escape(renderedValue[0])))
 			} else if set.File != "" {
 				flags = append(flags, "--set-file", fmt.Sprintf("%s=%s", escape(set.Name), st.storage().normalizePath(set.File)))
 			} else if len(set.Values) > 0 {
-				items := make([]string, len(set.Values))
-				for i, raw := range set.Values {
+				renderedValues, err := renderValsSecrets(st.valsRuntime, set.Values...)
+				if err != nil {
+					return nil, fmt.Errorf("Failed to render set values entry in %s for release %s: %v", st.FilePath, release.Name, err)
+				}
+				items := make([]string, len(renderedValues))
+				for i, raw := range renderedValues {
 					items[i] = escape(raw)
 				}
 				v := strings.Join(items, ",")
@@ -1638,6 +1658,27 @@ func (st *HelmState) namespaceAndValuesFlags(helm helmexec.Interface, release *R
 	return flags, nil
 }
 
+// renderValsSecrets helper function which renders 'ref+.*' secrets
+func renderValsSecrets(e vals.Evaluator, input ...string) ([]string, error) {
+	output := make([]string, len(input))
+	if len(input) > 0 {
+		mapRendered, err := e.Eval(map[string]interface{}{"values": input})
+		if err != nil {
+			return nil, err
+		}
+
+		rendered, ok := mapRendered["values"].([]interface{})
+		if !ok {
+			return nil, fmt.Errorf("type %T isn't supported", mapRendered["values"])
+		}
+
+		for i := 0; i < len(rendered); i++ {
+			output[i] = fmt.Sprintf("%v", rendered[i])
+		}
+	}
+	return output, nil
+}
+
 // DisplayAffectedReleases logs the upgraded, deleted and in error releases
 func (ar *AffectedReleases) DisplayAffectedReleases(logger *zap.SugaredLogger) {
 	if ar.Upgraded != nil {

pkg/state/state_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/testhelper"
+	"github.com/variantdev/vals"
 
 	"errors"
 	"strings"
@@ -17,6 +18,7 @@ import (
 )
 
 var logger = helmexec.NewLogger(os.Stdout, "warn")
+var valsRuntime, _ = vals.New(32)
 
 func injectFs(st *HelmState, fs *testhelper.TestFs) *HelmState {
 	st.glob = fs.Glob
@@ -523,6 +525,7 @@ func TestHelmState_flagsForUpgrade(t *testing.T) {
 				DeprecatedContext: "default",
 				Releases:          []ReleaseSpec{*tt.release},
 				HelmDefaults:      tt.defaults,
+				valsRuntime:       valsRuntime,
 			}
 			helm := helmexec.New(logger, "default", &helmexec.ShellRunner{
 				Logger: logger,
@@ -963,8 +966,9 @@ func TestHelmState_SyncReleases(t *testing.T) {
 		tt := tests[i]
 		t.Run(tt.name, func(t *testing.T) {
 			state := &HelmState{
-				Releases: tt.releases,
-				logger:   logger,
+				Releases:    tt.releases,
+				logger:      logger,
+				valsRuntime: valsRuntime,
 			}
 			if _ = state.SyncReleases(&AffectedReleases{}, tt.helm, []string{}, 1); !reflect.DeepEqual(tt.helm.releases, tt.wantReleases) {
 				t.Errorf("HelmState.SyncReleases() for [%s] = %v, want %v", tt.name, tt.helm.releases, tt.wantReleases)
@@ -1049,9 +1053,10 @@ func TestHelmState_SyncReleases_MissingValuesFileForUndesiredRelease(t *testing.
 		tt := tests[i]
 		t.Run(tt.name, func(t *testing.T) {
 			state := &HelmState{
-				basePath: ".",
-				Releases: []ReleaseSpec{tt.release},
-				logger:   logger,
+				basePath:    ".",
+				Releases:    []ReleaseSpec{tt.release},
+				logger:      logger,
+				valsRuntime: valsRuntime,
 			}
 			fs := testhelper.NewTestFs(map[string]string{})
 			state = injectFs(state, fs)
@@ -1163,8 +1168,9 @@ func TestHelmState_SyncReleasesAffectedRealeases(t *testing.T) {
 		tt := tests[i]
 		t.Run(tt.name, func(t *testing.T) {
 			state := &HelmState{
-				Releases: tt.releases,
-				logger:   logger,
+				Releases:    tt.releases,
+				logger:      logger,
+				valsRuntime: valsRuntime,
 			}
 			helm := &mockHelmExec{
 				lists: map[listKey]string{},
@@ -1264,8 +1270,9 @@ func TestGetDeployedVersion(t *testing.T) {
 		tt := tests[i]
 		t.Run(tt.name, func(t *testing.T) {
 			state := &HelmState{
-				Releases: []ReleaseSpec{tt.release},
-				logger:   logger,
+				Releases:    []ReleaseSpec{tt.release},
+				logger:      logger,
+				valsRuntime: valsRuntime,
 			}
 			helm := &mockHelmExec{
 				lists: map[listKey]string{},
@@ -1384,8 +1391,9 @@ func TestHelmState_DiffReleases(t *testing.T) {
 		tt := tests[i]
 		t.Run(tt.name, func(t *testing.T) {
 			state := &HelmState{
-				Releases: tt.releases,
-				logger:   logger,
+				Releases:    tt.releases,
+				logger:      logger,
+				valsRuntime: valsRuntime,
 			}
 			_, errs := state.DiffReleases(tt.helm, []string{}, 1, false, false, false)
 			if errs != nil && len(errs) > 0 {
@@ -1455,8 +1463,9 @@ func TestHelmState_SyncReleasesCleanup(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			numRemovedFiles := 0
 			state := &HelmState{
-				Releases: tt.releases,
-				logger:   logger,
+				Releases:    tt.releases,
+				logger:      logger,
+				valsRuntime: valsRuntime,
 				removeFile: func(f string) error {
 					numRemovedFiles += 1
 					return nil
@@ -1538,8 +1547,9 @@ func TestHelmState_DiffReleasesCleanup(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			numRemovedFiles := 0
 			state := &HelmState{
-				Releases: tt.releases,
-				logger:   logger,
+				Releases:    tt.releases,
+				logger:      logger,
+				valsRuntime: valsRuntime,
 				removeFile: func(f string) error {
 					numRemovedFiles += 1
 					return nil