feat: state values (#647)

This adds `values` to state files as proposed in #640.

```yaml
values:
- key1: val1
- defaults.yaml

environments:
  default:
  - values:
    - environments/default.yaml
  production:
  - values:
    - environments/production.yaml

```

`{{ .Valuese.key1 }}` evaluates to `val1` if and only if it is not overrode via the production or the default env, or command-line args.

Resolves #640

pkg/app/app_test.go

@@ -900,6 +900,130 @@ bar: "bar1"
 	}
 }
 
+func TestVisitDesiredStatesWithReleasesFiltered_StateValueOverrides(t *testing.T) {
+	envTmplExpr := "{{ .Values.foo }}-{{ .Values.bar }}-{{ .Values.baz }}-{{ .Values.hoge }}-{{ .Values.fuga }}-{{ .Values.a | first | pluck \"b\" | first | first | pluck \"c\" | first }}"
+	relTmplExpr := "\"{{`{{ .Values.foo }}-{{ .Values.bar }}-{{ .Values.baz }}-{{ .Values.hoge }}-{{ .Values.fuga }}-{{ .Values.a | first | pluck \\\"b\\\" | first | first | pluck \\\"c\\\" | first }}`}}\""
+
+	testcases := []struct {
+		expr, env, expected string
+	}{
+		{
+			expr:     envTmplExpr,
+			env:      "default",
+			expected: "foo-bar_default-baz_override-hoge_set-fuga_set-C",
+		},
+		{
+			expr:     envTmplExpr,
+			env:      "production",
+			expected: "foo-bar_production-baz_override-hoge_set-fuga_set-C",
+		},
+		{
+			expr:     relTmplExpr,
+			env:      "default",
+			expected: "foo-bar_default-baz_override-hoge_set-fuga_set-C",
+		},
+		{
+			expr:     relTmplExpr,
+			env:      "production",
+			expected: "foo-bar_production-baz_override-hoge_set-fuga_set-C",
+		},
+	}
+	for i := range testcases {
+		testcase := testcases[i]
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			files := map[string]string{
+				"/path/to/helmfile.yaml": fmt.Sprintf(`
+# The top-level "values" are "base" values has inherited to state values with the lowest priority.
+# The lowest priority results in environment-specific values to override values defined in the base.
+values:
+- values.yaml
+
+environments:
+  default:
+    values:
+    - default.yaml
+  production:
+    values:
+    - production.yaml
+---
+releases:
+- name: %s
+  chart: %s
+  namespace: %s
+`, testcase.expr, testcase.expr, testcase.expr),
+				"/path/to/values.yaml": `
+foo: foo
+bar: bar
+baz: baz
+hoge: hoge
+fuga: fuga
+
+a: []
+`,
+				"/path/to/default.yaml": `
+bar: "bar_default"
+baz: "baz_default"
+
+a:
+- b: []
+`,
+				"/path/to/production.yaml": `
+bar: "bar_production"
+baz: "baz_production"
+
+a:
+- b: []
+`,
+				"/path/to/overrides.yaml": `
+baz: baz_override
+hoge: hoge_override
+
+a:
+- b:
+  - c: C
+`,
+			}
+
+			actual := []state.ReleaseSpec{}
+
+			collectReleases := func(st *state.HelmState, helm helmexec.Interface) []error {
+				for _, r := range st.Releases {
+					actual = append(actual, r)
+				}
+				return []error{}
+			}
+			app := appWithFs(&App{
+				KubeContext: "default",
+				Logger:      helmexec.NewLogger(os.Stderr, "debug"),
+				Reverse:     false,
+				Namespace:   "",
+				Selectors:   []string{},
+				Env:         testcase.env,
+				ValuesFiles: []string{"overrides.yaml"},
+				Set:         map[string]interface{}{"hoge": "hoge_set", "fuga": "fuga_set"},
+			}, files)
+			err := app.VisitDesiredStatesWithReleasesFiltered(
+				"helmfile.yaml", collectReleases,
+			)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if len(actual) != 1 {
+				t.Errorf("unexpected number of processed releases: expected=1, got=%d", len(actual))
+			}
+			if actual[0].Name != testcase.expected {
+				t.Errorf("unexpected name: expected=%s, got=%s", testcase.expected, actual[0].Name)
+			}
+			if actual[0].Chart != testcase.expected {
+				t.Errorf("unexpected chart: expected=%s, got=%s", testcase.expected, actual[0].Chart)
+			}
+			if actual[0].Namespace != testcase.expected {
+				t.Errorf("unexpected namespace: expected=%s, got=%s", testcase.expected, actual[0].Namespace)
+			}
+		})
+	}
+}
+
 func TestLoadDesiredStateFromYaml_DuplicateReleaseName(t *testing.T) {
 	yamlFile := "example/path/to/yaml/file"
 	yamlContent := []byte(`releases:

pkg/app/desired_state_file_loader.go

@@ -83,6 +83,7 @@ func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, e
 func (ld *desiredStateLoader) loadFile(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
 	return ld.loadFileWithOverrides(inheritedEnv, nil, baseDir, file, evaluateBases)
 }
+
 func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
 	var f string
 	if filepath.IsAbs(file) {

pkg/app/two_pass_renderer.go

@@ -19,8 +19,12 @@ func prependLineNumbers(text string) string {
 	return buf.String()
 }
 
-func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) *environment.Environment {
+func (r *desiredStateLoader) renderPrestate(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) (*environment.Environment, *state.HelmState) {
-	tmplData := state.EnvironmentTemplateData{Environment: *firstPassEnv, Namespace: r.namespace}
+	tmplData := state.EnvironmentTemplateData{
+		Environment: *firstPassEnv,
+		Namespace:   r.namespace,
+		Values:      map[string]interface{}{},
+	}
 	firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData)
 
 	// parse as much as we can, tolerate errors, this is a preparse
@@ -29,7 +33,7 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environ
 		r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", filename, prependLineNumbers(string(content)))
 		if yamlBuf == nil { // we have a template syntax error, let the second parse report
 			r.logger.Debugf("template syntax error: %v", err)
-			return firstPassEnv
+			return firstPassEnv, nil
 		}
 	}
 	yamlData := yamlBuf.String()
@@ -57,7 +61,8 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environ
 	if prestate != nil {
 		firstPassEnv = &prestate.Env
 	}
-	return firstPassEnv
+
+	return firstPassEnv, prestate
 }
 
 type RenderOpts struct {
@@ -88,13 +93,18 @@ func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *en
 		r.logger.Debugf("first-pass uses: %v", initEnv)
 	}
 
-	renderedEnv := r.renderEnvironment(initEnv, baseDir, filename, content)
+	renderedEnv, prestate := r.renderPrestate(initEnv, baseDir, filename, content)
 
 	if r.logger != nil {
 		r.logger.Debugf("first-pass produced: %v", renderedEnv)
 	}
 
-	finalEnv, err := renderedEnv.Merge(overrode)
+	finalEnv, err := inherited.Merge(renderedEnv)
+	if err != nil {
+		return nil, err
+	}
+
+	finalEnv, err = finalEnv.Merge(overrode)
 	if err != nil {
 		return nil, err
 	}
@@ -103,7 +113,23 @@ func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *en
 		r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, *finalEnv)
 	}
 
-	tmplData := state.EnvironmentTemplateData{Environment: *finalEnv, Namespace: r.namespace}
+	vals := map[string]interface{}{}
+	if prestate != nil {
+		prestate.Env = *finalEnv
+		vals, err = prestate.Values()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if prestate != nil {
+		r.logger.Debugf("vals:\n%v\ndefaultVals:%v", vals, prestate.DefaultValues)
+	}
+
+	tmplData := state.EnvironmentTemplateData{
+		Environment: *finalEnv,
+		Namespace:   r.namespace,
+		Values:      vals,
+	}
 	secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData)
 	yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content)
 	if err != nil {

pkg/environment/environment.go

@@ -7,28 +7,44 @@ import (
 )
 
 type Environment struct {
-	Name   string
+	Name     string
-	Values map[string]interface{}
+	Values   map[string]interface{}
+	Defaults map[string]interface{}
 }
 
 var EmptyEnvironment Environment
 
 func (e Environment) DeepCopy() Environment {
-	bytes, err := yaml.Marshal(e.Values)
+	valuesBytes, err := yaml.Marshal(e.Values)
 	if err != nil {
 		panic(err)
 	}
 	var values map[string]interface{}
-	if err := yaml.Unmarshal(bytes, &values); err != nil {
+	if err := yaml.Unmarshal(valuesBytes, &values); err != nil {
 		panic(err)
 	}
 	values, err = maputil.CastKeysToStrings(values)
 	if err != nil {
 		panic(err)
 	}
+
+	defaultsBytes, err := yaml.Marshal(e.Defaults)
+	if err != nil {
+		panic(err)
+	}
+	var defaults map[string]interface{}
+	if err := yaml.Unmarshal(defaultsBytes, &defaults); err != nil {
+		panic(err)
+	}
+	defaults, err = maputil.CastKeysToStrings(defaults)
+	if err != nil {
+		panic(err)
+	}
+
 	return Environment{
-		Name:   e.Name,
+		Name:     e.Name,
-		Values: values,
+		Values:   values,
+		Defaults: defaults,
 	}
 }
 

pkg/maputil/maputil_test.go

@@ -0,0 +1,61 @@
+package maputil
+
+import "testing"
+
+func TestMapUtil_StrKeys(t *testing.T) {
+	m := map[string]interface{}{
+		"a": []interface{}{
+			map[string]interface{}{
+				"b": []interface{}{
+					map[string]interface{}{
+						"c": "C",
+					},
+				},
+			},
+		},
+	}
+
+	r, err := CastKeysToStrings(m)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	a := r["a"].([]interface{})
+	a0 := a[0].(map[string]interface{})
+	b := a0["b"].([]interface{})
+	b0 := b[0].(map[string]interface{})
+	c := b0["c"]
+
+	if c != "C" {
+		t.Errorf("unexpected c: expected=C, got=%s", c)
+	}
+}
+
+func TestMapUtil_IFKeys(t *testing.T) {
+	m := map[interface{}]interface{}{
+		"a": []interface{}{
+			map[interface{}]interface{}{
+				"b": []interface{}{
+					map[interface{}]interface{}{
+						"c": "C",
+					},
+				},
+			},
+		},
+	}
+
+	r, err := CastKeysToStrings(m)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	a := r["a"].([]interface{})
+	a0 := a[0].(map[string]interface{})
+	b := a0["b"].([]interface{})
+	b0 := b[0].(map[string]interface{})
+	c := b0["c"]
+
+	if c != "C" {
+		t.Errorf("unexpected c: expected=C, got=%s", c)
+	}
+}

pkg/state/create.go

@@ -114,6 +114,12 @@ func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *envi
 	if err != nil {
 		return nil, &StateLoadError{fmt.Sprintf("failed to read %s", state.FilePath), err}
 	}
+
+	e.Defaults, err = state.loadValuesEntries(nil, state.DefaultValues)
+	if err != nil {
+		return nil, err
+	}
+
 	state.Env = *e
 
 	return &state, nil
@@ -137,7 +143,12 @@ func (c *StateCreator) ParseAndLoad(content []byte, baseDir, file string, envNam
 		return nil, err
 	}
 
-	return c.LoadEnvValues(state, envName, envValues)
+	state, err = c.LoadEnvValues(state, envName, envValues)
+	if err != nil {
+		return nil, err
+	}
+
+	return state, nil
 }
 
 func (c *StateCreator) loadBases(envValues *environment.Environment, st *HelmState, baseDir string) (*HelmState, error) {
@@ -164,13 +175,8 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 	envVals := map[string]interface{}{}
 	envSpec, ok := st.Environments[name]
 	if ok {
-		envValues := append([]interface{}{}, envSpec.Values...)
-		ld := &EnvironmentValuesLoader{
-			storage:  st.storage(),
-			readFile: st.readFile,
-		}
 		var err error
-		envVals, err = ld.LoadEnvironmentValues(envSpec.MissingFileHandler, envValues)
+		envVals, err = st.loadValuesEntries(envSpec.MissingFileHandler, envSpec.Values)
 		if err != nil {
 			return nil, err
 		}
@@ -237,3 +243,20 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 
 	return newEnv, nil
 }
+
+func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []interface{}) (map[string]interface{}, error) {
+	envVals := map[string]interface{}{}
+
+	valuesEntries := append([]interface{}{}, entries...)
+	ld := &EnvironmentValuesLoader{
+		storage:  st.storage(),
+		readFile: st.readFile,
+	}
+	var err error
+	envVals, err = ld.LoadEnvironmentValues(missingFileHandler, valuesEntries)
+	if err != nil {
+		return nil, err
+	}
+
+	return envVals, nil
+}

pkg/state/environment_values_loader.go

@@ -39,7 +39,7 @@ func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *str
 			}
 
 			for _, envvalFullPath := range resolved {
-				tmplData := EnvironmentTemplateData{Environment: environment.EmptyEnvironment, Namespace: ""}
+				tmplData := EnvironmentTemplateData{environment.EmptyEnvironment, "", map[string]interface{}{}}
 				r := tmpl.NewFileRenderer(ld.readFile, filepath.Dir(envvalFullPath), tmplData)
 				bytes, err := r.RenderToBytes(envvalFullPath)
 				if err != nil {

pkg/state/release.go

@@ -15,6 +15,14 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
 		return nil, fmt.Errorf("failed executing template expressions in release \"%s\": %v", r.Name, err)
 	}
 
+	{
+		ts := result.Name
+		result.Name, err = renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".name = \"%s\": %v", r.Name, ts, err)
+		}
+	}
+
 	{
 		ts := result.Chart
 		result.Chart, err = renderer.RenderTemplateContentToString([]byte(ts))

pkg/state/state.go

@@ -27,6 +27,9 @@ type HelmState struct {
 	basePath string
 	FilePath string
 
+	// DefaultValues is the default values to be overrode by environment values and command-line overrides
+	DefaultValues []interface{} `yaml:"values"`
+
 	Environments map[string]EnvironmentSpec `yaml:"environments"`
 
 	Bases              []string          `yaml:"bases"`
@@ -1243,7 +1246,7 @@ func (st *HelmState) flagsForLint(helm helmexec.Interface, release *ReleaseSpec,
 }
 
 func (st *HelmState) RenderValuesFileToBytes(path string) ([]byte, error) {
-	r := tmpl.NewFileRenderer(st.readFile, filepath.Dir(path), st.envTemplateData())
+	r := tmpl.NewFileRenderer(st.readFile, filepath.Dir(path), st.valuesFileTemplateData())
 	return r.RenderToBytes(path)
 }
 

pkg/state/state_exec_tmpl.go

@@ -2,23 +2,58 @@ package state
 
 import (
 	"fmt"
+	"github.com/imdario/mergo"
+	"github.com/roboll/helmfile/pkg/maputil"
 	"github.com/roboll/helmfile/pkg/tmpl"
 )
 
-func (st *HelmState) envTemplateData() EnvironmentTemplateData {
+func (st *HelmState) Values() (map[string]interface{}, error) {
+	vals := map[string]interface{}{}
+
+	if err := mergo.Merge(&vals, st.Env.Defaults, mergo.WithOverride); err != nil {
+		return nil, err
+	}
+	if err := mergo.Merge(&vals, st.Env.Values, mergo.WithOverride); err != nil {
+		return nil, err
+	}
+
+	vals, err := maputil.CastKeysToStrings(vals)
+	if err != nil {
+		return nil, err
+	}
+
+	return vals, nil
+}
+
+func (st *HelmState) mustLoadVals() map[string]interface{} {
+	vals, err := st.Values()
+	if err != nil {
+		panic(err)
+	}
+	return vals
+}
+
+func (st *HelmState) valuesFileTemplateData() EnvironmentTemplateData {
 	return EnvironmentTemplateData{
-		st.Env,
+		Environment: st.Env,
-		st.Namespace,
+		Namespace:   st.Namespace,
+		Values:      st.mustLoadVals(),
 	}
 }
 
 func (st *HelmState) ExecuteTemplates() (*HelmState, error) {
 	r := *st
 
+	vals, err := st.Values()
+	if err != nil {
+		return nil, err
+	}
+
 	for i, rt := range st.Releases {
-		tmplData := ReleaseTemplateData{
+		tmplData := releaseTemplateData{
 			Environment: st.Env,
 			Release:     rt,
+			Values:      vals,
 		}
 		renderer := tmpl.NewFileRenderer(st.readFile, st.basePath, tmplData)
 		r, err := rt.ExecuteTemplateExpressions(renderer)

pkg/state/types.go

@@ -15,12 +15,16 @@ type EnvironmentTemplateData struct {
 	Environment environment.Environment
 	// Namespace is accessible as `.Namespace` from any non-values template executed by the renderer
 	Namespace string
+	// Values is accessible as `.Values` and it contains default state values overrode by environment values and override values.
+	Values map[string]interface{}
 }
 
-// ReleaseTemplateData provides variables accessible while executing golang text/template expressions in releases of a helmfile YAML file
+// releaseTemplateData provides variables accessible while executing golang text/template expressions in releases of a helmfile YAML file
-type ReleaseTemplateData struct {
+type releaseTemplateData struct {
 	// Environment is accessible as `.Environment` from any template expression executed by the renderer
 	Environment environment.Environment
 	// Release is accessible as `.Release` from any template expression executed by the renderer
 	Release ReleaseSpec
+	// Values is accessible as `.Values` and it contains default state values overrode by environment values and override values.
+	Values map[string]interface{}
 }