Merge pull request #587 from roboll/layering-enhancements

feat: helmfile.yaml layering enhancements

The current  [Layering](https://github.com/roboll/helmfile/blob/master/docs/writing-helmfile.md#layering) system didn't work as documented, as it relies on helmfile to template each "part" of your helmfile.yaml THEN merge them one by one.

The reality was that helmfile template all the parts of your helmfile.yaml at once, and then merge those YAML documents. In https://github.com/roboll/helmfile/issues/388#issuecomment-436186278, @sruon was making a GREAT point that we may need to change helmfile to render templates earlier - that is to evaluate a template per each helmfile.yaml part separated by `---`. Sorry I missed my expertise to follow your great idea last year @sruon  😭 

Anyways, this, in combination with the wrong documentation, has made so many people confused. To finally overcome this situation, here's a fairly large PR that introduces the 2 enhancements:

- `bases:` for easier layering without go template expressions, especially `{{ readFunc "path/to/file" }}`s. This is the first commit of this PR.
- `helmfile.yaml` is splited by the separator `---` at first. Each part is then rendered as a go template(double-render applies as before). Finally, All the results are merged in the order of occurence. I assume this as an enhanced version of @sruon's work. This is the second commit of this PR.

Resolves #388
Resolve #584
Resolves #585 (`HELMFILE_EXPERIMENTA=true -f helmfile.yaml helmfile` disables the whole-file templating, treating the helmfile.yaml as a regular YAML file as the file ext. denotes. Use `helmfile.yaml.gotmpl` or `helmfile.gotmpl` to enable)
Fixes #568 (Use `bases` or `readFile` rather than not importing implicitly with `helmfile.d`

docs/writing-helmfile.md

@@ -100,10 +100,10 @@ Use Layering to extract the common parts into a dedicated *library helmfile*s, s
 Let's assume that your `helmfile.yaml` looks like:
 
 ```
-{ readFile "commons.yaml" }}
+bases:
----
+- commons.yaml
-{{ readFile "environments.yaml" }}
+- environments.yaml
----
+
 releases:
 - name: myapp
   chart: mychart
@@ -125,23 +125,26 @@ environments:
   production:
 ```
 
-At run time, template expressions in your `helmfile.yaml` are executed:
+At run time, `bases` in your `helmfile.yaml` are evaluated to produce:
 
 ```yaml
+# commons.yaml
 releases:
 - name: metricbaet
   chart: stable/metricbeat
 ---
+# environments.yaml
 environments:
   development:
   production:
 ---
+# helmfile.yaml
 releases:
 - name: myapp
   chart: mychart
 ```
 
-Resulting YAML documents are merged in the order of occurrence,
+Finally the resulting YAML documents are merged in the order of occurrence,
 so that your `helmfile.yaml` becomes:
 
 ```yaml
@@ -159,3 +162,5 @@ releases:
 Great!
 
 Now, repeat the above steps for each your `helmfile.yaml`, so that all your helmfiles becomes DRY.
+
+Please also see [the discussion in the issue 388](https://github.com/roboll/helmfile/issues/388#issuecomment-491710348) for more advanced layering examples.

pkg/app/app.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/signal"
 	"strings"
+	"syscall"
 
 	"github.com/roboll/helmfile/helmexec"
 	"github.com/roboll/helmfile/state"
@@ -14,7 +15,6 @@ import (
 
 	"path/filepath"
 	"sort"
-	"syscall"
 )
 
 type App struct {
@@ -111,34 +111,42 @@ func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error {
 	return nil
 }
 
+func (a *App) loadDesiredStateFromYaml(file string) (*state.HelmState, error) {
+	ld := &desiredStateLoader{
+		readFile:  a.readFile,
+		env:       a.Env,
+		namespace: a.Namespace,
+		logger:    a.Logger,
+		abs:       a.abs,
+
+		Reverse:     a.Reverse,
+		KubeContext: a.KubeContext,
+		glob:        a.glob,
+	}
+	return ld.Load(file)
+}
+
 func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error {
 	noMatchInHelmfiles := true
 
 	err := a.visitStateFiles(fileOrDir, func(f string) error {
-		content, err := a.readFile(f)
+		st, err := a.loadDesiredStateFromYaml(f)
-		if err != nil {
-			return err
-		}
-		// render template, in two runs
-		r := &twoPassRenderer{
-			reader:    a.readFile,
-			env:       a.Env,
-			namespace: a.Namespace,
-			filename:  f,
-			logger:    a.Logger,
-			abs:       a.abs,
-		}
-		yamlBuf, err := r.renderTemplate(content)
-		if err != nil {
-			return fmt.Errorf("error during %s parsing: %v", f, err)
-		}
 
-		st, err := a.loadDesiredStateFromYaml(
+		sigs := make(chan os.Signal, 1)
-			yamlBuf.Bytes(),
+		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
-			f,
+		go func() {
-			a.Namespace,
+			sig := <-sigs
-			a.Env,
+
-		)
+			errs := []error{fmt.Errorf("Received [%s] to shutdown ", sig)}
+			_ = context{a, st}.clean(errs)
+			// See http://tldp.org/LDP/abs/html/exitcodes.html
+			switch sig {
+			case syscall.SIGINT:
+				os.Exit(130)
+			case syscall.SIGTERM:
+				os.Exit(143)
+			}
+		}()
 
 		ctx := context{a, st}
 
@@ -313,78 +321,6 @@ func directoryExistsAt(path string) bool {
 	return err == nil && fileInfo.Mode().IsDir()
 }
 
-func (a *App) loadDesiredStateFromYaml(yaml []byte, file string, namespace string, env string) (*state.HelmState, error) {
-	c := state.NewCreator(a.Logger, a.readFile, a.abs)
-	st, err := c.CreateFromYaml(yaml, file, env)
-	if err != nil {
-		return nil, err
-	}
-
-	helmfiles := []state.SubHelmfileSpec{}
-	for _, hf := range st.Helmfiles {
-		globPattern := hf.Path
-		var absPathPattern string
-		if filepath.IsAbs(globPattern) {
-			absPathPattern = globPattern
-		} else {
-			absPathPattern = st.JoinBase(globPattern)
-		}
-		matches, err := a.glob(absPathPattern)
-		if err != nil {
-			return nil, fmt.Errorf("failed processing %s: %v", globPattern, err)
-		}
-		sort.Strings(matches)
-		for _, match := range matches {
-			newHelmfile := hf
-			newHelmfile.Path = match
-			helmfiles = append(helmfiles, newHelmfile)
-		}
-
-	}
-	st.Helmfiles = helmfiles
-
-	if a.Reverse {
-		rev := func(i, j int) bool {
-			return j < i
-		}
-		sort.Slice(st.Releases, rev)
-		sort.Slice(st.Helmfiles, rev)
-	}
-
-	if a.KubeContext != "" {
-		if st.HelmDefaults.KubeContext != "" {
-			log.Printf("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.")
-			os.Exit(1)
-		}
-		st.HelmDefaults.KubeContext = a.KubeContext
-	}
-	if namespace != "" {
-		if st.Namespace != "" {
-			log.Printf("err: Cannot use option --namespace and set attribute namespace.")
-			os.Exit(1)
-		}
-		st.Namespace = namespace
-	}
-
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
-	go func() {
-		sig := <-sigs
-
-		errs := []error{fmt.Errorf("Received [%s] to shutdown ", sig)}
-		_ = context{a, st}.clean(errs)
-		// See http://tldp.org/LDP/abs/html/exitcodes.html
-		switch sig {
-		case syscall.SIGINT:
-			os.Exit(130)
-		case syscall.SIGTERM:
-			os.Exit(143)
-		}
-	}()
-
-	return st, nil
-}
-
 type Error struct {
 	msg string
 

pkg/app/app_test.go

@@ -2,7 +2,6 @@ package app
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -83,7 +82,14 @@ func (f *testFs) readFile(filename string) ([]byte, error) {
 	return []byte(str), nil
 }
 
-func (f *testFs) glob(pattern string) ([]string, error) {
+func (f *testFs) glob(relPattern string) ([]string, error) {
+	var pattern string
+	if relPattern[0] == '/' {
+		pattern = relPattern
+	} else {
+		pattern = filepath.Join(f.wd, relPattern)
+	}
+
 	matches := []string{}
 	for name, _ := range f.files {
 		matched, err := filepath.Match(pattern, name)
@@ -95,7 +101,7 @@ func (f *testFs) glob(pattern string) ([]string, error) {
 		}
 	}
 	if len(matches) == 0 {
-		return []string(nil), fmt.Errorf("no file matched: %s", pattern)
+		return []string(nil), fmt.Errorf("no file matched %s for files: %v", pattern, f.files)
 	}
 	return matches, nil
 }
@@ -640,15 +646,199 @@ func TestLoadDesiredStateFromYaml_DuplicateReleaseName(t *testing.T) {
   labels:
     stage: post
 `)
+	readFile := func(filename string) ([]byte, error) {
+		if filename != yamlFile {
+			return nil, fmt.Errorf("unexpected filename: %s", filename)
+		}
+		return yamlContent, nil
+	}
 	app := &App{
-		readFile:    ioutil.ReadFile,
+		readFile:    readFile,
 		glob:        filepath.Glob,
 		abs:         filepath.Abs,
 		KubeContext: "default",
+		Env:         "default",
 		Logger:      helmexec.NewLogger(os.Stderr, "debug"),
 	}
-	_, err := app.loadDesiredStateFromYaml(yamlContent, yamlFile, "default", "default")
+	_, err := app.loadDesiredStateFromYaml(yamlFile)
 	if err != nil {
-		t.Error("unexpected error")
+		t.Errorf("unexpected error: %v", err)
+	}
+}
+
+func TestLoadDesiredStateFromYaml_Bases(t *testing.T) {
+	yamlFile := "/path/to/yaml/file"
+	yamlContent := []byte(`bases:
+- ../base.yaml
+- ../base.gotmpl
+
+{{ readFile "templates.yaml" }}
+
+releases:
+- name: myrelease1
+  chart: mychart1
+  labels:
+    stage: pre
+    foo: bar
+- name: myrelease1
+  chart: mychart2
+  labels:
+    stage: post
+  <<: *default
+`)
+	files := map[string][]byte{
+		yamlFile: yamlContent,
+		"/path/to/base.yaml": []byte(`environments:
+  default:
+    values:
+    - environments/default/1.yaml
+`),
+		"/path/to/yaml/environments/default/1.yaml": []byte(`foo: FOO`),
+		"/path/to/base.gotmpl": []byte(`environments:
+  default:
+    values:
+    - environments/default/2.yaml
+
+helmDefaults:
+  tillerNamespace: {{ .Environment.Values.tillerNs }}
+`),
+		"/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`),
+		"/path/to/yaml/templates.yaml": []byte(`templates:
+  default: &default
+    missingFileHandler: Warn
+    values: ["` + "{{`" + `{{.Release.Name}}` + "`}}" + `/values.yaml"]
+`),
+	}
+	readFile := func(filename string) ([]byte, error) {
+		content, ok := files[filename]
+		if !ok {
+			return nil, fmt.Errorf("unexpected filename: %s", filename)
+		}
+		return content, nil
+	}
+	app := &App{
+		readFile:    readFile,
+		glob:        filepath.Glob,
+		abs:         filepath.Abs,
+		KubeContext: "default",
+		Env:         "default",
+		Logger:      helmexec.NewLogger(os.Stderr, "debug"),
+	}
+	st, err := app.loadDesiredStateFromYaml(yamlFile)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	if st.HelmDefaults.TillerNamespace != "TILLER_NS" {
+		t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace)
+	}
+
+	if *st.Releases[1].MissingFileHandler != "Warn" {
+		t.Errorf("unexpected releases[0].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler)
+	}
+
+	if st.Releases[1].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
+		t.Errorf("unexpected releases[0].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
+	}
+}
+
+func TestLoadDesiredStateFromYaml_MultiPartTemplate(t *testing.T) {
+	yamlFile := "/path/to/yaml/file"
+	yamlContent := []byte(`bases:
+- ../base.yaml
+---
+bases:
+- ../base.gotmpl
+---
+helmDefaults:
+  kubeContext: {{ .Environment.Values.foo }}
+---
+releases:
+- name: myrelease0
+  chart: mychart0
+---
+
+{{ readFile "templates.yaml" }}
+
+releases:
+- name: myrelease1
+  chart: mychart1
+  labels:
+    stage: pre
+    foo: bar
+- name: myrelease1
+  chart: mychart2
+  labels:
+    stage: post
+  <<: *default
+`)
+	files := map[string][]byte{
+		yamlFile: yamlContent,
+		"/path/to/base.yaml": []byte(`environments:
+  default:
+    values:
+    - environments/default/1.yaml
+`),
+		"/path/to/yaml/environments/default/1.yaml": []byte(`foo: FOO`),
+		"/path/to/base.gotmpl": []byte(`environments:
+  default:
+    values:
+    - environments/default/2.yaml
+
+helmDefaults:
+  tillerNamespace: {{ .Environment.Values.tillerNs }}
+`),
+		"/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`),
+		"/path/to/yaml/templates.yaml": []byte(`templates:
+  default: &default
+    missingFileHandler: Warn
+    values: ["` + "{{`" + `{{.Release.Name}}` + "`}}" + `/values.yaml"]
+`),
+	}
+	readFile := func(filename string) ([]byte, error) {
+		content, ok := files[filename]
+		if !ok {
+			return nil, fmt.Errorf("unexpected filename: %s", filename)
+		}
+		return content, nil
+	}
+	app := &App{
+		readFile: readFile,
+		glob:     filepath.Glob,
+		abs:      filepath.Abs,
+		Env:      "default",
+		Logger:   helmexec.NewLogger(os.Stderr, "debug"),
+	}
+	st, err := app.loadDesiredStateFromYaml(yamlFile)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	if st.HelmDefaults.TillerNamespace != "TILLER_NS" {
+		t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace)
+	}
+
+	if st.Releases[0].Name != "myrelease0" {
+		t.Errorf("unexpected releases[0].name: expected=myrelease0, got=%s", st.Releases[0].Name)
+	}
+	if st.Releases[1].Name != "myrelease1" {
+		t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", st.Releases[1].Name)
+	}
+	if st.Releases[2].Name != "myrelease1" {
+		t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", st.Releases[2].Name)
+	}
+	if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
+		t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
+	}
+	if *st.Releases[2].MissingFileHandler != "Warn" {
+		t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler)
+	}
+
+	if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
+		t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
+	}
+
+	if st.HelmDefaults.KubeContext != "FOO" {
+		t.Errorf("unexpected helmDefaults.kubeContext: expected=FOO, got=%s", st.HelmDefaults.KubeContext)
 	}
 }

pkg/app/constants.go

@@ -13,6 +13,10 @@ const (
 	ExperimentalSelectorExplicit = "explicit-selector-inheritance" // value to remove default selector inheritance to sub-helmfiles and use the explicit one
 )
 
-func isExplicitSelectorInheritanceEnabled() bool {
+func experimentalModeEnabled() bool {
-	return os.Getenv(ExperimentalEnvVar) == "true" || strings.Contains(os.Getenv(ExperimentalEnvVar), ExperimentalSelectorExplicit)
+	return os.Getenv(ExperimentalEnvVar) == "true"
+}
+
+func isExplicitSelectorInheritanceEnabled() bool {
+	return experimentalModeEnabled() || strings.Contains(os.Getenv(ExperimentalEnvVar), ExperimentalSelectorExplicit)
 }

pkg/app/desired_state_file_loader.go

@@ -0,0 +1,203 @@
+package app
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"github.com/imdario/mergo"
+	"github.com/roboll/helmfile/environment"
+	"github.com/roboll/helmfile/state"
+	"go.uber.org/zap"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+)
+
+type desiredStateLoader struct {
+	KubeContext string
+	Reverse     bool
+
+	env       string
+	namespace string
+
+	readFile func(string) ([]byte, error)
+	abs      func(string) (string, error)
+	glob     func(string) ([]string, error)
+
+	logger *zap.SugaredLogger
+}
+
+func (ld *desiredStateLoader) Load(f string) (*state.HelmState, error) {
+	return ld.loadFile(filepath.Dir(f), filepath.Base(f), true)
+}
+
+func (ld *desiredStateLoader) loadFile(baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
+	var f string
+	if filepath.IsAbs(file) {
+		f = file
+	} else {
+		f = filepath.Join(baseDir, file)
+	}
+
+	fileBytes, err := ld.readFile(f)
+	if err != nil {
+		return nil, err
+	}
+
+	ext := filepath.Ext(f)
+
+	var self *state.HelmState
+
+	if !experimentalModeEnabled() || ext == ".gotmpl" {
+		self, err = ld.renderAndLoad(
+			baseDir,
+			f,
+			fileBytes,
+			evaluateBases,
+		)
+	} else {
+		self, err = ld.load(
+			fileBytes,
+			baseDir,
+			file,
+			evaluateBases,
+		)
+	}
+
+	return self, err
+}
+
+func (a *desiredStateLoader) load(yaml []byte, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
+	c := state.NewCreator(a.logger, a.readFile, a.abs)
+	st, err := c.ParseAndLoadEnv(yaml, baseDir, file, a.env)
+	if err != nil {
+		return nil, err
+	}
+
+	helmfiles := []state.SubHelmfileSpec{}
+	for _, hf := range st.Helmfiles {
+		globPattern := hf.Path
+		var absPathPattern string
+		if filepath.IsAbs(globPattern) {
+			absPathPattern = globPattern
+		} else {
+			absPathPattern = st.JoinBase(globPattern)
+		}
+		matches, err := a.glob(absPathPattern)
+		if err != nil {
+			return nil, fmt.Errorf("failed processing %s: %v", globPattern, err)
+		}
+		sort.Strings(matches)
+		for _, match := range matches {
+			newHelmfile := hf
+			newHelmfile.Path = match
+			helmfiles = append(helmfiles, newHelmfile)
+		}
+
+	}
+	st.Helmfiles = helmfiles
+
+	if a.Reverse {
+		rev := func(i, j int) bool {
+			return j < i
+		}
+		sort.Slice(st.Releases, rev)
+		sort.Slice(st.Helmfiles, rev)
+	}
+
+	if a.KubeContext != "" {
+		if st.HelmDefaults.KubeContext != "" {
+			log.Printf("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.")
+			os.Exit(1)
+		}
+		st.HelmDefaults.KubeContext = a.KubeContext
+	}
+	if a.namespace != "" {
+		if st.Namespace != "" {
+			log.Printf("err: Cannot use option --namespace and set attribute namespace.")
+			os.Exit(1)
+		}
+		st.Namespace = a.namespace
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	if !evaluateBases {
+		if len(st.Bases) > 0 {
+			return nil, errors.New("nested `base` helmfile is unsupported. please submit a feature request if you need this!")
+		}
+
+		return st, nil
+	}
+
+	layers := []*state.HelmState{}
+	for _, b := range st.Bases {
+		base, err := a.loadFile(baseDir, b, false)
+		if err != nil {
+			return nil, err
+		}
+		layers = append(layers, base)
+	}
+	layers = append(layers, st)
+
+	for i := 1; i < len(layers); i++ {
+		if err := mergo.Merge(layers[0], layers[i], mergo.WithAppendSlice); err != nil {
+			return nil, err
+		}
+	}
+
+	return layers[0], nil
+}
+
+func (ld *desiredStateLoader) renderAndLoad(baseDir, filename string, content []byte, evaluateBases bool) (*state.HelmState, error) {
+	parts := bytes.Split(content, []byte("\n---\n"))
+
+	var finalState *state.HelmState
+	var env *environment.Environment
+
+	for i, part := range parts {
+		var yamlBuf *bytes.Buffer
+		var err error
+
+		id := fmt.Sprintf("%s.part.%d", filename, i)
+
+		if env == nil {
+			yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part)
+			if err != nil {
+				return nil, fmt.Errorf("error during %s parsing: %v", id, err)
+			}
+		} else {
+			yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part, *env)
+			if err != nil {
+				return nil, fmt.Errorf("error during %s parsing: %v", id, err)
+			}
+		}
+
+		currentState, err := ld.load(
+			yamlBuf.Bytes(),
+			baseDir,
+			filename,
+			evaluateBases,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		if finalState == nil {
+			finalState = currentState
+		} else {
+			if err := mergo.Merge(finalState, currentState, mergo.WithAppendSlice); err != nil {
+				return nil, err
+			}
+		}
+
+		env = &finalState.Env
+
+		ld.logger.Debugf("merged environment: %v", env)
+	}
+
+	return finalState, nil
+}

pkg/app/two_pass_renderer.go

@@ -3,11 +3,10 @@ package app
 import (
 	"bytes"
 	"fmt"
+	"github.com/imdario/mergo"
 	"github.com/roboll/helmfile/environment"
 	"github.com/roboll/helmfile/state"
 	"github.com/roboll/helmfile/tmpl"
-	"go.uber.org/zap"
-	"path/filepath"
 	"strings"
 )
 
@@ -20,61 +19,81 @@ func prependLineNumbers(text string) string {
 	return buf.String()
 }
 
-type twoPassRenderer struct {
+func (r *desiredStateLoader) renderEnvironment(firstPassEnv environment.Environment, baseDir, filename string, content []byte) environment.Environment {
-	reader    func(string) ([]byte, error)
-	env       string
-	namespace string
-	filename  string
-	logger    *zap.SugaredLogger
-	abs       func(string) (string, error)
-}
-
-func (r *twoPassRenderer) renderEnvironment(content []byte) environment.Environment {
-	firstPassEnv := environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
 	tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
-	firstPassRenderer := tmpl.NewFirstPassRenderer(filepath.Dir(r.filename), tmplData)
+	firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData)
 
 	// parse as much as we can, tolerate errors, this is a preparse
 	yamlBuf, err := firstPassRenderer.RenderTemplateContentToBuffer(content)
 	if err != nil && r.logger != nil {
-		r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content)))
+		r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", filename, prependLineNumbers(string(content)))
 		if yamlBuf == nil { // we have a template syntax error, let the second parse report
 			r.logger.Debugf("template syntax error: %v", err)
 			return firstPassEnv
 		}
 	}
-	c := state.NewCreator(r.logger, r.reader, r.abs)
+	c := state.NewCreator(r.logger, r.readFile, r.abs)
 	c.Strict = false
 	// create preliminary state, as we may have an environment. Tolerate errors.
-	prestate, err := c.CreateFromYaml(yamlBuf.Bytes(), r.filename, r.env)
+	prestate, err := c.ParseAndLoadEnv(yamlBuf.Bytes(), baseDir, filename, r.env)
 	if err != nil && r.logger != nil {
 		switch err.(type) {
 		case *state.StateLoadError:
 			r.logger.Infof("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err)
 		}
-		r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String()))
+		r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String()))
 	}
+
 	if prestate != nil {
-		firstPassEnv = prestate.Env
+		intEnv := environment.Environment{Name: firstPassEnv.Name}
+		if err := mergo.Merge(&intEnv, &firstPassEnv, mergo.WithAppendSlice); err != nil {
+			r.logger.Debugf("error in first-pass rendering: result of \"%s\": %v", filename, err)
+			return firstPassEnv
+		}
+		if err := mergo.Merge(&intEnv, &prestate.Env, mergo.WithAppendSlice); err != nil {
+			r.logger.Debugf("error in first-pass rendering: result of \"%s\": %v", filename, err)
+			return firstPassEnv
+		}
+		firstPassEnv = intEnv
 	}
 	return firstPassEnv
 }
 
-func (r *twoPassRenderer) renderTemplate(content []byte) (*bytes.Buffer, error) {
+func (r *desiredStateLoader) renderTemplatesToYaml(baseDir, filename string, content []byte, context ...environment.Environment) (*bytes.Buffer, error) {
+	var env environment.Environment
+
+	if len(context) > 0 {
+		env = context[0]
+	} else {
+		env = environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
+	}
+
+	return r.twoPassRenderTemplateToYaml(env, baseDir, filename, content)
+}
+
+func (r *desiredStateLoader) twoPassRenderTemplateToYaml(initEnv environment.Environment, baseDir, filename string, content []byte) (*bytes.Buffer, error) {
 	// try a first pass render. This will always succeed, but can produce a limited env
-	firstPassEnv := r.renderEnvironment(content)
+	if r.logger != nil {
+		r.logger.Debugf("first-pass rendering input of \"%s\": %v", filename, initEnv)
+	}
+
+	firstPassEnv := r.renderEnvironment(initEnv, baseDir, filename, content)
+
+	if r.logger != nil {
+		r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, firstPassEnv)
+	}
 
 	tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
-	secondPassRenderer := tmpl.NewFileRenderer(r.reader, filepath.Dir(r.filename), tmplData)
+	secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData)
 	yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content)
 	if err != nil {
 		if r.logger != nil {
-			r.logger.Debugf("second-pass rendering failed, input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content)))
+			r.logger.Debugf("second-pass rendering failed, input of \"%s\":\n%s", filename, prependLineNumbers(string(content)))
 		}
 		return nil, err
 	}
 	if r.logger != nil {
-		r.logger.Debugf("second-pass rendering result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String()))
+		r.logger.Debugf("second-pass rendering result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String()))
 	}
 	return yamlBuf, nil
 }

pkg/app/two_pass_renderer_test.go

@@ -12,12 +12,11 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-func makeRenderer(readFile func(string) ([]byte, error), env string) *twoPassRenderer {
+func makeLoader(readFile func(string) ([]byte, error), env string) *desiredStateLoader {
-	return &twoPassRenderer{
+	return &desiredStateLoader{
-		reader:    readFile,
+		readFile:  readFile,
 		env:       env,
 		namespace: "namespace",
-		filename:  "",
 		logger:    helmexec.NewLogger(os.Stdout, "debug"),
 		abs:       filepath.Abs,
 	}
@@ -51,8 +50,8 @@ releases:
 		return []byte(""), nil
 	}
 
-	r := makeRenderer(fileReader, "staging")
+	r := makeLoader(fileReader, "staging")
-	yamlBuf, err := r.renderTemplate(yamlContent)
+	yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent)
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
@@ -102,9 +101,9 @@ releases:
 		return defaultValuesYaml, nil
 	}
 
-	r := makeRenderer(fileReader, "staging")
+	r := makeLoader(fileReader, "staging")
 	// test the double rendering
-	yamlBuf, err := r.renderTemplate(yamlContent)
+	yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -151,9 +150,9 @@ releases:
 		return defaultValuesYaml, nil
 	}
 
-	r := makeRenderer(fileReader, "staging")
+	r := makeLoader(fileReader, "staging")
 	// test the double rendering
-	_, err := r.renderTemplate(yamlContent)
+	_, err := r.renderTemplatesToYaml("", "", yamlContent)
 
 	if !strings.Contains(err.Error(), "stringTemplate:8") {
 		t.Fatalf("error should contain a stringTemplate error (reference to unknow key) %v", err)
@@ -190,8 +189,8 @@ releases:
 		return defaultValuesYamlGotmpl, nil
 	}
 
-	r := makeRenderer(fileReader, "staging")
+	r := makeLoader(fileReader, "staging")
-	rendered, _ := r.renderTemplate(yamlContent)
+	rendered, _ := r.renderTemplatesToYaml("", "", yamlContent)
 
 	var state state.HelmState
 	yaml.Unmarshal(rendered.Bytes(), &state)
@@ -217,8 +216,8 @@ func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) {
 		return defaultValuesYaml, nil
 	}
 
-	r := makeRenderer(fileReader, "staging")
+	r := makeLoader(fileReader, "staging")
-	yamlBuf, err := r.renderTemplate(yamlContent)
+	yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -231,7 +230,7 @@ func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) {
 	}
 }
 
-func TestReadFromYaml_HelfileShouldBeResilentToTemplateErrors(t *testing.T) {
+func TestReadFromYaml_HelmfileShouldBeResilentToTemplateErrors(t *testing.T) {
 	yamlContent := []byte(`environments:
   staging:
 	production:
@@ -248,8 +247,8 @@ releases:
 		return yamlContent, nil
 	}
 
-	r := makeRenderer(fileReader, "staging")
+	r := makeLoader(fileReader, "staging")
-	_, err := r.renderTemplate(yamlContent)
+	_, err := r.renderTemplatesToYaml("", "", yamlContent)
 	if err == nil {
 		t.Fatalf("wanted error, none returned")
 	}

state/create.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -33,16 +32,6 @@ func (e *UndefinedEnvError) Error() string {
 	return e.msg
 }
 
-func createFromYaml(content []byte, file string, env string, logger *zap.SugaredLogger) (*HelmState, error) {
-	c := &creator{
-		logger,
-		ioutil.ReadFile,
-		filepath.Abs,
-		true,
-	}
-	return c.CreateFromYaml(content, file, env)
-}
-
 type creator struct {
 	logger   *zap.SugaredLogger
 	readFile func(string) ([]byte, error)
@@ -60,15 +49,12 @@ func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error)
 	}
 }
 
-func (c *creator) CreateFromYaml(content []byte, file string, env string) (*HelmState, error) {
+// Parses YAML into HelmState, while loading environment values files relative to the `cwd`
+func (c *creator) ParseAndLoadEnv(content []byte, baseDir, file string, env string) (*HelmState, error) {
 	var state HelmState
 
-	basePath, err := c.abs(filepath.Dir(file))
-	if err != nil {
-		return nil, &StateLoadError{fmt.Sprintf("failed to read %s", file), err}
-	}
 	state.FilePath = file
-	state.basePath = basePath
+	state.basePath = baseDir
 
 	decoder := yaml.NewDecoder(bytes.NewReader(content))
 	if !c.Strict {

state/create_test.go

@@ -2,6 +2,8 @@ package state
 
 import (
 	"fmt"
+	"go.uber.org/zap"
+	"io/ioutil"
 	"path/filepath"
 	"reflect"
 	"testing"
@@ -10,6 +12,16 @@ import (
 	"gotest.tools/assert/cmp"
 )
 
+func createFromYaml(content []byte, file string, env string, logger *zap.SugaredLogger) (*HelmState, error) {
+	c := &creator{
+		logger,
+		ioutil.ReadFile,
+		filepath.Abs,
+		true,
+	}
+	return c.ParseAndLoadEnv(content, filepath.Dir(file), file, env)
+}
+
 func TestReadFromYaml(t *testing.T) {
 	yamlFile := "example/path/to/yaml/file"
 	yamlContent := []byte(`releases:
@@ -101,7 +113,7 @@ bar: {{ readFile "bar.txt" }}
 		return nil, fmt.Errorf("unexpected filename: %s", filename)
 	}
 
-	state, err := NewCreator(logger, readFile, filepath.Abs).CreateFromYaml(yamlContent, yamlFile, "production")
+	state, err := NewCreator(logger, readFile, filepath.Abs).ParseAndLoadEnv(yamlContent, filepath.Dir(yamlFile), yamlFile, "production")
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}

state/state.go

@@ -30,6 +30,8 @@ type HelmState struct {
 	basePath     string
 	Environments map[string]EnvironmentSpec
 	FilePath     string
+
+	Bases              []string          `yaml:"bases"`
 	HelmDefaults       HelmSpec          `yaml:"helmDefaults"`
 	Helmfiles          []SubHelmfileSpec `yaml:"helmfiles"`
 	DeprecatedContext  string            `yaml:"context"`