feat: split-render-merge helmfile.yaml parts

This splits your helmfile.yaml by the YAML document separator "---" before evaluating go template expressions as outlined in https://github.com/roboll/helmfile/issues/388#issuecomment-491710348

docs/writing-helmfile.md

@@ -125,23 +125,26 @@ environments:
   production:
 ```
 
-At run time, template expressions in your `helmfile.yaml` are executed:
+At run time, `bases` in your `helmfile.yaml` are evaluated to produce:
 
 ```yaml
+# commons.yaml
 releases:
 - name: metricbaet
   chart: stable/metricbeat
 ---
+# environments.yaml
 environments:
   development:
   production:
 ---
+# helmfile.yaml
 releases:
 - name: myapp
   chart: mychart
 ```
 
-Resulting YAML documents are merged in the order of occurrence,
+Finally the resulting YAML documents are merged in the order of occurrence,
 so that your `helmfile.yaml` becomes:
 
 ```yaml
@@ -159,3 +162,5 @@ releases:
 Great!
 
 Now, repeat the above steps for each your `helmfile.yaml`, so that all your helmfiles becomes DRY.
+
+Please also see [the discussion in the issue 388](https://github.com/roboll/helmfile/issues/388#issuecomment-491710348) for more advanced layering examples.

pkg/app/app_test.go

@@ -702,8 +702,8 @@ releases:
 helmDefaults:
   tillerNamespace: {{ .Environment.Values.tillerNs }}
 `),
-"/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`),
+		"/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`),
-"/path/to/yaml/templates.yaml": []byte(`templates:
+		"/path/to/yaml/templates.yaml": []byte(`templates:
   default: &default
     missingFileHandler: Warn
     values: ["` + "{{`" + `{{.Release.Name}}` + "`}}" + `/values.yaml"]
@@ -741,3 +741,104 @@ helmDefaults:
 		t.Errorf("unexpected releases[0].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
 	}
 }
+
+func TestLoadDesiredStateFromYaml_MultiPartTemplate(t *testing.T) {
+	yamlFile := "/path/to/yaml/file"
+	yamlContent := []byte(`bases:
+- ../base.yaml
+---
+bases:
+- ../base.gotmpl
+---
+helmDefaults:
+  kubeContext: {{ .Environment.Values.foo }}
+---
+releases:
+- name: myrelease0
+  chart: mychart0
+---
+
+{{ readFile "templates.yaml" }}
+
+releases:
+- name: myrelease1
+  chart: mychart1
+  labels:
+    stage: pre
+    foo: bar
+- name: myrelease1
+  chart: mychart2
+  labels:
+    stage: post
+  <<: *default
+`)
+	files := map[string][]byte{
+		yamlFile: yamlContent,
+		"/path/to/base.yaml": []byte(`environments:
+  default:
+    values:
+    - environments/default/1.yaml
+`),
+		"/path/to/yaml/environments/default/1.yaml": []byte(`foo: FOO`),
+		"/path/to/base.gotmpl": []byte(`environments:
+  default:
+    values:
+    - environments/default/2.yaml
+
+helmDefaults:
+  tillerNamespace: {{ .Environment.Values.tillerNs }}
+`),
+		"/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`),
+		"/path/to/yaml/templates.yaml": []byte(`templates:
+  default: &default
+    missingFileHandler: Warn
+    values: ["` + "{{`" + `{{.Release.Name}}` + "`}}" + `/values.yaml"]
+`),
+	}
+	readFile := func(filename string) ([]byte, error) {
+		content, ok := files[filename]
+		if !ok {
+			return nil, fmt.Errorf("unexpected filename: %s", filename)
+		}
+		return content, nil
+	}
+	app := &App{
+		readFile: readFile,
+		glob:     filepath.Glob,
+		abs:      filepath.Abs,
+		Env:      "default",
+		Logger:   helmexec.NewLogger(os.Stderr, "debug"),
+	}
+	st, err := app.loadDesiredStateFromYaml(yamlFile)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	if st.HelmDefaults.TillerNamespace != "TILLER_NS" {
+		t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace)
+	}
+
+	if st.Releases[0].Name != "myrelease0" {
+		t.Errorf("unexpected releases[0].name: expected=myrelease0, got=%s", st.Releases[0].Name)
+	}
+	if st.Releases[1].Name != "myrelease1" {
+		t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", st.Releases[1].Name)
+	}
+	if st.Releases[2].Name != "myrelease1" {
+		t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", st.Releases[2].Name)
+	}
+	if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
+		t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
+	}
+	if *st.Releases[2].MissingFileHandler != "Warn" {
+		t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler)
+	}
+
+	if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
+		t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
+	}
+
+	if st.HelmDefaults.KubeContext != "FOO" {
+		t.Errorf("unexpected helmDefaults.kubeContext: expected=FOO, got=%s", st.HelmDefaults.KubeContext)
+	}
+}

pkg/app/desired_state_file_loader.go

@@ -1,8 +1,11 @@
 package app
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
 	"github.com/imdario/mergo"
+	"github.com/roboll/helmfile/environment"
 	"github.com/roboll/helmfile/state"
 	"go.uber.org/zap"
 	"log"
@@ -26,10 +29,10 @@ type desiredStateLoader struct {
 }
 
 func (ld *desiredStateLoader) Load(f string) (*state.HelmState, error) {
-	return ld.load(filepath.Dir(f), filepath.Base(f), true)
+	return ld.loadFile(filepath.Dir(f), filepath.Base(f), true)
 }
 
-func (ld *desiredStateLoader) load(baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
+func (ld *desiredStateLoader) loadFile(baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
 	var f string
 	if filepath.IsAbs(file) {
 		f = file
@@ -44,51 +47,28 @@ func (ld *desiredStateLoader) load(baseDir, file string, evaluateBases bool) (*s
 
 	ext := filepath.Ext(f)
 
-	var yamlBytes []byte
+	var self *state.HelmState
+
 	if !experimentalModeEnabled() || ext == ".gotmpl" {
-		yamlBuf, err := ld.renderTemplateToYaml(baseDir, f, fileBytes)
+		self, err = ld.renderAndLoad(
-		if err != nil {
+			baseDir,
-			return nil, fmt.Errorf("error during %s parsing: %v", f, err)
+			f,
-		}
+			fileBytes,
-		yamlBytes = yamlBuf.Bytes()
+			evaluateBases,
+		)
 	} else {
-		yamlBytes = fileBytes
+		self, err = ld.load(
+			fileBytes,
+			baseDir,
+			file,
+			evaluateBases,
+		)
 	}
 
-	self, err := ld.loadYaml(
+	return self, err
-		yamlBytes,
-		baseDir,
-		file,
-	)
-
-	if err != nil {
-		return nil, err
-	}
-
-	if !evaluateBases {
-		return self, nil
-	}
-
-	layers := []*state.HelmState{}
-	for _, b := range self.Bases {
-		base, err := ld.load(baseDir, b, false)
-		if err != nil {
-			return nil, err
-		}
-		layers = append(layers, base)
-	}
-	layers = append(layers, self)
-
-	for i := 1; i < len(layers); i++ {
-		if err := mergo.Merge(layers[0], layers[i], mergo.WithAppendSlice); err != nil {
-			return nil, err
-		}
-	}
-
-	return layers[0], nil
 }
 
-func (a *desiredStateLoader) loadYaml(yaml []byte, baseDir, file string) (*state.HelmState, error) {
+func (a *desiredStateLoader) load(yaml []byte, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
 	c := state.NewCreator(a.logger, a.readFile, a.abs)
 	st, err := c.ParseAndLoadEnv(yaml, baseDir, file, a.env)
 	if err != nil {
@@ -141,5 +121,83 @@ func (a *desiredStateLoader) loadYaml(yaml []byte, baseDir, file string) (*state
 		st.Namespace = a.namespace
 	}
 
-	return st, nil
+	if err != nil {
+		return nil, err
+	}
+
+	if !evaluateBases {
+		if len(st.Bases) > 0 {
+			return nil, errors.New("nested `base` helmfile is unsupported. please submit a feature request if you need this!")
+		}
+
+		return st, nil
+	}
+
+	layers := []*state.HelmState{}
+	for _, b := range st.Bases {
+		base, err := a.loadFile(baseDir, b, false)
+		if err != nil {
+			return nil, err
+		}
+		layers = append(layers, base)
+	}
+	layers = append(layers, st)
+
+	for i := 1; i < len(layers); i++ {
+		if err := mergo.Merge(layers[0], layers[i], mergo.WithAppendSlice); err != nil {
+			return nil, err
+		}
+	}
+
+	return layers[0], nil
+}
+
+func (ld *desiredStateLoader) renderAndLoad(baseDir, filename string, content []byte, evaluateBases bool) (*state.HelmState, error) {
+	parts := bytes.Split(content, []byte("\n---\n"))
+
+	var finalState *state.HelmState
+	var env *environment.Environment
+
+	for i, part := range parts {
+		var yamlBuf *bytes.Buffer
+		var err error
+
+		id := fmt.Sprintf("%s.part.%d", filename, i)
+
+		if env == nil {
+			yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part)
+			if err != nil {
+				return nil, fmt.Errorf("error during %s parsing: %v", id, err)
+			}
+		} else {
+			yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part, *env)
+			if err != nil {
+				return nil, fmt.Errorf("error during %s parsing: %v", id, err)
+			}
+		}
+
+		currentState, err := ld.load(
+			yamlBuf.Bytes(),
+			baseDir,
+			filename,
+			evaluateBases,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		if finalState == nil {
+			finalState = currentState
+		} else {
+			if err := mergo.Merge(finalState, currentState, mergo.WithAppendSlice); err != nil {
+				return nil, err
+			}
+		}
+
+		env = &finalState.Env
+
+		ld.logger.Debugf("merged environment: %v", env)
+	}
+
+	return finalState, nil
 }

pkg/app/two_pass_renderer.go

@@ -3,6 +3,7 @@ package app
 import (
 	"bytes"
 	"fmt"
+	"github.com/imdario/mergo"
 	"github.com/roboll/helmfile/environment"
 	"github.com/roboll/helmfile/state"
 	"github.com/roboll/helmfile/tmpl"
@@ -18,8 +19,7 @@ func prependLineNumbers(text string) string {
 	return buf.String()
 }
 
-func (r *desiredStateLoader) renderEnvironment(baseDir, filename string, content []byte) environment.Environment {
+func (r *desiredStateLoader) renderEnvironment(firstPassEnv environment.Environment, baseDir, filename string, content []byte) environment.Environment {
-	firstPassEnv := environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
 	tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
 	firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData)
 
@@ -43,15 +43,45 @@ func (r *desiredStateLoader) renderEnvironment(baseDir, filename string, content
 		}
 		r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String()))
 	}
+
 	if prestate != nil {
-		firstPassEnv = prestate.Env
+		intEnv := environment.Environment{Name: firstPassEnv.Name}
+		if err := mergo.Merge(&intEnv, &firstPassEnv, mergo.WithAppendSlice); err != nil {
+			r.logger.Debugf("error in first-pass rendering: result of \"%s\": %v", filename, err)
+			return firstPassEnv
+		}
+		if err := mergo.Merge(&intEnv, &prestate.Env, mergo.WithAppendSlice); err != nil {
+			r.logger.Debugf("error in first-pass rendering: result of \"%s\": %v", filename, err)
+			return firstPassEnv
+		}
+		firstPassEnv = intEnv
 	}
 	return firstPassEnv
 }
 
-func (r *desiredStateLoader) renderTemplateToYaml(baseDir, filename string, content []byte) (*bytes.Buffer, error) {
+func (r *desiredStateLoader) renderTemplatesToYaml(baseDir, filename string, content []byte, context ...environment.Environment) (*bytes.Buffer, error) {
+	var env environment.Environment
+
+	if len(context) > 0 {
+		env = context[0]
+	} else {
+		env = environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
+	}
+
+	return r.twoPassRenderTemplateToYaml(env, baseDir, filename, content)
+}
+
+func (r *desiredStateLoader) twoPassRenderTemplateToYaml(initEnv environment.Environment, baseDir, filename string, content []byte) (*bytes.Buffer, error) {
 	// try a first pass render. This will always succeed, but can produce a limited env
-	firstPassEnv := r.renderEnvironment(baseDir, filename, content)
+	if r.logger != nil {
+		r.logger.Debugf("first-pass rendering input of \"%s\": %v", filename, initEnv)
+	}
+
+	firstPassEnv := r.renderEnvironment(initEnv, baseDir, filename, content)
+
+	if r.logger != nil {
+		r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, firstPassEnv)
+	}
 
 	tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
 	secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData)

pkg/app/two_pass_renderer_test.go

@@ -51,7 +51,7 @@ releases:
 	}
 
 	r := makeLoader(fileReader, "staging")
-	yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent)
+	yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent)
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
@@ -103,7 +103,7 @@ releases:
 
 	r := makeLoader(fileReader, "staging")
 	// test the double rendering
-	yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent)
+	yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -152,7 +152,7 @@ releases:
 
 	r := makeLoader(fileReader, "staging")
 	// test the double rendering
-	_, err := r.renderTemplateToYaml("", "", yamlContent)
+	_, err := r.renderTemplatesToYaml("", "", yamlContent)
 
 	if !strings.Contains(err.Error(), "stringTemplate:8") {
 		t.Fatalf("error should contain a stringTemplate error (reference to unknow key) %v", err)
@@ -190,7 +190,7 @@ releases:
 	}
 
 	r := makeLoader(fileReader, "staging")
-	rendered, _ := r.renderTemplateToYaml("", "", yamlContent)
+	rendered, _ := r.renderTemplatesToYaml("", "", yamlContent)
 
 	var state state.HelmState
 	yaml.Unmarshal(rendered.Bytes(), &state)
@@ -217,7 +217,7 @@ func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) {
 	}
 
 	r := makeLoader(fileReader, "staging")
-	yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent)
+	yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -248,7 +248,7 @@ releases:
 	}
 
 	r := makeLoader(fileReader, "staging")
-	_, err := r.renderTemplateToYaml("", "", yamlContent)
+	_, err := r.renderTemplatesToYaml("", "", yamlContent)
 	if err == nil {
 		t.Fatalf("wanted error, none returned")
 	}