feat: double render the helmfile (#308)

This allows using the environment values defined in the environments: section of helmfile.yaml to be used from other sections of the file.

This works by having two template renderers, the first-pass and the second-pass renderer.

The first-pass render renders a helmfile.yaml template with replacing template functions has side-effects with noop. So, use only funcs that don't have side-effects to compose your environment values.

Then the second-pass renderer renders the same helmfile.yaml template, but with the environment values loaded by the first-pass renderer.

The implementation uses a buffer instead of re-reading the file twice.

Resolves #297

README.md

@@ -40,7 +40,7 @@ repositories:
 context: kube-context					 # kube-context (--kube-context)
 
 #default values to set for args along with dedicated keys that can be set by contributers, cli args take precedence overe these
-helmDefaults:           
+helmDefaults:
   tillerNamespace: tiller-namespace  #dedicated default key for tiller-namespace
   kubeContext: kube-context          #dedicated default key for kube-context
   # additional and global args passed to helm
@@ -396,7 +396,7 @@ releases:
   # snip
 {{ end }}
 - name: myapp
-  # snip  
+  # snip
 ```
 
 ## Environment Values
@@ -424,6 +424,7 @@ releases:
 
 ```yaml
 domain: prod.example.com
+releaseName: prod
 ```
 
 `values.yaml.gotmpl`
@@ -435,6 +436,38 @@ domain: {{ .Environment.Values.domain | default "dev.example.com" }}
 `helmfile sync` installs `myapp` with the value `domain=dev.example.com`,
 whereas `helmfile --environment production sync` installs the app with the value `domain=production.example.com`.
 
+For even more flexibility, you can now use values declared in the `environments:` section in other parts of your helmfiles:
+
+consider:
+`default.yaml`
+
+```yaml
+domain: dev.example.com
+releaseName: dev
+```
+
+```yaml
+environments:
+  default:
+    values:
+    - default.yaml
+  production:
+    values:
+    - production.yaml #  template directives with potential side-effects like `exec` and `readFile` will NOT be executed
+    - other.yaml.gotmpl  #  `exec` and `readFile` will be honoured
+
+releases:
+- name: myapp-{{ .Environment.Values.releaseName }} # release name will be one of `dev` or `prod` depending on selected environment
+  values:
+  - values.yaml.gotmpl
+
+{{ if eq (.Environment.Values.releaseName "prod" ) }}
+# this release would be installed only if selected environment is `production`
+- name: production-specific-release
+  ...
+{{ end }}
+```
+
 ## Environment Secrets
 
 Environment Secrets are encrypted versions of `Environment Values`.

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
 	"log"
 	"os"
@@ -588,6 +589,67 @@ func (e *noMatchingHelmfileError) Error() string {
 	)
 }
 
+func prependLineNumbers(text string) string {
+	buf := bytes.NewBufferString("")
+	lines := strings.Split(text, "\n")
+	for i, line := range lines {
+		buf.WriteString(fmt.Sprintf("%2d: %s\n", i, line))
+	}
+	return buf.String()
+}
+
+type twoPassRenderer struct {
+	reader   func(string) ([]byte, error)
+	env      string
+	filename string
+	logger   *zap.SugaredLogger
+	abs      func(string) (string, error)
+}
+
+func (r *twoPassRenderer) renderEnvironment(content []byte) environment.Environment {
+	firstPassEnv := environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
+	firstPassRenderer := tmpl.NewFirstPassRenderer(firstPassEnv)
+
+	// parse as much as we can, tolerate errors, this is a preparse
+	yamlBuf, err := firstPassRenderer.RenderTemplateContentToBuffer(content)
+	if err != nil && logger != nil {
+		r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content)))
+	}
+	c := state.NewCreator(r.logger, r.reader, r.abs)
+	c.Strict = false
+	// create preliminary state, as we may have an environment. Tolerate errors.
+	prestate, err := c.CreateFromYaml(yamlBuf.Bytes(), r.filename, r.env)
+	if err != nil && r.logger != nil {
+		switch err.(type) {
+		case *state.StateLoadError:
+			r.logger.Infof("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err)
+		}
+		r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String()))
+	}
+	if prestate != nil {
+		firstPassEnv = prestate.Env
+	}
+	return firstPassEnv
+}
+
+func (r *twoPassRenderer) renderTemplate(content []byte) (*bytes.Buffer, error) {
+	// try a first pass render. This will always succeed, but can produce a limited env
+	firstPassEnv := r.renderEnvironment(content)
+
+	secondPassRenderer := tmpl.NewFileRenderer(r.reader, "", firstPassEnv)
+	yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content)
+	if err != nil {
+		if r.logger != nil {
+			r.logger.Debugf("second-pass rendering failed, input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content)))
+		}
+		return nil, err
+	}
+	if r.logger != nil {
+		r.logger.Debugf("second-pass rendering result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String()))
+	}
+	return yamlBuf, nil
+}
+
 func (a *app) FindAndIterateOverDesiredStates(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) []error, namespace string, selectors []string, env string) error {
 	desiredStateFiles, err := a.findDesiredStateFiles(fileOrDir)
 	if err != nil {
@@ -596,10 +658,23 @@ func (a *app) FindAndIterateOverDesiredStates(fileOrDir string, converge func(*s
 	noMatchInHelmfiles := true
 	for _, f := range desiredStateFiles {
 		a.logger.Debugf("Processing %s", f)
-		yamlBuf, err := tmpl.NewFileRenderer(a.readFile, "", environment.Environment{Name: env, Values: map[string]interface{}(nil)}).RenderTemplateFileToBuffer(f)
+
+		content, err := a.readFile(f)
 		if err != nil {
 			return err
 		}
+		// render template, in two runs
+		r := &twoPassRenderer{
+			reader:   a.readFile,
+			env:      env,
+			filename: f,
+			logger:   a.logger,
+			abs:      a.abs,
+		}
+		yamlBuf, err := r.renderTemplate(content)
+		if err != nil {
+			return fmt.Errorf("error during %s parsing: %v", f, err)
+		}
 
 		st, noMatchInThisHelmfile, err := a.loadDesiredStateFromYaml(
 			yamlBuf.Bytes(),
@@ -757,7 +832,7 @@ func (a *app) loadDesiredStateFromYaml(yaml []byte, file string, namespace strin
 
 	releaseNameCounts := map[string]int{}
 	for _, r := range st.Releases {
-		releaseNameCounts[r.Name] += 1
+		releaseNameCounts[r.Name]++
 	}
 	for name, c := range releaseNameCounts {
 		if c > 1 {

main_test.go

@@ -1,9 +1,14 @@
 package main
 
 import (
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
+	"strings"
 	"testing"
+
+	"github.com/roboll/helmfile/state"
+	"gopkg.in/yaml.v2"
 )
 
 // See https://github.com/roboll/helmfile/issues/193
@@ -35,3 +40,195 @@ func TestReadFromYaml_DuplicateReleaseName(t *testing.T) {
 		t.Errorf("unexpected error happened: %v", err)
 	}
 }
+
+func makeRenderer(readFile func(string) ([]byte, error), env string) *twoPassRenderer {
+	return &twoPassRenderer{
+		reader:   readFile,
+		env:      env,
+		filename: "",
+		logger:   logger,
+		abs:      filepath.Abs,
+	}
+}
+
+func TestReadFromYaml_MakeEnvironmentHasNoSideEffects(t *testing.T) {
+
+	yamlContent := []byte(`
+environments:
+  staging:
+    values:
+    - default/values.yaml
+  production:
+
+releases:
+- name: {{ readFile "other/default/values.yaml" }}
+  chart: mychart1
+`)
+
+	fileReaderCalls := 0
+	// make a reader that returns a simulated context
+	fileReader := func(filename string) ([]byte, error) {
+		expectedFilename := filepath.Clean("default/values.yaml")
+		if !strings.HasSuffix(filename, expectedFilename) {
+			return nil, fmt.Errorf("unexpected filename: expected=%s, actual=%s", expectedFilename, filename)
+		}
+		fileReaderCalls++
+		if fileReaderCalls == 2 {
+			return []byte("SecondPass"), nil
+		}
+		return []byte(""), nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	yamlBuf, err := r.renderTemplate(yamlContent)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	var state state.HelmState
+	err = yaml.Unmarshal(yamlBuf.Bytes(), &state)
+
+	if fileReaderCalls > 2 {
+		t.Error("reader should be called only twice")
+	}
+
+	if state.Releases[0].Name != "SecondPass" {
+		t.Errorf("release name should have ben set as SecondPass")
+	}
+}
+
+func TestReadFromYaml_RenderTemplate(t *testing.T) {
+
+	defaultValuesYalm := []byte(`
+releaseName: "hello"
+conditionalReleaseTag: "yes"
+`)
+
+	yamlContent := []byte(`
+environments:
+  staging:
+    values:
+    - default/values.yaml
+  production:
+
+releases:
+- name: {{ .Environment.Values.releaseName }}
+  chart: mychart1
+
+{{ if (eq .Environment.Values.conditionalReleaseTag "yes") }}
+- name: conditionalRelease
+{{ end }}
+
+`)
+
+	// make a reader that returns a simulated context
+	fileReader := func(filename string) ([]byte, error) {
+		expectedFilename := filepath.Clean("default/values.yaml")
+		if !strings.HasSuffix(filename, expectedFilename) {
+			return nil, fmt.Errorf("unexpected filename: expected=%s, actual=%s", expectedFilename, filename)
+		}
+		return defaultValuesYalm, nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	// test the double rendering
+	yamlBuf, err := r.renderTemplate(yamlContent)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	var state state.HelmState
+	err = yaml.Unmarshal(yamlBuf.Bytes(), &state)
+
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	if len(state.Releases) != 2 {
+		t.Fatal("there should be 2 releases")
+	}
+
+	if state.Releases[0].Name != "hello" {
+		t.Errorf("release name should be hello")
+	}
+
+	if state.Releases[1].Name != "conditionalRelease" {
+		t.Error("conditional release should have been present")
+	}
+}
+
+func TestReadFromYaml_RenderTemplateWithValuesReferenceError(t *testing.T) {
+	defaultValuesYalm := []byte("")
+
+	yamlContent := []byte(`
+environments:
+  staging:
+    values:
+    - default/values.yaml
+  production:
+
+{{ if (eq .Environment.Values.releaseName "a") }} # line 8
+releases:
+- name: a
+	chart: mychart1
+{{ end }}
+`)
+
+	// make a reader that returns a simulated context
+	fileReader := func(filename string) ([]byte, error) {
+		return defaultValuesYalm, nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	// test the double rendering
+	_, err := r.renderTemplate(yamlContent)
+
+	if !strings.Contains(err.Error(), "stringTemplate:8") {
+		t.Fatalf("error should contain a stringTemplate error (reference to unknow key) %v", err)
+	}
+}
+
+// This test shows that a gotmpl reference will get rendered correctly
+// even if the pre-render disables the readFile and exec functions.
+// This does not apply to .gotmpl files, which is a nice side-effect.
+func TestReadFromYaml_RenderTemplateWithGotmpl(t *testing.T) {
+
+	defaultValuesYalmGotmpl := []byte(`
+releaseName: {{ readFile "nonIgnoredFile" }}
+`)
+
+	yamlContent := []byte(`
+environments:
+  staging:
+    values:
+    - values.yaml.gotmpl
+  production:
+
+{{ if (eq .Environment.Values.releaseName "release-a") }} # line 8
+releases:
+- name: a
+  chart: mychart1
+{{ end }}
+`)
+
+	fileReader := func(filename string) ([]byte, error) {
+		if strings.HasSuffix(filename, "nonIgnoredFile") {
+			return []byte("release-a"), nil
+		}
+		return defaultValuesYalmGotmpl, nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	rendered, _ := r.renderTemplate(yamlContent)
+
+	var state state.HelmState
+	yaml.Unmarshal(rendered.Bytes(), &state)
+
+	if len(state.Releases) != 1 {
+		t.Fatal("there should be 1 release")
+	}
+
+	if state.Releases[0].Name != "a" {
+		t.Fatal("release should have been declared")
+	}
+}

state/create.go

@@ -35,6 +35,7 @@ func createFromYaml(content []byte, file string, env string, logger *zap.Sugared
 		logger,
 		ioutil.ReadFile,
 		filepath.Abs,
+		true,
 	}
 	return c.CreateFromYaml(content, file, env)
 }
@@ -43,6 +44,8 @@ type creator struct {
 	logger   *zap.SugaredLogger
 	readFile func(string) ([]byte, error)
 	abs      func(string) (string, error)
+
+	Strict bool
 }
 
 func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), abs func(string) (string, error)) *creator {
@@ -50,6 +53,7 @@ func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error)
 		logger:   logger,
 		readFile: readFile,
 		abs:      abs,
+		Strict:   true,
 	}
 }
 
@@ -62,7 +66,11 @@ func (c *creator) CreateFromYaml(content []byte, file string, env string) (*Helm
 	}
 	state.basePath = basePath
 
-	if err := yaml.UnmarshalStrict(content, &state); err != nil {
+	unmarshal := yaml.UnmarshalStrict
+	if !c.Strict {
+		unmarshal = yaml.Unmarshal
+	}
+	if err := unmarshal(content, &state); err != nil {
 		return nil, &StateLoadError{fmt.Sprintf("failed to read %s", file), err}
 	}
 	state.FilePath = file
@@ -81,7 +89,7 @@ func (c *creator) CreateFromYaml(content []byte, file string, env string) (*Helm
 	if err != nil {
 		return nil, &StateLoadError{fmt.Sprintf("failed to read %s", file), err}
 	}
-	state.env = *e
+	state.Env = *e
 
 	state.readFile = c.readFile
 

state/create_test.go

@@ -103,7 +103,7 @@ bar: {{ readFile "bar.txt" }}
 		t.Errorf("unexpected error: %v", err)
 	}
 
-	actual := state.env.Values
+	actual := state.Env.Values
 	if !reflect.DeepEqual(actual, expected) {
 		t.Errorf("unexpected environment values: expected=%v, actual=%v", expected, actual)
 	}

state/state.go

@@ -37,7 +37,7 @@ type HelmState struct {
 	Repositories       []RepositorySpec `yaml:"repositories"`
 	Releases           []ReleaseSpec    `yaml:"releases"`
 
-	env environment.Environment
+	Env environment.Environment
 
 	logger *zap.SugaredLogger
 
@@ -876,7 +876,7 @@ func (state *HelmState) flagsForLint(helm helmexec.Interface, release *ReleaseSp
 }
 
 func (state *HelmState) RenderValuesFileToBytes(path string) ([]byte, error) {
-	r := valuesfile.NewRenderer(state.readFile, state.basePath, state.env)
+	r := valuesfile.NewRenderer(state.readFile, state.basePath, state.Env)
 	return r.RenderToBytes(path)
 }
 

tmpl/context.go

@@ -1,6 +1,7 @@
 package tmpl
 
 type Context struct {
-	basePath string
-	readFile func(string) ([]byte, error)
+	preRender bool
+	basePath  string
+	readFile  func(string) ([]byte, error)
 }

tmpl/file.go

@@ -2,6 +2,8 @@ package tmpl
 
 import (
 	"bytes"
+	"io/ioutil"
+
 	"github.com/roboll/helmfile/environment"
 )
 
@@ -33,11 +35,29 @@ func NewFileRenderer(readFile func(filename string) ([]byte, error), basePath st
 	}
 }
 
+func NewFirstPassRenderer(env environment.Environment) *templateFileRenderer {
+	return &templateFileRenderer{
+		ReadFile: ioutil.ReadFile,
+		Context: &Context{
+			preRender: true,
+			basePath:  "",
+			readFile:  ioutil.ReadFile,
+		},
+		Data: TemplateData{
+			Environment: env,
+		},
+	}
+}
+
 func (r *templateFileRenderer) RenderTemplateFileToBuffer(file string) (*bytes.Buffer, error) {
 	content, err := r.ReadFile(file)
 	if err != nil {
 		return nil, err
 	}
 
+	return r.RenderTemplateContentToBuffer(content)
+}
+
+func (r *templateFileRenderer) RenderTemplateContentToBuffer(content []byte) (*bytes.Buffer, error) {
 	return r.Context.RenderTemplateToBuffer(string(content), r.Data)
 }

tmpl/funcs.go

@@ -15,7 +15,7 @@ import (
 type Values = map[string]interface{}
 
 func (c *Context) createFuncMap() template.FuncMap {
-	return template.FuncMap{
+	funcMap := template.FuncMap{
 		"exec":           c.Exec,
 		"readFile":       c.ReadFile,
 		"toYaml":         ToYaml,
@@ -23,6 +23,17 @@ func (c *Context) createFuncMap() template.FuncMap {
 		"setValueAtPath": SetValueAtPath,
 		"requiredEnv":    RequiredEnv,
 	}
+	if c.preRender {
+		// disable potential side-effect template calls
+		funcMap["exec"] = func(string, []interface{}, ...string) (string, error) {
+			return "", nil
+		}
+		funcMap["readFile"] = func(string) (string, error) {
+			return "", nil
+		}
+	}
+
+	return funcMap
 }
 
 func (c *Context) Exec(command string, args []interface{}, inputs ...string) (string, error) {

tmpl/tmpl.go

@@ -11,7 +11,13 @@ func (c *Context) stringTemplate() *template.Template {
 	for name, f := range c.createFuncMap() {
 		funcMap[name] = f
 	}
-	return template.New("stringTemplate").Funcs(funcMap)
+	tmpl := template.New("stringTemplate").Funcs(funcMap)
+	if c.preRender {
+		tmpl.Option("missingkey=zero")
+	} else {
+		tmpl.Option("missingkey=error")
+	}
+	return tmpl
 }
 
 func (c *Context) RenderTemplateToBuffer(s string, data ...interface{}) (*bytes.Buffer, error) {
@@ -28,7 +34,7 @@ func (c *Context) RenderTemplateToBuffer(s string, data ...interface{}) (*bytes.
 	var execErr = t.Execute(&tplString, d)
 
 	if execErr != nil {
-		return nil, execErr
+		return &tplString, execErr
 	}
 
 	return &tplString, nil