feat: Template functions to replace `set`s (#242)

Resolves #227

README.md

@@ -270,6 +270,96 @@ The `selector` parameter can be specified multiple times. Each parameter is reso
 
 `--selector tier=frontend --selector tier=backend` will select all the charts
 
+## Templates
+
+You can use go's text/template expressions in `helmfile.yaml` and `values.yaml`(helm values files).
+
+In addition to built-in ones, the following custom template functions are available:
+
+- `readFile` reads the specified local file and generate a golang string
+- `fromYaml` reads a golang string and generates a map
+- `setValuleAtPath PATH NEW_VALUE` traverses a golang map, replaces the value at the PATH with NEW_VALUE
+- `toYaml` marshals a map into a string
+
+### Values Files Templates
+
+You can reference a template of values file in your `helmfile.yaml` like below:
+
+```yaml
+releases
+- name: myapp
+  chart: mychart
+  values:
+  - values.yaml.tpl
+```
+
+whereas `values.yaml.tpl` would be something like:
+
+```yaml
+{{ readFile "values.yaml" | fromYaml | setValueAtPath "foo.bar" "FOO_BAR" | toYaml }}
+```
+
+Suppose `values.yaml` was:
+
+```yaml
+foo:
+  bar: ""
+```
+
+The resulting, temporary values.yaml that is generated from `values.yaml.tpl` would become:
+
+```yaml
+foo:
+  # Notice `setValueAtPath "foo.bar" "FOO_BAR"` in the template above
+  bar: FOO_BAR
+```
+
+## Refactoring `helmfile.yaml` with values files templates
+
+One of expected use-cases of values files templates is to keep `helmfile.yaml` small and concise.
+
+See the example `helmfile.yaml` below:
+
+```yaml
+releases:
+  - name: {{ requiredEnv "NAME" }}-vault
+    namespace: {{ requiredEnv "NAME" }}
+    chart: roboll/vault-secret-manager
+    values:
+      - db:
+          username: {{ requiredEnv "DB_USERNAME" }}
+          password: {{ requiredEnv "DB_PASSWORD" }}
+    set:
+      - name: proxy.domain
+        value: {{ requiredEnv "PLATFORM_ID" }}.my-domain.com
+      - name: proxy.scheme
+        value: {{ env "SCHEME" | default "https" }}
+```
+
+The `values` and `set` sections of the config file can be separated out into a template:
+
+`helmfile.yaml`:
+
+```yaml
+releases:
+  - name: {{ requiredEnv "NAME" }}-vault
+    namespace: {{ requiredEnv "NAME" }}
+    chart: roboll/vault-secret-manager
+    values:
+    - values.yaml.tmpl
+```
+
+`values.yaml.tmpl`:
+
+```yaml
+db:
+  username: {{ requiredEnv "DB_USERNAME" }}
+  password: {{ requiredEnv "DB_PASSWORD" }}
+proxy:
+  domain: {{ requiredEnv "PLATFORM_ID" }}.my-domain.com
+  scheme: {{ env "SCHEME" | default "https" }}
+```
+
 ## Using env files
 
 helmfile itself doesn't have an ability to load env files. But you can write some bash script to achieve the goal:

state/state.go

@@ -3,6 +3,7 @@ package state
 import (
 	"errors"
 	"fmt"
+	"github.com/roboll/helmfile/helmexec"
 	"io/ioutil"
 	"os"
 	"path"
@@ -10,17 +11,13 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"text/template"
-
-	"github.com/Masterminds/sprig"
-
-	"github.com/roboll/helmfile/helmexec"
 
 	"bytes"
 	"regexp"
 
+	"github.com/roboll/helmfile/tmpl"
 	"go.uber.org/zap"
-	yaml "gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v2"
 )
 
 // HelmState structure for the helmfile
@@ -133,24 +130,6 @@ func CreateFromYaml(content []byte, file string, logger *zap.SugaredLogger) (*He
 	return &state, nil
 }
 
-func stringTemplate() *template.Template {
-	funcMap := sprig.TxtFuncMap()
-	alterFuncMap(&funcMap)
-	return template.New("stringTemplate").Funcs(funcMap)
-}
-
-func alterFuncMap(funcMap *template.FuncMap) {
-	(*funcMap)["requiredEnv"] = getRequiredEnv
-}
-
-func getRequiredEnv(name string) (string, error) {
-	if val, exists := os.LookupEnv(name); exists && len(val) > 0 {
-		return val, nil
-	}
-
-	return "", fmt.Errorf("required env var `%s` is not set", name)
-}
-
 func RenderTemplateFileToBuffer(file string) (*bytes.Buffer, error) {
 	content, err := ioutil.ReadFile(file)
 	if err != nil {
@@ -161,27 +140,7 @@ func RenderTemplateFileToBuffer(file string) (*bytes.Buffer, error) {
 }
 
 func renderTemplateToBuffer(s string) (*bytes.Buffer, error) {
-	var t, parseErr = stringTemplate().Parse(s)
+	return tmpl.DefaultContext.RenderTemplateToBuffer(s)
-	if parseErr != nil {
-		return nil, parseErr
-	}
-
-	var tplString bytes.Buffer
-	var execErr = t.Execute(&tplString, nil)
-
-	if execErr != nil {
-		return nil, execErr
-	}
-
-	return &tplString, nil
-}
-
-func renderTemplateString(s string) (string, error) {
-	tplString, err := renderTemplateToBuffer(s)
-	if err != nil {
-		return "", err
-	}
-	return tplString.String(), nil
 }
 
 func (state *HelmState) applyDefaultsTo(spec *ReleaseSpec) {

state/state_test.go

@@ -12,6 +12,14 @@ import (
 
 var logger = helmexec.NewLogger(os.Stdout, "warn")
 
+func renderTemplateToString(s string) (string, error) {
+	tplString, err := renderTemplateToBuffer(s)
+	if err != nil {
+		return "", err
+	}
+	return tplString.String(), nil
+}
+
 func TestReadFromYaml(t *testing.T) {
 	yamlFile := "example/path/to/yaml/file"
 	yamlContent := []byte(`releases:
@@ -512,7 +520,7 @@ func TestHelmState_flagsForUpgrade(t *testing.T) {
 	}
 }
 
-func Test_renderTemplateString(t *testing.T) {
+func Test_renderTemplateToString(t *testing.T) {
 	type args struct {
 		s    string
 		envs map[string]string
@@ -613,16 +621,16 @@ func Test_renderTemplateString(t *testing.T) {
 			for k, v := range tt.args.envs {
 				err := os.Setenv(k, v)
 				if err != nil {
-					t.Error("renderTemplateString() could not set env var for testing")
+					t.Error("renderTemplateToString() could not set env var for testing")
 				}
 			}
-			got, err := renderTemplateString(tt.args.s)
+			got, err := renderTemplateToString(tt.args.s)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("renderTemplateString() for %s error = %v, wantErr %v", tt.name, err, tt.wantErr)
+				t.Errorf("renderTemplateToString() for %s error = %v, wantErr %v", tt.name, err, tt.wantErr)
 				return
 			}
 			if got != tt.want {
-				t.Errorf("renderTemplateString() for %s = %v, want %v", tt.name, got, tt.want)
+				t.Errorf("renderTemplateToString() for %s = %v, want %v", tt.name, got, tt.want)
 			}
 		})
 	}

tmpl/context.go

@@ -0,0 +1,15 @@
+package tmpl
+
+import "io/ioutil"
+
+var DefaultContext *Context
+
+func init() {
+	DefaultContext = &Context{
+		readFile: ioutil.ReadFile,
+	}
+}
+
+type Context struct {
+	readFile func(string) ([]byte, error)
+}

tmpl/funcs.go

@@ -0,0 +1,99 @@
+package tmpl
+
+import (
+	"fmt"
+	"gopkg.in/yaml.v2"
+	"os"
+	"strings"
+	"text/template"
+)
+
+type Values = map[string]interface{}
+
+func (c *Context) createFuncMap() template.FuncMap {
+	return template.FuncMap{
+		"readFile":       c.ReadFile,
+		"toYaml":         ToYaml,
+		"fromYaml":       FromYaml,
+		"setValueAtPath": SetValueAtPath,
+		"requiredEnv":    RequiredEnv,
+	}
+}
+
+func (c *Context) ReadFile(filename string) (string, error) {
+	bytes, err := c.readFile(filename)
+	if err != nil {
+		return "", err
+	}
+	return string(bytes), nil
+}
+
+func ToYaml(v interface{}) (string, error) {
+	data, err := yaml.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+func FromYaml(str string) (Values, error) {
+	m := Values{}
+
+	if err := yaml.Unmarshal([]byte(str), &m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func SetValueAtPath(path string, value interface{}, values Values) (Values, error) {
+	var current interface{}
+	current = values
+	components := strings.Split(path, ".")
+	pathToMap := components[:len(components)-1]
+	key := components[len(components)-1]
+	for _, k := range pathToMap {
+		var elem interface{}
+
+		switch typedCurrent := current.(type) {
+		case map[string]interface{}:
+			v, exists := typedCurrent[k]
+			if !exists {
+				return nil, fmt.Errorf("failed to set value at path \"%s\": value for key \"%s\" does not exist", path, k)
+			}
+			elem = v
+		case map[interface{}]interface{}:
+			v, exists := typedCurrent[k]
+			if !exists {
+				return nil, fmt.Errorf("failed to set value at path \"%s\": value for key \"%s\" does not exist", path, k)
+			}
+			elem = v
+		default:
+			return nil, fmt.Errorf("failed to set value at path \"%s\": value for key \"%s\" was not a map", path, k)
+		}
+
+		switch typedElem := elem.(type) {
+		case map[string]interface{}, map[interface{}]interface{}:
+			current = typedElem
+		default:
+			return nil, fmt.Errorf("failed to set value at path \"%s\": value for key \"%s\" was not a map", path, k)
+		}
+	}
+
+	switch typedCurrent := current.(type) {
+	case map[string]interface{}:
+		typedCurrent[key] = value
+	case map[interface{}]interface{}:
+		typedCurrent[key] = value
+	default:
+		return nil, fmt.Errorf("failed to set value at path \"%s\": value for key \"%s\" was not a map", path, key)
+	}
+	return values, nil
+}
+
+func RequiredEnv(name string) (string, error) {
+	if val, exists := os.LookupEnv(name); exists && len(val) > 0 {
+		return val, nil
+	}
+
+	return "", fmt.Errorf("required env var `%s` is not set", name)
+}

tmpl/funcs_test.go

@@ -0,0 +1,99 @@
+package tmpl
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestReadFile(t *testing.T) {
+	expected := `foo:
+  bar: BAR
+`
+	expectedFilename := "values.yaml"
+	ctx := &Context{readFile: func(filename string) ([]byte, error) {
+		if filename != expectedFilename {
+			return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", expectedFilename, filename)
+		}
+		return []byte(expected), nil
+	}}
+	actual, err := ctx.ReadFile(expectedFilename)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual)
+	}
+}
+
+func TestToYaml(t *testing.T) {
+	expected := `foo:
+  bar: BAR
+`
+	vals := Values(map[string]interface{}{
+		"foo": map[interface{}]interface{}{
+			"bar": "BAR",
+		},
+	})
+	actual, err := ToYaml(vals)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual)
+	}
+}
+
+func TestFromYaml(t *testing.T) {
+	raw := `foo:
+  bar: BAR
+`
+	expected := Values(map[string]interface{}{
+		"foo": map[interface{}]interface{}{
+			"bar": "BAR",
+		},
+	})
+	actual, err := FromYaml(raw)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual)
+	}
+}
+
+func TestSetValueAtPath_OneComponent(t *testing.T) {
+	input := map[string]interface{}{
+		"foo": "",
+	}
+	expected := map[string]interface{}{
+		"foo": "FOO",
+	}
+	actual, err := SetValueAtPath("foo", "FOO", input)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual)
+	}
+}
+
+func TestSetValueAtPath_TwoComponents(t *testing.T) {
+	input := map[string]interface{}{
+		"foo": map[interface{}]interface{}{
+			"bar": "",
+		},
+	}
+	expected := map[string]interface{}{
+		"foo": map[interface{}]interface{}{
+			"bar": "FOO_BAR",
+		},
+	}
+	actual, err := SetValueAtPath("foo.bar", "FOO_BAR", input)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual)
+	}
+}

tmpl/tmpl.go

@@ -0,0 +1,31 @@
+package tmpl
+
+import (
+	"bytes"
+	"github.com/Masterminds/sprig"
+	"text/template"
+)
+
+func (c *Context) stringTemplate() *template.Template {
+	funcMap := sprig.TxtFuncMap()
+	for name, f := range c.createFuncMap() {
+		funcMap[name] = f
+	}
+	return template.New("stringTemplate").Funcs(funcMap)
+}
+
+func (c *Context) RenderTemplateToBuffer(s string) (*bytes.Buffer, error) {
+	var t, parseErr = c.stringTemplate().Parse(s)
+	if parseErr != nil {
+		return nil, parseErr
+	}
+
+	var tplString bytes.Buffer
+	var execErr = t.Execute(&tplString, nil)
+
+	if execErr != nil {
+		return nil, execErr
+	}
+
+	return &tplString, nil
+}

tmpl/tmpl_test.go

@@ -0,0 +1,31 @@
+package tmpl
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestRenderTemplate(t *testing.T) {
+	valuesYamlContent := `foo:
+  bar: BAR
+`
+	expected := `foo:
+  bar: FOO_BAR
+`
+	expectedFilename := "values.yaml"
+	ctx := &Context{readFile: func(filename string) ([]byte, error) {
+		if filename != expectedFilename {
+			return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", expectedFilename, filename)
+		}
+		return []byte(valuesYamlContent), nil
+	}}
+	buf, err := ctx.RenderTemplateToBuffer(`{{ readFile "values.yaml" | fromYaml | setValueAtPath "foo.bar" "FOO_BAR" | toYaml }}`)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	actual := buf.String()
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual)
+	}
+}