diff --git a/README.md b/README.md index 64a2c7eb..01071945 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,96 @@ The `selector` parameter can be specified multiple times. Each parameter is reso `--selector tier=frontend --selector tier=backend` will select all the charts +## Templates + +You can use go's text/template expressions in `helmfile.yaml` and `values.yaml`(helm values files). + +In addition to built-in ones, the following custom template functions are available: + +- `readFile` reads the specified local file and generate a golang string +- `fromYaml` reads a golang string and generates a map +- `setValuleAtPath PATH NEW_VALUE` traverses a golang map, replaces the value at the PATH with NEW_VALUE +- `toYaml` marshals a map into a string + +### Values Files Templates + +You can reference a template of values file in your `helmfile.yaml` like below: + +```yaml +releases +- name: myapp + chart: mychart + values: + - values.yaml.tpl +``` + +whereas `values.yaml.tpl` would be something like: + +```yaml +{{ readFile "values.yaml" | fromYaml | setValueAtPath "foo.bar" "FOO_BAR" | toYaml }} +``` + +Suppose `values.yaml` was: + +```yaml +foo: + bar: "" +``` + +The resulting, temporary values.yaml that is generated from `values.yaml.tpl` would become: + +```yaml +foo: + # Notice `setValueAtPath "foo.bar" "FOO_BAR"` in the template above + bar: FOO_BAR +``` + +## Refactoring `helmfile.yaml` with values files templates + +One of expected use-cases of values files templates is to keep `helmfile.yaml` small and concise. + +See the example `helmfile.yaml` below: + +```yaml +releases: + - name: {{ requiredEnv "NAME" }}-vault + namespace: {{ requiredEnv "NAME" }} + chart: roboll/vault-secret-manager + values: + - db: + username: {{ requiredEnv "DB_USERNAME" }} + password: {{ requiredEnv "DB_PASSWORD" }} + set: + - name: proxy.domain + value: {{ requiredEnv "PLATFORM_ID" }}.my-domain.com + - name: proxy.scheme + value: {{ env "SCHEME" | default "https" }} +``` + +The `values` and `set` sections of the config file can be separated out into a template: + +`helmfile.yaml`: + +```yaml +releases: + - name: {{ requiredEnv "NAME" }}-vault + namespace: {{ requiredEnv "NAME" }} + chart: roboll/vault-secret-manager + values: + - values.yaml.tmpl +``` + +`values.yaml.tmpl`: + +```yaml +db: + username: {{ requiredEnv "DB_USERNAME" }} + password: {{ requiredEnv "DB_PASSWORD" }} +proxy: + domain: {{ requiredEnv "PLATFORM_ID" }}.my-domain.com + scheme: {{ env "SCHEME" | default "https" }} +``` + ## Using env files helmfile itself doesn't have an ability to load env files. But you can write some bash script to achieve the goal: diff --git a/state/state.go b/state/state.go index 9b5c0117..aee8578e 100644 --- a/state/state.go +++ b/state/state.go @@ -3,6 +3,7 @@ package state import ( "errors" "fmt" + "github.com/roboll/helmfile/helmexec" "io/ioutil" "os" "path" @@ -10,17 +11,13 @@ import ( "strconv" "strings" "sync" - "text/template" - - "github.com/Masterminds/sprig" - - "github.com/roboll/helmfile/helmexec" "bytes" "regexp" + "github.com/roboll/helmfile/tmpl" "go.uber.org/zap" - yaml "gopkg.in/yaml.v2" + "gopkg.in/yaml.v2" ) // HelmState structure for the helmfile @@ -133,24 +130,6 @@ func CreateFromYaml(content []byte, file string, logger *zap.SugaredLogger) (*He return &state, nil } -func stringTemplate() *template.Template { - funcMap := sprig.TxtFuncMap() - alterFuncMap(&funcMap) - return template.New("stringTemplate").Funcs(funcMap) -} - -func alterFuncMap(funcMap *template.FuncMap) { - (*funcMap)["requiredEnv"] = getRequiredEnv -} - -func getRequiredEnv(name string) (string, error) { - if val, exists := os.LookupEnv(name); exists && len(val) > 0 { - return val, nil - } - - return "", fmt.Errorf("required env var `%s` is not set", name) -} - func RenderTemplateFileToBuffer(file string) (*bytes.Buffer, error) { content, err := ioutil.ReadFile(file) if err != nil { @@ -161,27 +140,7 @@ func RenderTemplateFileToBuffer(file string) (*bytes.Buffer, error) { } func renderTemplateToBuffer(s string) (*bytes.Buffer, error) { - var t, parseErr = stringTemplate().Parse(s) - if parseErr != nil { - return nil, parseErr - } - - var tplString bytes.Buffer - var execErr = t.Execute(&tplString, nil) - - if execErr != nil { - return nil, execErr - } - - return &tplString, nil -} - -func renderTemplateString(s string) (string, error) { - tplString, err := renderTemplateToBuffer(s) - if err != nil { - return "", err - } - return tplString.String(), nil + return tmpl.DefaultContext.RenderTemplateToBuffer(s) } func (state *HelmState) applyDefaultsTo(spec *ReleaseSpec) { diff --git a/state/state_test.go b/state/state_test.go index bef5c3d4..4193da80 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -12,6 +12,14 @@ import ( var logger = helmexec.NewLogger(os.Stdout, "warn") +func renderTemplateToString(s string) (string, error) { + tplString, err := renderTemplateToBuffer(s) + if err != nil { + return "", err + } + return tplString.String(), nil +} + func TestReadFromYaml(t *testing.T) { yamlFile := "example/path/to/yaml/file" yamlContent := []byte(`releases: @@ -512,7 +520,7 @@ func TestHelmState_flagsForUpgrade(t *testing.T) { } } -func Test_renderTemplateString(t *testing.T) { +func Test_renderTemplateToString(t *testing.T) { type args struct { s string envs map[string]string @@ -613,16 +621,16 @@ func Test_renderTemplateString(t *testing.T) { for k, v := range tt.args.envs { err := os.Setenv(k, v) if err != nil { - t.Error("renderTemplateString() could not set env var for testing") + t.Error("renderTemplateToString() could not set env var for testing") } } - got, err := renderTemplateString(tt.args.s) + got, err := renderTemplateToString(tt.args.s) if (err != nil) != tt.wantErr { - t.Errorf("renderTemplateString() for %s error = %v, wantErr %v", tt.name, err, tt.wantErr) + t.Errorf("renderTemplateToString() for %s error = %v, wantErr %v", tt.name, err, tt.wantErr) return } if got != tt.want { - t.Errorf("renderTemplateString() for %s = %v, want %v", tt.name, got, tt.want) + t.Errorf("renderTemplateToString() for %s = %v, want %v", tt.name, got, tt.want) } }) } diff --git a/tmpl/context.go b/tmpl/context.go new file mode 100644 index 00000000..77900a1c --- /dev/null +++ b/tmpl/context.go @@ -0,0 +1,15 @@ +package tmpl + +import "io/ioutil" + +var DefaultContext *Context + +func init() { + DefaultContext = &Context{ + readFile: ioutil.ReadFile, + } +} + +type Context struct { + readFile func(string) ([]byte, error) +} diff --git a/tmpl/funcs.go b/tmpl/funcs.go new file mode 100644 index 00000000..ed4058a0 --- /dev/null +++ b/tmpl/funcs.go @@ -0,0 +1,99 @@ +package tmpl + +import ( + "fmt" + "gopkg.in/yaml.v2" + "os" + "strings" + "text/template" +) + +type Values = map[string]interface{} + +func (c *Context) createFuncMap() template.FuncMap { + return template.FuncMap{ + "readFile": c.ReadFile, + "toYaml": ToYaml, + "fromYaml": FromYaml, + "setValueAtPath": SetValueAtPath, + "requiredEnv": RequiredEnv, + } +} + +func (c *Context) ReadFile(filename string) (string, error) { + bytes, err := c.readFile(filename) + if err != nil { + return "", err + } + return string(bytes), nil +} + +func ToYaml(v interface{}) (string, error) { + data, err := yaml.Marshal(v) + if err != nil { + return "", err + } + return string(data), nil +} + +func FromYaml(str string) (Values, error) { + m := Values{} + + if err := yaml.Unmarshal([]byte(str), &m); err != nil { + return nil, err + } + return m, nil +} + +func SetValueAtPath(path string, value interface{}, values Values) (Values, error) { + var current interface{} + current = values + components := strings.Split(path, ".") + pathToMap := components[:len(components)-1] + key := components[len(components)-1] + for _, k := range pathToMap { + var elem interface{} + + switch typedCurrent := current.(type) { + case map[string]interface{}: + v, exists := typedCurrent[k] + if !exists { + return nil, fmt.Errorf("failed to set value at path \"%s\": value for key \"%s\" does not exist", path, k) + } + elem = v + case map[interface{}]interface{}: + v, exists := typedCurrent[k] + if !exists { + return nil, fmt.Errorf("failed to set value at path \"%s\": value for key \"%s\" does not exist", path, k) + } + elem = v + default: + return nil, fmt.Errorf("failed to set value at path \"%s\": value for key \"%s\" was not a map", path, k) + } + + switch typedElem := elem.(type) { + case map[string]interface{}, map[interface{}]interface{}: + current = typedElem + default: + return nil, fmt.Errorf("failed to set value at path \"%s\": value for key \"%s\" was not a map", path, k) + } + } + + switch typedCurrent := current.(type) { + case map[string]interface{}: + typedCurrent[key] = value + case map[interface{}]interface{}: + typedCurrent[key] = value + default: + return nil, fmt.Errorf("failed to set value at path \"%s\": value for key \"%s\" was not a map", path, key) + } + return values, nil +} + +func RequiredEnv(name string) (string, error) { + if val, exists := os.LookupEnv(name); exists && len(val) > 0 { + return val, nil + } + + return "", fmt.Errorf("required env var `%s` is not set", name) +} diff --git a/tmpl/funcs_test.go b/tmpl/funcs_test.go new file mode 100644 index 00000000..20b2abfc --- /dev/null +++ b/tmpl/funcs_test.go @@ -0,0 +1,99 @@ +package tmpl + +import ( + "fmt" + "reflect" + "testing" +) + +func TestReadFile(t *testing.T) { + expected := `foo: + bar: BAR +` + expectedFilename := "values.yaml" + ctx := &Context{readFile: func(filename string) ([]byte, error) { + if filename != expectedFilename { + return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", expectedFilename, filename) + } + return []byte(expected), nil + }} + actual, err := ctx.ReadFile(expectedFilename) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !reflect.DeepEqual(actual, expected) { + t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual) + } +} + +func TestToYaml(t *testing.T) { + expected := `foo: + bar: BAR +` + vals := Values(map[string]interface{}{ + "foo": map[interface{}]interface{}{ + "bar": "BAR", + }, + }) + actual, err := ToYaml(vals) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !reflect.DeepEqual(actual, expected) { + t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual) + } +} + +func TestFromYaml(t *testing.T) { + raw := `foo: + bar: BAR +` + expected := Values(map[string]interface{}{ + "foo": map[interface{}]interface{}{ + "bar": "BAR", + }, + }) + actual, err := FromYaml(raw) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !reflect.DeepEqual(actual, expected) { + t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual) + } +} + +func TestSetValueAtPath_OneComponent(t *testing.T) { + input := map[string]interface{}{ + "foo": "", + } + expected := map[string]interface{}{ + "foo": "FOO", + } + actual, err := SetValueAtPath("foo", "FOO", input) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !reflect.DeepEqual(actual, expected) { + t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual) + } +} + +func TestSetValueAtPath_TwoComponents(t *testing.T) { + input := map[string]interface{}{ + "foo": map[interface{}]interface{}{ + "bar": "", + }, + } + expected := map[string]interface{}{ + "foo": map[interface{}]interface{}{ + "bar": "FOO_BAR", + }, + } + actual, err := SetValueAtPath("foo.bar", "FOO_BAR", input) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !reflect.DeepEqual(actual, expected) { + t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual) + } +} diff --git a/tmpl/tmpl.go b/tmpl/tmpl.go new file mode 100644 index 00000000..90a8edd6 --- /dev/null +++ b/tmpl/tmpl.go @@ -0,0 +1,31 @@ +package tmpl + +import ( + "bytes" + "github.com/Masterminds/sprig" + "text/template" +) + +func (c *Context) stringTemplate() *template.Template { + funcMap := sprig.TxtFuncMap() + for name, f := range c.createFuncMap() { + funcMap[name] = f + } + return template.New("stringTemplate").Funcs(funcMap) +} + +func (c *Context) RenderTemplateToBuffer(s string) (*bytes.Buffer, error) { + var t, parseErr = c.stringTemplate().Parse(s) + if parseErr != nil { + return nil, parseErr + } + + var tplString bytes.Buffer + var execErr = t.Execute(&tplString, nil) + + if execErr != nil { + return nil, execErr + } + + return &tplString, nil +} diff --git a/tmpl/tmpl_test.go b/tmpl/tmpl_test.go new file mode 100644 index 00000000..75687bfc --- /dev/null +++ b/tmpl/tmpl_test.go @@ -0,0 +1,31 @@ +package tmpl + +import ( + "fmt" + "reflect" + "testing" +) + +func TestRenderTemplate(t *testing.T) { + valuesYamlContent := `foo: + bar: BAR +` + expected := `foo: + bar: FOO_BAR +` + expectedFilename := "values.yaml" + ctx := &Context{readFile: func(filename string) ([]byte, error) { + if filename != expectedFilename { + return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", expectedFilename, filename) + } + return []byte(valuesYamlContent), nil + }} + buf, err := ctx.RenderTemplateToBuffer(`{{ readFile "values.yaml" | fromYaml | setValueAtPath "foo.bar" "FOO_BAR" | toYaml }}`) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + actual := buf.String() + if !reflect.DeepEqual(actual, expected) { + t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual) + } +}