feat: include func support (#1187)

* feat: include func support

Signed-off-by: yxxhero <aiopsclub@163.com>
This commit is contained in:
yxxhero 2023-12-04 21:51:01 +08:00 committed by GitHub
parent 06504477f6
commit 7d6ed97333
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 380 additions and 53 deletions

View File

@ -133,3 +133,11 @@ The `expandSecretRefs` function takes an object as the argument and expands ever
```yaml
{{ $expandSecretRefs := $value | expandSecretRefs }}
```
#### `include`
The 'include' function allows including and rendering nested templates. The function returns the created template or an error if any occurred. It will load functions from `_*.tpl` files in the directory where the helmfile.yaml is located.
For nested helmfile.yaml files, it will load `_*.tpl` files in the directory where each nested helmfile.yaml is located. example: [include](https://github.com/helmfile/helmfile/tree/main/test/integration/test-cases/include-template-funch/input)
```yaml
{{ include "my-template" . }}
```

2
go.mod
View File

@ -89,7 +89,7 @@ require (
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/otiai10/copy v1.1.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/errors v0.9.1
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect

View File

@ -141,6 +141,9 @@ func TestTrigger(t *testing.T) {
readFile := func(filename string) ([]byte, error) {
return nil, fmt.Errorf("unexpected call to readFile: %s", filename)
}
glob := func(pattern string) ([]string, error) {
return nil, nil
}
for _, c := range cases {
hooks := []Hook{}
if c.hook != nil {
@ -155,7 +158,7 @@ func TestTrigger(t *testing.T) {
Namespace: "myns",
Env: environment.Environment{Name: "prod"},
Logger: zeLogger,
Fs: &ffs.FileSystem{ReadFile: readFile},
Fs: &ffs.FileSystem{ReadFile: readFile, Glob: glob},
}
bus.Runner = &runner{}

View File

@ -30,6 +30,7 @@ type FileSystem struct {
Glob func(string) ([]string, error)
FileExistsAt func(string) bool
DirectoryExistsAt func(string) bool
Dir func(string) string
Stat func(string) (os.FileInfo, error)
Getwd func() (string, error)
Chdir func(string) error
@ -41,11 +42,11 @@ func DefaultFileSystem() *FileSystem {
dfs := FileSystem{
ReadDir: os.ReadDir,
DeleteFile: os.Remove,
Stat: os.Stat,
Glob: filepath.Glob,
Getwd: os.Getwd,
Chdir: os.Chdir,
EvalSymlinks: filepath.EvalSymlinks,
Dir: filepath.Dir,
}
dfs.Stat = dfs.stat
@ -96,6 +97,9 @@ func FromFileSystem(params FileSystem) *FileSystem {
if params.EvalSymlinks != nil {
dfs.EvalSymlinks = params.EvalSymlinks
}
if params.Dir != nil {
dfs.Dir = params.Dir
}
return dfs
}

View File

@ -9,6 +9,7 @@ import (
"github.com/go-test/deep"
"github.com/helmfile/helmfile/pkg/environment"
"github.com/helmfile/helmfile/pkg/filesystem"
)
func boolPtrToString(ptr *bool) string {
@ -128,6 +129,8 @@ func TestHelmState_executeTemplates(t *testing.T) {
tt := tests[i]
t.Run(tt.name, func(t *testing.T) {
state := &HelmState{
fs: &filesystem.FileSystem{
Glob: func(s string) ([]string, error) { return nil, nil }},
basePath: ".",
ReleaseSetSpec: ReleaseSetSpec{
HelmDefaults: HelmSpec{
@ -226,6 +229,9 @@ func TestHelmState_recursiveRefsTemplates(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
state := &HelmState{
basePath: ".",
fs: &filesystem.FileSystem{
Glob: func(s string) ([]string, error) { return nil, nil },
},
ReleaseSetSpec: ReleaseSetSpec{
HelmDefaults: HelmSpec{
KubeContext: "test_context",

View File

@ -307,7 +307,13 @@ func TestSetValueAtPath_TwoComponents(t *testing.T) {
}
func TestTpl(t *testing.T) {
ctx := &Context{basePath: "."}
ctx := &Context{
basePath: ".",
fs: &filesystem.FileSystem{
Glob: func(s string) ([]string, error) {
return nil, nil
}},
}
tests := []struct {
name string
input string

View File

@ -2,28 +2,45 @@ package tmpl
import (
"bytes"
"fmt"
"path/filepath"
"strings"
"text/template"
"github.com/Masterminds/sprig/v3"
"github.com/pkg/errors"
)
const recursionMaxNums = 1000
// CreateFuncMap creates a template.FuncMap for the Context struct.
// It combines the functions from sprig.TxtFuncMap() with the functions
// defined in the Context's createFuncMap() method.
// It also adds aliases for certain functions based on the aliases map.
// The resulting FuncMap is returned.
func (c *Context) CreateFuncMap() template.FuncMap {
// function aliases
aliased := template.FuncMap{}
// map of function aliases
aliases := map[string]string{
"get": "sprigGet",
}
// get the default sprig functions
funcMap := sprig.TxtFuncMap()
// add aliases to the aliased FuncMap
for orig, alias := range aliases {
aliased[alias] = funcMap[orig]
}
// add functions from the Context's createFuncMap() method to the funcMap
for name, f := range c.createFuncMap() {
funcMap[name] = f
}
// add aliased functions to the funcMap
for name, f := range aliased {
funcMap[name] = f
}
@ -31,22 +48,90 @@ func (c *Context) CreateFuncMap() template.FuncMap {
return funcMap
}
func (c *Context) newTemplate() *template.Template {
type tplInfo struct {
name string
content string
}
// helperTPLs returns the contents of all files with names starting with "_" and ending with ".tpl"
// in the root directory of the Context. It reads each file and appends its content to the contents slice.
// If any error occurs during the file reading or globbing process, it returns an error.
func (c *Context) helperTPLs() ([]tplInfo, error) {
tplInfos := []tplInfo{}
files, err := c.fs.Glob(filepath.Join(c.basePath, "_*.tpl"))
if err != nil {
return nil, fmt.Errorf("failed to glob helper templates: %v", err)
}
for _, file := range files {
content, err := c.fs.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("failed to read helper template %s: %v", file, err)
}
tplInfos = append(tplInfos, tplInfo{name: file, content: string(content)})
}
return tplInfos, nil
}
// newTemplate creates a new template based on the context.
// It initializes the template with the specified options and parses the helper templates.
// It also adds the 'include' function to the template's function map.
// The 'include' function allows including and rendering nested templates.
// The function returns the created template or an error if any occurred.
func (c *Context) newTemplate() (*template.Template, error) {
funcMap := c.CreateFuncMap()
tmpl := template.New("stringTemplate").Funcs(funcMap)
tmpl := template.New("stringTemplate")
if c.preRender {
tmpl = tmpl.Option("missingkey=zero")
} else {
tmpl = tmpl.Option("missingkey=error")
}
return tmpl
tpls, err := c.helperTPLs()
if err != nil {
return nil, err
}
for _, tpl := range tpls {
tmpl, err = tmpl.Parse(tpl.content)
if err != nil {
return nil, fmt.Errorf("failed to parse helper template %s: %v", tpl.name, err)
}
}
includedNames := make(map[string]int)
// Add the 'include' function here so we can close over t.
funcMap["include"] = func(name string, data interface{}) (string, error) {
var buf strings.Builder
if v, ok := includedNames[name]; ok {
if v > recursionMaxNums {
return "", errors.Wrapf(fmt.Errorf("unable to execute template"), "rendering template has a nested reference name: %s", name)
}
includedNames[name]++
} else {
includedNames[name] = 1
}
err := tmpl.ExecuteTemplate(&buf, name, data)
includedNames[name]--
return buf.String(), err
}
tmpl.Funcs(funcMap)
return tmpl, nil
}
// RenderTemplateToBuffer renders the provided template string with the given data and returns the result as a *bytes.Buffer.
// The template string is parsed and executed using the Context's newTemplate method.
// If an error occurs during parsing or execution, it is returned along with the partially rendered template.
// The data parameter is optional and can be used to provide additional data for template rendering.
// If no data is provided, the template is rendered with an empty data context.
func (c *Context) RenderTemplateToBuffer(s string, data ...any) (*bytes.Buffer, error) {
var t, parseErr = c.newTemplate().Parse(s)
if parseErr != nil {
return nil, parseErr
t, err := c.newTemplate()
if err != nil {
return nil, err
}
t, err = t.Parse(s)
if err != nil {
return nil, err
}
var tplString bytes.Buffer

View File

@ -16,12 +16,16 @@ func TestRenderTemplate_Values(t *testing.T) {
bar: FOO_BAR
`
expectedFilename := "values.yaml"
ctx := &Context{fs: &ffs.FileSystem{ReadFile: func(filename string) ([]byte, error) {
if filename != expectedFilename {
return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", expectedFilename, filename)
}
return []byte(valuesYamlContent), nil
}}}
ctx := &Context{fs: &ffs.FileSystem{
Glob: func(s string) ([]string, error) {
return nil, nil
},
ReadFile: func(filename string) ([]byte, error) {
if filename != expectedFilename {
return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", expectedFilename, filename)
}
return []byte(valuesYamlContent), nil
}}}
buf, err := ctx.RenderTemplateToBuffer(`{{ readFile "values.yaml" | fromYaml | setValueAtPath "foo.bar" "FOO_BAR" | toYaml }}`)
if err != nil {
t.Errorf("unexpected error: %v", err)
@ -45,12 +49,16 @@ func TestRenderTemplate_WithData(t *testing.T) {
"bar": "FOO_BAR",
},
}
ctx := &Context{fs: &ffs.FileSystem{ReadFile: func(filename string) ([]byte, error) {
if filename != expectedFilename {
return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", expectedFilename, filename)
}
return []byte(valuesYamlContent), nil
}}}
ctx := &Context{fs: &ffs.FileSystem{
Glob: func(s string) ([]string, error) {
return nil, nil
},
ReadFile: func(filename string) ([]byte, error) {
if filename != expectedFilename {
return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", expectedFilename, filename)
}
return []byte(valuesYamlContent), nil
}}}
buf, err := ctx.RenderTemplateToBuffer(valuesYamlContent, data)
if err != nil {
t.Errorf("unexpected error: %v", err)
@ -70,12 +78,16 @@ func TestRenderTemplate_AccessingMissingKeyWithGetOrNil(t *testing.T) {
`
expectedFilename := "values.yaml"
data := map[string]any{}
ctx := &Context{fs: &ffs.FileSystem{ReadFile: func(filename string) ([]byte, error) {
if filename != expectedFilename {
return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", expectedFilename, filename)
}
return []byte(valuesYamlContent), nil
}}}
ctx := &Context{fs: &ffs.FileSystem{
Glob: func(s string) ([]string, error) {
return nil, nil
},
ReadFile: func(filename string) ([]byte, error) {
if filename != expectedFilename {
return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", expectedFilename, filename)
}
return []byte(valuesYamlContent), nil
}}}
buf, err := ctx.RenderTemplateToBuffer(valuesYamlContent, data)
if err != nil {
t.Errorf("unexpected error: %v", err)
@ -95,12 +107,16 @@ func TestRenderTemplate_Defaulting(t *testing.T) {
`
expectedFilename := "values.yaml"
data := map[string]any{}
ctx := &Context{fs: &ffs.FileSystem{ReadFile: func(filename string) ([]byte, error) {
if filename != expectedFilename {
return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", expectedFilename, filename)
}
return []byte(valuesYamlContent), nil
}}}
ctx := &Context{fs: &ffs.FileSystem{
Glob: func(s string) ([]string, error) {
return nil, nil
},
ReadFile: func(filename string) ([]byte, error) {
if filename != expectedFilename {
return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", expectedFilename, filename)
}
return []byte(valuesYamlContent), nil
}}}
buf, err := ctx.RenderTemplateToBuffer(valuesYamlContent, data)
if err != nil {
t.Errorf("unexpected error: %v", err)
@ -112,9 +128,13 @@ func TestRenderTemplate_Defaulting(t *testing.T) {
}
func renderTemplateToString(s string, data ...any) (string, error) {
ctx := &Context{fs: &ffs.FileSystem{ReadFile: func(filename string) ([]byte, error) {
return nil, fmt.Errorf("unexpected call to readFile: filename=%s", filename)
}}}
ctx := &Context{fs: &ffs.FileSystem{
Glob: func(s string) ([]string, error) {
return nil, nil
},
ReadFile: func(filename string) ([]byte, error) {
return nil, fmt.Errorf("unexpected call to readFile: filename=%s", filename)
}}}
tplString, err := ctx.RenderTemplateToBuffer(s, data...)
if err != nil {
return "", err
@ -326,3 +346,76 @@ func TestRenderTemplate_Required(t *testing.T) {
}
}
}
func TestContext_helperTPLs(t *testing.T) {
c := &Context{
fs: &ffs.FileSystem{
Glob: func(s string) ([]string, error) {
return []string{
"/helmfiletmpl/_template1.tpl",
"/helmfiletmpl/_template2.tpl",
}, nil
},
ReadFile: func(filename string) ([]byte, error) {
switch filename {
case "/helmfiletmpl/_template1.tpl":
return []byte("Template 1 content"), nil
case "/helmfiletmpl/_template2.tpl":
return []byte("Template 2 content"), nil
default:
return nil, fmt.Errorf("unexpected filename: %s", filename)
}
},
},
}
want := []tplInfo{
{
name: "/helmfiletmpl/_template1.tpl",
content: "Template 1 content",
},
{
name: "/helmfiletmpl/_template2.tpl",
content: "Template 2 content",
},
}
got, err := c.helperTPLs()
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if !reflect.DeepEqual(got, want) {
t.Errorf("unexpected result: got=%v, want=%v", got, want)
}
}
func TestContext_RenderTemplateToBuffer(t *testing.T) {
c := &Context{
basePath: "/helmfile",
fs: &ffs.FileSystem{
Glob: func(s string) ([]string, error) {
return []string{
"/helmfile/_template1.tpl",
}, nil
},
ReadFile: func(filename string) ([]byte, error) {
if filename == "/helmfile/_template1.tpl" {
return []byte("{{- define \"name\" -}}\n{{ .Name }}\n{{- end }}"), nil
}
return nil, fmt.Errorf("unexpected filename: %s", filename)
},
},
}
s := "Hello, {{ include \"name\" . }}!"
data := map[string]interface{}{
"Name": "Alice",
}
expected := "Hello, Alice!"
buf, err := c.RenderTemplateToBuffer(s, data)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
actual := buf.String()
if actual != expected {
t.Errorf("unexpected result: expected=%s, actual=%s", expected, actual)
}
}

View File

@ -24,15 +24,19 @@ func TestRenderToBytes_Gotmpl(t *testing.T) {
`
dataFile := "data.txt"
valuesTmplFile := "values.yaml.gotmpl"
r := NewFileRenderer(&filesystem.FileSystem{ReadFile: func(filename string) ([]byte, error) {
switch filename {
case valuesTmplFile:
return []byte(valuesYamlTmplContent), nil
case dataFile:
return []byte(dataFileContent), nil
}
return nil, fmt.Errorf("unexpected filename: expected=%v or %v, actual=%s", dataFile, valuesTmplFile, filename)
}}, "", emptyEnvTmplData)
r := NewFileRenderer(&filesystem.FileSystem{
Glob: func(pattern string) ([]string, error) {
return nil, nil
},
ReadFile: func(filename string) ([]byte, error) {
switch filename {
case valuesTmplFile:
return []byte(valuesYamlTmplContent), nil
case dataFile:
return []byte(dataFileContent), nil
}
return nil, fmt.Errorf("unexpected filename: expected=%v or %v, actual=%s", dataFile, valuesTmplFile, filename)
}}, "", emptyEnvTmplData)
buf, err := r.RenderToBytes(valuesTmplFile)
if err != nil {
t.Errorf("unexpected error: %v", err)
@ -51,12 +55,16 @@ func TestRenderToBytes_Yaml(t *testing.T) {
bar: '{{ readFile "data.txt" }}'
`
valuesFile := "values.yaml"
r := NewFileRenderer(&filesystem.FileSystem{ReadFile: func(filename string) ([]byte, error) {
if filename == valuesFile {
return []byte(valuesYamlContent), nil
}
return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", valuesFile, filename)
}}, "", emptyEnvTmplData)
r := NewFileRenderer(&filesystem.FileSystem{
Glob: func(pattern string) ([]string, error) {
return nil, nil
},
ReadFile: func(filename string) ([]byte, error) {
if filename == valuesFile {
return []byte(valuesYamlContent), nil
}
return nil, fmt.Errorf("unexpected filename: expected=%v, actual=%s", valuesFile, filename)
}}, "", emptyEnvTmplData)
buf, err := r.RenderToBytes(valuesFile)
if err != nil {
t.Errorf("unexpected error: %v", err)

View File

@ -2,10 +2,18 @@ package tmpl
import (
"testing"
"github.com/helmfile/helmfile/pkg/filesystem"
)
func TestMergeOverwrite(t *testing.T) {
ctx := &Context{}
ctx := &Context{
fs: &filesystem.FileSystem{
Glob: func(pattern string) ([]string, error) {
return nil, nil
},
},
}
buf, err := ctx.RenderTemplateToBuffer(`
{{- $v1 := dict "bool" true "int" 2 "str" "v1" "str2" "v1" -}}
{{- $v2 := dict "bool" false "int" 0 "str" "v2" "str2" "" -}}

View File

@ -76,6 +76,7 @@ ${kubectl} create namespace ${test_ns} || fail "Could not create namespace ${tes
# TEST CASES----------------------------------------------------------------------------------------------------------
. ${dir}/test-cases/include-template-func.sh
. ${dir}/test-cases/happypath.sh
. ${dir}/test-cases/chartify-with-non-chart-dir.sh
. ${dir}/test-cases/diff-args.sh

View File

@ -0,0 +1,23 @@
include_template_func_case_input_dir="${cases_dir}/include-template-func/input"
include_template_func_case_output_dir="${cases_dir}/include-template-func/output"
config_file="helmfile.yaml.gotmpl"
include_template_func_template_out_file=${include_template_func_case_output_dir}/template-result
if [[ $EXTRA_HELMFILE_FLAGS == *--enable-live-output* ]]; then
include_template_func_template_out_file=${include_template_func_case_output_dir}/template-result-live
fi
include_template_func_template_tmp=$(mktemp -d)
include_template_func_template_reverse=${include_template_func_template_tmp}/include_template_func.template.build.yaml
test_start "include_template_func template"
info "Comparing include_template_func template output ${include_template_func_template_reverse} with ${include_template_func_case_output_dir}/result.yaml"
for i in $(seq 10); do
info "Comparing build/include_template_func-template #$i"
${helmfile} -f ${include_template_func_case_input_dir}/${config_file} template --concurrency 1
${helmfile} -f ${include_template_func_case_input_dir}/${config_file} template --concurrency 1 &> ${include_template_func_template_reverse} || fail "\"helmfile template\" shouldn't fail"
diff -u ${include_template_func_template_out_file} ${include_template_func_template_reverse} || fail "\"helmfile template\" should be consistent"
echo code=$?
done
test_pass "include_template_func template"

View File

@ -0,0 +1,3 @@
{{- define "echo" -}}
{{ .Echo }}
{{- end }}

View File

@ -0,0 +1,8 @@
releases:
- name: '{{ include "echo" (dict "Echo" "include") }}'
chart: ../../../charts/raw
values:
- values/configmap.gotmpl
helmfiles:
- nested/helmfile.yaml.gotmpl

View File

@ -0,0 +1,3 @@
{{- define "echo" -}}
nested-{{ .Echo }}
{{- end }}

View File

@ -0,0 +1,12 @@
releases:
- name: '{{ include "echo" (dict "Echo" "include") }}'
chart: ../../../../charts/raw
values:
- templates:
- |
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "echo" (dict "Echo" "include") }}
data:
name: {{ include "echo" (dict "Echo" "include") }}

View File

@ -0,0 +1,3 @@
{{- define "echo" -}}
{{ .Echo }}
{{- end }}

View File

@ -0,0 +1,8 @@
templates:
- |
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "echo" (dict "Echo" "include") }}
data:
name: {{ include "echo" (dict "Echo" "include") }}

View File

@ -0,0 +1,22 @@
Building dependency release=nested-include, chart=../../../../charts/raw
Templating release=nested-include, chart=../../../../charts/raw
---
# Source: raw/templates/resources.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: nested-include
data:
name: nested-include
Building dependency release=include, chart=../../../charts/raw
Templating release=include, chart=../../../charts/raw
---
# Source: raw/templates/resources.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: include
data:
name: include

View File

@ -0,0 +1,23 @@
Live output is enabled
Building dependency release=nested-include, chart=../../../../charts/raw
Templating release=nested-include, chart=../../../../charts/raw
---
# Source: raw/templates/resources.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: nested-include
data:
name: nested-include
Building dependency release=include, chart=../../../charts/raw
Templating release=include, chart=../../../charts/raw
---
# Source: raw/templates/resources.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: include
data:
name: include