public_git
/
helmfile
mirror of https://github.com/helmfile/helmfile.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #587 from roboll/layering-enhancements 

feat: helmfile.yaml layering enhancements

The current  [Layering](https://github.com/roboll/helmfile/blob/master/docs/writing-helmfile.md#layering) system didn't work as documented, as it relies on helmfile to template each "part" of your helmfile.yaml THEN merge them one by one.

The reality was that helmfile template all the parts of your helmfile.yaml at once, and then merge those YAML documents. In https://github.com/roboll/helmfile/issues/388#issuecomment-436186278, @sruon was making a GREAT point that we may need to change helmfile to render templates earlier - that is to evaluate a template per each helmfile.yaml part separated by `---`. Sorry I missed my expertise to follow your great idea last year @sruon  😭 

Anyways, this, in combination with the wrong documentation, has made so many people confused. To finally overcome this situation, here's a fairly large PR that introduces the 2 enhancements:

- `bases:` for easier layering without go template expressions, especially `{{ readFunc "path/to/file" }}`s. This is the first commit of this PR.
- `helmfile.yaml` is splited by the separator `---` at first. Each part is then rendered as a go template(double-render applies as before). Finally, All the results are merged in the order of occurence. I assume this as an enhanced version of @sruon's work. This is the second commit of this PR.

Resolves #388
Resolve #584
Resolves #585 (`HELMFILE_EXPERIMENTA=true -f helmfile.yaml helmfile` disables the whole-file templating, treating the helmfile.yaml as a regular YAML file as the file ext. denotes. Use `helmfile.yaml.gotmpl` or `helmfile.gotmpl` to enable)
Fixes #568 (Use `bases` or `readFile` rather than not importing implicitly with `helmfile.d`
This commit is contained in:
KUOKA Yusuke 2019-05-14 09:48:20 +09:00 committed by GitHub
parent 4f83e69bf6 aef366660b
commit 255d92064e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 528 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs/writing-helmfile.md
View File

@ -100,10 +100,10 @@ Use Layering to extract the common parts into a dedicated *library helmfile*s, s
Let's assume that your `helmfile.yaml` looks like:
```
{ readFile "commons.yaml" }}
---
{{ readFile "environments.yaml" }}
---
bases:
- commons.yaml
- environments.yaml
releases:
- name: myapp
chart: mychart
@ -125,23 +125,26 @@ environments:
production:
```
At run time, template expressions in your `helmfile.yaml` are executed:
At run time, `bases` in your `helmfile.yaml` are evaluated to produce:
```yaml
# commons.yaml
releases:
- name: metricbaet
chart: stable/metricbeat
---
# environments.yaml
environments:
development:
production:
---
# helmfile.yaml
releases:
- name: myapp
chart: mychart
```
Resulting YAML documents are merged in the order of occurrence,
Finally the resulting YAML documents are merged in the order of occurrence,
so that your `helmfile.yaml` becomes:
```yaml
@ -159,3 +162,5 @@ releases:
Great!
Now, repeat the above steps for each your `helmfile.yaml`, so that all your helmfiles becomes DRY.
Please also see [the discussion in the issue 388](https://github.com/roboll/helmfile/issues/388#issuecomment-491710348) for more advanced layering examples.

128
pkg/app/app.go
View File

@ -7,6 +7,7 @@ import (
"os"
"os/signal"
"strings"
"syscall"
"github.com/roboll/helmfile/helmexec"
"github.com/roboll/helmfile/state"
@ -14,7 +15,6 @@ import (
"path/filepath"
"sort"
"syscall"
)
type App struct {
@ -111,34 +111,42 @@ func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error {
return nil
}
func (a *App) loadDesiredStateFromYaml(file string) (*state.HelmState, error) {
ld := &desiredStateLoader{
readFile: a.readFile,
env: a.Env,
namespace: a.Namespace,
logger: a.Logger,
abs: a.abs,
Reverse: a.Reverse,
KubeContext: a.KubeContext,
glob: a.glob,
}
return ld.Load(file)
}
func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error {
noMatchInHelmfiles := true
err := a.visitStateFiles(fileOrDir, func(f string) error {
content, err := a.readFile(f)
if err != nil {
return err
}
// render template, in two runs
r := &twoPassRenderer{
reader: a.readFile,
env: a.Env,
namespace: a.Namespace,
filename: f,
logger: a.Logger,
abs: a.abs,
}
yamlBuf, err := r.renderTemplate(content)
if err != nil {
return fmt.Errorf("error during %s parsing: %v", f, err)
}
st, err := a.loadDesiredStateFromYaml(f)
st, err := a.loadDesiredStateFromYaml(
yamlBuf.Bytes(),
f,
a.Namespace,
a.Env,
)
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-sigs
errs := []error{fmt.Errorf("Received [%s] to shutdown ", sig)}
_ = context{a, st}.clean(errs)
// See http://tldp.org/LDP/abs/html/exitcodes.html
switch sig {
case syscall.SIGINT:
os.Exit(130)
case syscall.SIGTERM:
os.Exit(143)
}
}()
ctx := context{a, st}
@ -313,78 +321,6 @@ func directoryExistsAt(path string) bool {
return err == nil && fileInfo.Mode().IsDir()
}
func (a *App) loadDesiredStateFromYaml(yaml []byte, file string, namespace string, env string) (*state.HelmState, error) {
c := state.NewCreator(a.Logger, a.readFile, a.abs)
st, err := c.CreateFromYaml(yaml, file, env)
if err != nil {
return nil, err
}
helmfiles := []state.SubHelmfileSpec{}
for _, hf := range st.Helmfiles {
globPattern := hf.Path
var absPathPattern string
if filepath.IsAbs(globPattern) {
absPathPattern = globPattern
} else {
absPathPattern = st.JoinBase(globPattern)
}
matches, err := a.glob(absPathPattern)
if err != nil {
return nil, fmt.Errorf("failed processing %s: %v", globPattern, err)
}
sort.Strings(matches)
for _, match := range matches {
newHelmfile := hf
newHelmfile.Path = match
helmfiles = append(helmfiles, newHelmfile)
}
}
st.Helmfiles = helmfiles
if a.Reverse {
rev := func(i, j int) bool {
return j < i
}
sort.Slice(st.Releases, rev)
sort.Slice(st.Helmfiles, rev)
}
if a.KubeContext != "" {
if st.HelmDefaults.KubeContext != "" {
log.Printf("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.")
os.Exit(1)
}
st.HelmDefaults.KubeContext = a.KubeContext
}
if namespace != "" {
if st.Namespace != "" {
log.Printf("err: Cannot use option --namespace and set attribute namespace.")
os.Exit(1)
}
st.Namespace = namespace
}
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-sigs
errs := []error{fmt.Errorf("Received [%s] to shutdown ", sig)}
_ = context{a, st}.clean(errs)
// See http://tldp.org/LDP/abs/html/exitcodes.html
switch sig {
case syscall.SIGINT:
os.Exit(130)
case syscall.SIGTERM:
os.Exit(143)
}
}()
return st, nil
}
type Error struct {
msg string

202
pkg/app/app_test.go
View File

@ -2,7 +2,6 @@ package app
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
@ -83,7 +82,14 @@ func (f *testFs) readFile(filename string) ([]byte, error) {
return []byte(str), nil
}
func (f *testFs) glob(pattern string) ([]string, error) {
func (f *testFs) glob(relPattern string) ([]string, error) {
var pattern string
if relPattern[0] == '/' {
pattern = relPattern
} else {
pattern = filepath.Join(f.wd, relPattern)
}
matches := []string{}
for name, _ := range f.files {
matched, err := filepath.Match(pattern, name)
@ -95,7 +101,7 @@ func (f *testFs) glob(pattern string) ([]string, error) {
}
}
if len(matches) == 0 {
return []string(nil), fmt.Errorf("no file matched: %s", pattern)
return []string(nil), fmt.Errorf("no file matched %s for files: %v", pattern, f.files)
}
return matches, nil
}
@ -640,15 +646,199 @@ func TestLoadDesiredStateFromYaml_DuplicateReleaseName(t *testing.T) {
labels:
stage: post
`)
readFile := func(filename string) ([]byte, error) {
if filename != yamlFile {
return nil, fmt.Errorf("unexpected filename: %s", filename)
}
return yamlContent, nil
}
app := &App{
readFile: ioutil.ReadFile,
readFile: readFile,
glob: filepath.Glob,
abs: filepath.Abs,
KubeContext: "default",
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
_, err := app.loadDesiredStateFromYaml(yamlContent, yamlFile, "default", "default")
_, err := app.loadDesiredStateFromYaml(yamlFile)
if err != nil {
t.Error("unexpected error")
t.Errorf("unexpected error: %v", err)
}
}
func TestLoadDesiredStateFromYaml_Bases(t *testing.T) {
yamlFile := "/path/to/yaml/file"
yamlContent := []byte(`bases:
- ../base.yaml
- ../base.gotmpl
{{ readFile "templates.yaml" }}
releases:
- name: myrelease1
chart: mychart1
labels:
stage: pre
foo: bar
- name: myrelease1
chart: mychart2
labels:
stage: post
<<: *default
`)
files := map[string][]byte{
yamlFile: yamlContent,
"/path/to/base.yaml": []byte(`environments:
default:
values:
- environments/default/1.yaml
`),
"/path/to/yaml/environments/default/1.yaml": []byte(`foo: FOO`),
"/path/to/base.gotmpl": []byte(`environments:
default:
values:
- environments/default/2.yaml
helmDefaults:
tillerNamespace: {{ .Environment.Values.tillerNs }}
`),
"/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`),
"/path/to/yaml/templates.yaml": []byte(`templates:
default: &default
missingFileHandler: Warn
values: ["` + "{{`" + `{{.Release.Name}}` + "`}}" + `/values.yaml"]
`),
}
readFile := func(filename string) ([]byte, error) {
content, ok := files[filename]
if !ok {
return nil, fmt.Errorf("unexpected filename: %s", filename)
}
return content, nil
}
app := &App{
readFile: readFile,
glob: filepath.Glob,
abs: filepath.Abs,
KubeContext: "default",
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
st, err := app.loadDesiredStateFromYaml(yamlFile)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if st.HelmDefaults.TillerNamespace != "TILLER_NS" {
t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace)
}
if *st.Releases[1].MissingFileHandler != "Warn" {
t.Errorf("unexpected releases[0].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler)
}
if st.Releases[1].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
t.Errorf("unexpected releases[0].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
}
}
func TestLoadDesiredStateFromYaml_MultiPartTemplate(t *testing.T) {
yamlFile := "/path/to/yaml/file"
yamlContent := []byte(`bases:
- ../base.yaml
---
bases:
- ../base.gotmpl
---
helmDefaults:
kubeContext: {{ .Environment.Values.foo }}
---
releases:
- name: myrelease0
chart: mychart0
---
{{ readFile "templates.yaml" }}
releases:
- name: myrelease1
chart: mychart1
labels:
stage: pre
foo: bar
- name: myrelease1
chart: mychart2
labels:
stage: post
<<: *default
`)
files := map[string][]byte{
yamlFile: yamlContent,
"/path/to/base.yaml": []byte(`environments:
default:
values:
- environments/default/1.yaml
`),
"/path/to/yaml/environments/default/1.yaml": []byte(`foo: FOO`),
"/path/to/base.gotmpl": []byte(`environments:
default:
values:
- environments/default/2.yaml
helmDefaults:
tillerNamespace: {{ .Environment.Values.tillerNs }}
`),
"/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`),
"/path/to/yaml/templates.yaml": []byte(`templates:
default: &default
missingFileHandler: Warn
values: ["` + "{{`" + `{{.Release.Name}}` + "`}}" + `/values.yaml"]
`),
}
readFile := func(filename string) ([]byte, error) {
content, ok := files[filename]
if !ok {
return nil, fmt.Errorf("unexpected filename: %s", filename)
}
return content, nil
}
app := &App{
readFile: readFile,
glob: filepath.Glob,
abs: filepath.Abs,
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
st, err := app.loadDesiredStateFromYaml(yamlFile)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if st.HelmDefaults.TillerNamespace != "TILLER_NS" {
t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace)
}
if st.Releases[0].Name != "myrelease0" {
t.Errorf("unexpected releases[0].name: expected=myrelease0, got=%s", st.Releases[0].Name)
}
if st.Releases[1].Name != "myrelease1" {
t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", st.Releases[1].Name)
}
if st.Releases[2].Name != "myrelease1" {
t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", st.Releases[2].Name)
}
if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
}
if *st.Releases[2].MissingFileHandler != "Warn" {
t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler)
}
if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
}
if st.HelmDefaults.KubeContext != "FOO" {
t.Errorf("unexpected helmDefaults.kubeContext: expected=FOO, got=%s", st.HelmDefaults.KubeContext)
}
}

8
pkg/app/constants.go
View File

@ -13,6 +13,10 @@ const (
ExperimentalSelectorExplicit = "explicit-selector-inheritance" // value to remove default selector inheritance to sub-helmfiles and use the explicit one
)
func isExplicitSelectorInheritanceEnabled() bool {
return os.Getenv(ExperimentalEnvVar) == "true" || strings.Contains(os.Getenv(ExperimentalEnvVar), ExperimentalSelectorExplicit)
func experimentalModeEnabled() bool {
return os.Getenv(ExperimentalEnvVar) == "true"
}
func isExplicitSelectorInheritanceEnabled() bool {
return experimentalModeEnabled() || strings.Contains(os.Getenv(ExperimentalEnvVar), ExperimentalSelectorExplicit)
}

203
pkg/app/desired_state_file_loader.go Normal file
View File

@ -0,0 +1,203 @@
package app
import (
"bytes"
"errors"
"fmt"
"github.com/imdario/mergo"
"github.com/roboll/helmfile/environment"
"github.com/roboll/helmfile/state"
"go.uber.org/zap"
"log"
"os"
"path/filepath"
"sort"
)
type desiredStateLoader struct {
KubeContext string
Reverse bool
env string
namespace string
readFile func(string) ([]byte, error)
abs func(string) (string, error)
glob func(string) ([]string, error)
logger *zap.SugaredLogger
}
func (ld *desiredStateLoader) Load(f string) (*state.HelmState, error) {
return ld.loadFile(filepath.Dir(f), filepath.Base(f), true)
}
func (ld *desiredStateLoader) loadFile(baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
var f string
if filepath.IsAbs(file) {
f = file
} else {
f = filepath.Join(baseDir, file)
}
fileBytes, err := ld.readFile(f)
if err != nil {
return nil, err
}
ext := filepath.Ext(f)
var self *state.HelmState
if !experimentalModeEnabled() || ext == ".gotmpl" {
self, err = ld.renderAndLoad(
baseDir,
f,
fileBytes,
evaluateBases,
)
} else {
self, err = ld.load(
fileBytes,
baseDir,
file,
evaluateBases,
)
}
return self, err
}
func (a *desiredStateLoader) load(yaml []byte, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
c := state.NewCreator(a.logger, a.readFile, a.abs)
st, err := c.ParseAndLoadEnv(yaml, baseDir, file, a.env)
if err != nil {
return nil, err
}
helmfiles := []state.SubHelmfileSpec{}
for _, hf := range st.Helmfiles {
globPattern := hf.Path
var absPathPattern string
if filepath.IsAbs(globPattern) {
absPathPattern = globPattern
} else {
absPathPattern = st.JoinBase(globPattern)
}
matches, err := a.glob(absPathPattern)
if err != nil {
return nil, fmt.Errorf("failed processing %s: %v", globPattern, err)
}
sort.Strings(matches)
for _, match := range matches {
newHelmfile := hf
newHelmfile.Path = match
helmfiles = append(helmfiles, newHelmfile)
}
}
st.Helmfiles = helmfiles
if a.Reverse {
rev := func(i, j int) bool {
return j < i
}
sort.Slice(st.Releases, rev)
sort.Slice(st.Helmfiles, rev)
}
if a.KubeContext != "" {
if st.HelmDefaults.KubeContext != "" {
log.Printf("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.")
os.Exit(1)
}
st.HelmDefaults.KubeContext = a.KubeContext
}
if a.namespace != "" {
if st.Namespace != "" {
log.Printf("err: Cannot use option --namespace and set attribute namespace.")
os.Exit(1)
}
st.Namespace = a.namespace
}
if err != nil {
return nil, err
}
if !evaluateBases {
if len(st.Bases) > 0 {
return nil, errors.New("nested `base` helmfile is unsupported. please submit a feature request if you need this!")
}
return st, nil
}
layers := []*state.HelmState{}
for _, b := range st.Bases {
base, err := a.loadFile(baseDir, b, false)
if err != nil {
return nil, err
}
layers = append(layers, base)
}
layers = append(layers, st)
for i := 1; i < len(layers); i++ {
if err := mergo.Merge(layers[0], layers[i], mergo.WithAppendSlice); err != nil {
return nil, err
}
}
return layers[0], nil
}
func (ld *desiredStateLoader) renderAndLoad(baseDir, filename string, content []byte, evaluateBases bool) (*state.HelmState, error) {
parts := bytes.Split(content, []byte("\n---\n"))
var finalState *state.HelmState
var env *environment.Environment
for i, part := range parts {
var yamlBuf *bytes.Buffer
var err error
id := fmt.Sprintf("%s.part.%d", filename, i)
if env == nil {
yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part)
if err != nil {
return nil, fmt.Errorf("error during %s parsing: %v", id, err)
}
} else {
yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part, *env)
if err != nil {
return nil, fmt.Errorf("error during %s parsing: %v", id, err)
}
}
currentState, err := ld.load(
yamlBuf.Bytes(),
baseDir,
filename,
evaluateBases,
)
if err != nil {
return nil, err
}
if finalState == nil {
finalState = currentState
} else {
if err := mergo.Merge(finalState, currentState, mergo.WithAppendSlice); err != nil {
return nil, err
}
}
env = &finalState.Env
ld.logger.Debugf("merged environment: %v", env)
}
return finalState, nil
}

67
pkg/app/two_pass_renderer.go
View File

@ -3,11 +3,10 @@ package app
import (
"bytes"
"fmt"
"github.com/imdario/mergo"
"github.com/roboll/helmfile/environment"
"github.com/roboll/helmfile/state"
"github.com/roboll/helmfile/tmpl"
"go.uber.org/zap"
"path/filepath"
"strings"
)
@ -20,61 +19,81 @@ func prependLineNumbers(text string) string {
return buf.String()
}
type twoPassRenderer struct {
reader func(string) ([]byte, error)
env string
namespace string
filename string
logger *zap.SugaredLogger
abs func(string) (string, error)
}
func (r *twoPassRenderer) renderEnvironment(content []byte) environment.Environment {
firstPassEnv := environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
func (r *desiredStateLoader) renderEnvironment(firstPassEnv environment.Environment, baseDir, filename string, content []byte) environment.Environment {
tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
firstPassRenderer := tmpl.NewFirstPassRenderer(filepath.Dir(r.filename), tmplData)
firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData)
// parse as much as we can, tolerate errors, this is a preparse
yamlBuf, err := firstPassRenderer.RenderTemplateContentToBuffer(content)
if err != nil && r.logger != nil {
r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content)))
r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", filename, prependLineNumbers(string(content)))
if yamlBuf == nil { // we have a template syntax error, let the second parse report
r.logger.Debugf("template syntax error: %v", err)
return firstPassEnv
}
}
c := state.NewCreator(r.logger, r.reader, r.abs)
c := state.NewCreator(r.logger, r.readFile, r.abs)
c.Strict = false
// create preliminary state, as we may have an environment. Tolerate errors.
prestate, err := c.CreateFromYaml(yamlBuf.Bytes(), r.filename, r.env)
prestate, err := c.ParseAndLoadEnv(yamlBuf.Bytes(), baseDir, filename, r.env)
if err != nil && r.logger != nil {
switch err.(type) {
case *state.StateLoadError:
r.logger.Infof("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err)
}
r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String()))
r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String()))
}
if prestate != nil {
firstPassEnv = prestate.Env
intEnv := environment.Environment{Name: firstPassEnv.Name}
if err := mergo.Merge(&intEnv, &firstPassEnv, mergo.WithAppendSlice); err != nil {
r.logger.Debugf("error in first-pass rendering: result of \"%s\": %v", filename, err)
return firstPassEnv
}
if err := mergo.Merge(&intEnv, &prestate.Env, mergo.WithAppendSlice); err != nil {
r.logger.Debugf("error in first-pass rendering: result of \"%s\": %v", filename, err)
return firstPassEnv
}
firstPassEnv = intEnv
}
return firstPassEnv
}
func (r *twoPassRenderer) renderTemplate(content []byte) (*bytes.Buffer, error) {
func (r *desiredStateLoader) renderTemplatesToYaml(baseDir, filename string, content []byte, context ...environment.Environment) (*bytes.Buffer, error) {
var env environment.Environment
if len(context) > 0 {
env = context[0]
} else {
env = environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
}
return r.twoPassRenderTemplateToYaml(env, baseDir, filename, content)
}
func (r *desiredStateLoader) twoPassRenderTemplateToYaml(initEnv environment.Environment, baseDir, filename string, content []byte) (*bytes.Buffer, error) {
// try a first pass render. This will always succeed, but can produce a limited env
firstPassEnv := r.renderEnvironment(content)
if r.logger != nil {
r.logger.Debugf("first-pass rendering input of \"%s\": %v", filename, initEnv)
}
firstPassEnv := r.renderEnvironment(initEnv, baseDir, filename, content)
if r.logger != nil {
r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, firstPassEnv)
}
tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
secondPassRenderer := tmpl.NewFileRenderer(r.reader, filepath.Dir(r.filename), tmplData)
secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData)
yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content)
if err != nil {
if r.logger != nil {
r.logger.Debugf("second-pass rendering failed, input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content)))
r.logger.Debugf("second-pass rendering failed, input of \"%s\":\n%s", filename, prependLineNumbers(string(content)))
}
return nil, err
}
if r.logger != nil {
r.logger.Debugf("second-pass rendering result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String()))
r.logger.Debugf("second-pass rendering result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String()))
}
return yamlBuf, nil
}

33
pkg/app/two_pass_renderer_test.go
View File

@ -12,12 +12,11 @@ import (
"gopkg.in/yaml.v2"
)
func makeRenderer(readFile func(string) ([]byte, error), env string) *twoPassRenderer {
return &twoPassRenderer{
reader: readFile,
func makeLoader(readFile func(string) ([]byte, error), env string) *desiredStateLoader {
return &desiredStateLoader{
readFile: readFile,
env: env,
namespace: "namespace",
filename: "",
logger: helmexec.NewLogger(os.Stdout, "debug"),
abs: filepath.Abs,
}
@ -51,8 +50,8 @@ releases:
return []byte(""), nil
}
r := makeRenderer(fileReader, "staging")
yamlBuf, err := r.renderTemplate(yamlContent)
r := makeLoader(fileReader, "staging")
yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
@ -102,9 +101,9 @@ releases:
return defaultValuesYaml, nil
}
r := makeRenderer(fileReader, "staging")
r := makeLoader(fileReader, "staging")
// test the double rendering
yamlBuf, err := r.renderTemplate(yamlContent)
yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -151,9 +150,9 @@ releases:
return defaultValuesYaml, nil
}
r := makeRenderer(fileReader, "staging")
r := makeLoader(fileReader, "staging")
// test the double rendering
_, err := r.renderTemplate(yamlContent)
_, err := r.renderTemplatesToYaml("", "", yamlContent)
if !strings.Contains(err.Error(), "stringTemplate:8") {
t.Fatalf("error should contain a stringTemplate error (reference to unknow key) %v", err)
@ -190,8 +189,8 @@ releases:
return defaultValuesYamlGotmpl, nil
}
r := makeRenderer(fileReader, "staging")
rendered, _ := r.renderTemplate(yamlContent)
r := makeLoader(fileReader, "staging")
rendered, _ := r.renderTemplatesToYaml("", "", yamlContent)
var state state.HelmState
yaml.Unmarshal(rendered.Bytes(), &state)
@ -217,8 +216,8 @@ func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) {
return defaultValuesYaml, nil
}
r := makeRenderer(fileReader, "staging")
yamlBuf, err := r.renderTemplate(yamlContent)
r := makeLoader(fileReader, "staging")
yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -231,7 +230,7 @@ func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) {
}
}
func TestReadFromYaml_HelfileShouldBeResilentToTemplateErrors(t *testing.T) {
func TestReadFromYaml_HelmfileShouldBeResilentToTemplateErrors(t *testing.T) {
yamlContent := []byte(`environments:
staging:
production:
@ -248,8 +247,8 @@ releases:
return yamlContent, nil
}
r := makeRenderer(fileReader, "staging")
_, err := r.renderTemplate(yamlContent)
r := makeLoader(fileReader, "staging")
_, err := r.renderTemplatesToYaml("", "", yamlContent)
if err == nil {
t.Fatalf("wanted error, none returned")
}

20
state/create.go
View File

@ -4,7 +4,6 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
@ -33,16 +32,6 @@ func (e *UndefinedEnvError) Error() string {
return e.msg
}
func createFromYaml(content []byte, file string, env string, logger *zap.SugaredLogger) (*HelmState, error) {
c := &creator{
logger,
ioutil.ReadFile,
filepath.Abs,
true,
}
return c.CreateFromYaml(content, file, env)
}
type creator struct {
logger *zap.SugaredLogger
readFile func(string) ([]byte, error)
@ -60,15 +49,12 @@ func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error)
}
}
func (c *creator) CreateFromYaml(content []byte, file string, env string) (*HelmState, error) {
// Parses YAML into HelmState, while loading environment values files relative to the `cwd`
func (c *creator) ParseAndLoadEnv(content []byte, baseDir, file string, env string) (*HelmState, error) {
var state HelmState
basePath, err := c.abs(filepath.Dir(file))
if err != nil {
return nil, &StateLoadError{fmt.Sprintf("failed to read %s", file), err}
}
state.FilePath = file
state.basePath = basePath
state.basePath = baseDir
decoder := yaml.NewDecoder(bytes.NewReader(content))
if !c.Strict {

14
state/create_test.go
View File

@ -2,6 +2,8 @@ package state
import (
"fmt"
"go.uber.org/zap"
"io/ioutil"
"path/filepath"
"reflect"
"testing"
@ -10,6 +12,16 @@ import (
"gotest.tools/assert/cmp"
)
func createFromYaml(content []byte, file string, env string, logger *zap.SugaredLogger) (*HelmState, error) {
c := &creator{
logger,
ioutil.ReadFile,
filepath.Abs,
true,
}
return c.ParseAndLoadEnv(content, filepath.Dir(file), file, env)
}
func TestReadFromYaml(t *testing.T) {
yamlFile := "example/path/to/yaml/file"
yamlContent := []byte(`releases:
@ -101,7 +113,7 @@ bar: {{ readFile "bar.txt" }}
return nil, fmt.Errorf("unexpected filename: %s", filename)
}
state, err := NewCreator(logger, readFile, filepath.Abs).CreateFromYaml(yamlContent, yamlFile, "production")
state, err := NewCreator(logger, readFile, filepath.Abs).ParseAndLoadEnv(yamlContent, filepath.Dir(yamlFile), yamlFile, "production")
if err != nil {
t.Errorf("unexpected error: %v", err)
}

8
state/state.go
View File

@ -27,9 +27,11 @@ import (
// HelmState structure for the helmfile
type HelmState struct {
basePath string
Environments map[string]EnvironmentSpec
FilePath string
basePath string
Environments map[string]EnvironmentSpec
FilePath string
Bases []string `yaml:"bases"`
HelmDefaults HelmSpec `yaml:"helmDefaults"`
Helmfiles []SubHelmfileSpec `yaml:"helmfiles"`
DeprecatedContext string `yaml:"context"`