diff --git a/docs/writing-helmfile.md b/docs/writing-helmfile.md index 2067577c..e4eab93d 100644 --- a/docs/writing-helmfile.md +++ b/docs/writing-helmfile.md @@ -125,23 +125,26 @@ environments: production: ``` -At run time, template expressions in your `helmfile.yaml` are executed: +At run time, `bases` in your `helmfile.yaml` are evaluated to produce: ```yaml +# commons.yaml releases: - name: metricbaet chart: stable/metricbeat --- +# environments.yaml environments: development: production: --- +# helmfile.yaml releases: - name: myapp chart: mychart ``` -Resulting YAML documents are merged in the order of occurrence, +Finally the resulting YAML documents are merged in the order of occurrence, so that your `helmfile.yaml` becomes: ```yaml @@ -159,3 +162,5 @@ releases: Great! Now, repeat the above steps for each your `helmfile.yaml`, so that all your helmfiles becomes DRY. + +Please also see [the discussion in the issue 388](https://github.com/roboll/helmfile/issues/388#issuecomment-491710348) for more advanced layering examples. diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 14a6de63..52705cd9 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -702,8 +702,8 @@ releases: helmDefaults: tillerNamespace: {{ .Environment.Values.tillerNs }} `), -"/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`), -"/path/to/yaml/templates.yaml": []byte(`templates: + "/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`), + "/path/to/yaml/templates.yaml": []byte(`templates: default: &default missingFileHandler: Warn values: ["` + "{{`" + `{{.Release.Name}}` + "`}}" + `/values.yaml"] @@ -741,3 +741,104 @@ helmDefaults: t.Errorf("unexpected releases[0].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0]) } } + +func TestLoadDesiredStateFromYaml_MultiPartTemplate(t *testing.T) { + yamlFile := "/path/to/yaml/file" + yamlContent := []byte(`bases: +- ../base.yaml +--- +bases: +- ../base.gotmpl +--- +helmDefaults: + kubeContext: {{ .Environment.Values.foo }} +--- +releases: +- name: myrelease0 + chart: mychart0 +--- + +{{ readFile "templates.yaml" }} + +releases: +- name: myrelease1 + chart: mychart1 + labels: + stage: pre + foo: bar +- name: myrelease1 + chart: mychart2 + labels: + stage: post + <<: *default +`) + files := map[string][]byte{ + yamlFile: yamlContent, + "/path/to/base.yaml": []byte(`environments: + default: + values: + - environments/default/1.yaml +`), + "/path/to/yaml/environments/default/1.yaml": []byte(`foo: FOO`), + "/path/to/base.gotmpl": []byte(`environments: + default: + values: + - environments/default/2.yaml + +helmDefaults: + tillerNamespace: {{ .Environment.Values.tillerNs }} +`), + "/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`), + "/path/to/yaml/templates.yaml": []byte(`templates: + default: &default + missingFileHandler: Warn + values: ["` + "{{`" + `{{.Release.Name}}` + "`}}" + `/values.yaml"] +`), + } + readFile := func(filename string) ([]byte, error) { + content, ok := files[filename] + if !ok { + return nil, fmt.Errorf("unexpected filename: %s", filename) + } + return content, nil + } + app := &App{ + readFile: readFile, + glob: filepath.Glob, + abs: filepath.Abs, + Env: "default", + Logger: helmexec.NewLogger(os.Stderr, "debug"), + } + st, err := app.loadDesiredStateFromYaml(yamlFile) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if st.HelmDefaults.TillerNamespace != "TILLER_NS" { + t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace) + } + + if st.Releases[0].Name != "myrelease0" { + t.Errorf("unexpected releases[0].name: expected=myrelease0, got=%s", st.Releases[0].Name) + } + if st.Releases[1].Name != "myrelease1" { + t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", st.Releases[1].Name) + } + if st.Releases[2].Name != "myrelease1" { + t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", st.Releases[2].Name) + } + if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { + t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0]) + } + if *st.Releases[2].MissingFileHandler != "Warn" { + t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler) + } + + if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { + t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0]) + } + + if st.HelmDefaults.KubeContext != "FOO" { + t.Errorf("unexpected helmDefaults.kubeContext: expected=FOO, got=%s", st.HelmDefaults.KubeContext) + } +} diff --git a/pkg/app/desired_state_file_loader.go b/pkg/app/desired_state_file_loader.go index dda9e3e6..c8010d6a 100644 --- a/pkg/app/desired_state_file_loader.go +++ b/pkg/app/desired_state_file_loader.go @@ -1,8 +1,11 @@ package app import ( + "bytes" + "errors" "fmt" "github.com/imdario/mergo" + "github.com/roboll/helmfile/environment" "github.com/roboll/helmfile/state" "go.uber.org/zap" "log" @@ -26,10 +29,10 @@ type desiredStateLoader struct { } func (ld *desiredStateLoader) Load(f string) (*state.HelmState, error) { - return ld.load(filepath.Dir(f), filepath.Base(f), true) + return ld.loadFile(filepath.Dir(f), filepath.Base(f), true) } -func (ld *desiredStateLoader) load(baseDir, file string, evaluateBases bool) (*state.HelmState, error) { +func (ld *desiredStateLoader) loadFile(baseDir, file string, evaluateBases bool) (*state.HelmState, error) { var f string if filepath.IsAbs(file) { f = file @@ -44,51 +47,28 @@ func (ld *desiredStateLoader) load(baseDir, file string, evaluateBases bool) (*s ext := filepath.Ext(f) - var yamlBytes []byte + var self *state.HelmState + if !experimentalModeEnabled() || ext == ".gotmpl" { - yamlBuf, err := ld.renderTemplateToYaml(baseDir, f, fileBytes) - if err != nil { - return nil, fmt.Errorf("error during %s parsing: %v", f, err) - } - yamlBytes = yamlBuf.Bytes() + self, err = ld.renderAndLoad( + baseDir, + f, + fileBytes, + evaluateBases, + ) } else { - yamlBytes = fileBytes + self, err = ld.load( + fileBytes, + baseDir, + file, + evaluateBases, + ) } - self, err := ld.loadYaml( - yamlBytes, - baseDir, - file, - ) - - if err != nil { - return nil, err - } - - if !evaluateBases { - return self, nil - } - - layers := []*state.HelmState{} - for _, b := range self.Bases { - base, err := ld.load(baseDir, b, false) - if err != nil { - return nil, err - } - layers = append(layers, base) - } - layers = append(layers, self) - - for i := 1; i < len(layers); i++ { - if err := mergo.Merge(layers[0], layers[i], mergo.WithAppendSlice); err != nil { - return nil, err - } - } - - return layers[0], nil + return self, err } -func (a *desiredStateLoader) loadYaml(yaml []byte, baseDir, file string) (*state.HelmState, error) { +func (a *desiredStateLoader) load(yaml []byte, baseDir, file string, evaluateBases bool) (*state.HelmState, error) { c := state.NewCreator(a.logger, a.readFile, a.abs) st, err := c.ParseAndLoadEnv(yaml, baseDir, file, a.env) if err != nil { @@ -141,5 +121,83 @@ func (a *desiredStateLoader) loadYaml(yaml []byte, baseDir, file string) (*state st.Namespace = a.namespace } - return st, nil + if err != nil { + return nil, err + } + + if !evaluateBases { + if len(st.Bases) > 0 { + return nil, errors.New("nested `base` helmfile is unsupported. please submit a feature request if you need this!") + } + + return st, nil + } + + layers := []*state.HelmState{} + for _, b := range st.Bases { + base, err := a.loadFile(baseDir, b, false) + if err != nil { + return nil, err + } + layers = append(layers, base) + } + layers = append(layers, st) + + for i := 1; i < len(layers); i++ { + if err := mergo.Merge(layers[0], layers[i], mergo.WithAppendSlice); err != nil { + return nil, err + } + } + + return layers[0], nil +} + +func (ld *desiredStateLoader) renderAndLoad(baseDir, filename string, content []byte, evaluateBases bool) (*state.HelmState, error) { + parts := bytes.Split(content, []byte("\n---\n")) + + var finalState *state.HelmState + var env *environment.Environment + + for i, part := range parts { + var yamlBuf *bytes.Buffer + var err error + + id := fmt.Sprintf("%s.part.%d", filename, i) + + if env == nil { + yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part) + if err != nil { + return nil, fmt.Errorf("error during %s parsing: %v", id, err) + } + } else { + yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part, *env) + if err != nil { + return nil, fmt.Errorf("error during %s parsing: %v", id, err) + } + } + + currentState, err := ld.load( + yamlBuf.Bytes(), + baseDir, + filename, + evaluateBases, + ) + if err != nil { + return nil, err + } + + if finalState == nil { + finalState = currentState + } else { + if err := mergo.Merge(finalState, currentState, mergo.WithAppendSlice); err != nil { + return nil, err + } + } + + env = &finalState.Env + + ld.logger.Debugf("merged environment: %v", env) + } + + return finalState, nil } diff --git a/pkg/app/two_pass_renderer.go b/pkg/app/two_pass_renderer.go index 291f0a29..8c372b4b 100644 --- a/pkg/app/two_pass_renderer.go +++ b/pkg/app/two_pass_renderer.go @@ -3,6 +3,7 @@ package app import ( "bytes" "fmt" + "github.com/imdario/mergo" "github.com/roboll/helmfile/environment" "github.com/roboll/helmfile/state" "github.com/roboll/helmfile/tmpl" @@ -18,8 +19,7 @@ func prependLineNumbers(text string) string { return buf.String() } -func (r *desiredStateLoader) renderEnvironment(baseDir, filename string, content []byte) environment.Environment { - firstPassEnv := environment.Environment{Name: r.env, Values: map[string]interface{}(nil)} +func (r *desiredStateLoader) renderEnvironment(firstPassEnv environment.Environment, baseDir, filename string, content []byte) environment.Environment { tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace} firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData) @@ -43,15 +43,45 @@ func (r *desiredStateLoader) renderEnvironment(baseDir, filename string, content } r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String())) } + if prestate != nil { - firstPassEnv = prestate.Env + intEnv := environment.Environment{Name: firstPassEnv.Name} + if err := mergo.Merge(&intEnv, &firstPassEnv, mergo.WithAppendSlice); err != nil { + r.logger.Debugf("error in first-pass rendering: result of \"%s\": %v", filename, err) + return firstPassEnv + } + if err := mergo.Merge(&intEnv, &prestate.Env, mergo.WithAppendSlice); err != nil { + r.logger.Debugf("error in first-pass rendering: result of \"%s\": %v", filename, err) + return firstPassEnv + } + firstPassEnv = intEnv } return firstPassEnv } -func (r *desiredStateLoader) renderTemplateToYaml(baseDir, filename string, content []byte) (*bytes.Buffer, error) { +func (r *desiredStateLoader) renderTemplatesToYaml(baseDir, filename string, content []byte, context ...environment.Environment) (*bytes.Buffer, error) { + var env environment.Environment + + if len(context) > 0 { + env = context[0] + } else { + env = environment.Environment{Name: r.env, Values: map[string]interface{}(nil)} + } + + return r.twoPassRenderTemplateToYaml(env, baseDir, filename, content) +} + +func (r *desiredStateLoader) twoPassRenderTemplateToYaml(initEnv environment.Environment, baseDir, filename string, content []byte) (*bytes.Buffer, error) { // try a first pass render. This will always succeed, but can produce a limited env - firstPassEnv := r.renderEnvironment(baseDir, filename, content) + if r.logger != nil { + r.logger.Debugf("first-pass rendering input of \"%s\": %v", filename, initEnv) + } + + firstPassEnv := r.renderEnvironment(initEnv, baseDir, filename, content) + + if r.logger != nil { + r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, firstPassEnv) + } tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace} secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData) diff --git a/pkg/app/two_pass_renderer_test.go b/pkg/app/two_pass_renderer_test.go index bf8072fe..e042c1c4 100644 --- a/pkg/app/two_pass_renderer_test.go +++ b/pkg/app/two_pass_renderer_test.go @@ -51,7 +51,7 @@ releases: } r := makeLoader(fileReader, "staging") - yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent) + yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -103,7 +103,7 @@ releases: r := makeLoader(fileReader, "staging") // test the double rendering - yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent) + yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -152,7 +152,7 @@ releases: r := makeLoader(fileReader, "staging") // test the double rendering - _, err := r.renderTemplateToYaml("", "", yamlContent) + _, err := r.renderTemplatesToYaml("", "", yamlContent) if !strings.Contains(err.Error(), "stringTemplate:8") { t.Fatalf("error should contain a stringTemplate error (reference to unknow key) %v", err) @@ -190,7 +190,7 @@ releases: } r := makeLoader(fileReader, "staging") - rendered, _ := r.renderTemplateToYaml("", "", yamlContent) + rendered, _ := r.renderTemplatesToYaml("", "", yamlContent) var state state.HelmState yaml.Unmarshal(rendered.Bytes(), &state) @@ -217,7 +217,7 @@ func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) { } r := makeLoader(fileReader, "staging") - yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent) + yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -248,7 +248,7 @@ releases: } r := makeLoader(fileReader, "staging") - _, err := r.renderTemplateToYaml("", "", yamlContent) + _, err := r.renderTemplatesToYaml("", "", yamlContent) if err == nil { t.Fatalf("wanted error, none returned") }