diff --git a/README.md b/README.md index e7f46960..48bf50da 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,28 @@ releases: wait: true # -# Advanced Configuration: Helmfile Environments +# Advanced Configuration: Nested States +# +helmfiles: +- # Path to the helmfile state file being processed BEFORE releases in this state file + path: path/to/subhelmfile.yaml + # Label selector used for filtering releases in the nested state. + # For example, `name=prometheus` in this context is equivalent to processing the nested state like + # helmfile -f path/to/subhelmfile.yaml -l name=prometheus sync + selectors: + - name=prometheus + environment: + values: + # Environment values files merged into the nested state + - additiona.values.yaml + # Inline environment values merged into the nested state + - key1: val1 +- # All the nested state files under `helmfiles:` is processed in the order of definition. + # So it can be used for preparation for your main `releases`. An example would be creating CRDs required by `reelases` in the parent state file. + path: path/to/mycrd.helmfile.yaml + +# +# Advanced Configuration: Environments # # The list of environments managed by helmfile. @@ -703,19 +724,22 @@ Just run `helmfile sync` inside `myteam/`, and you are done. All the files are sorted alphabetically per group = array item inside `helmfiles:`, so that you have granular control over ordering, too. #### selectors + When composing helmfiles you can use selectors from the command line as well as explicit selectors inside the parent helmfile to filter the releases to be used. + ```yaml helmfiles: - apps/*/helmfile.yaml - path: apps/a-helmfile.yaml - selectors: # list of selectors - - name=prometheus - - tier=frontend + selectors: # list of selectors + - name=prometheus + - tier=frontend - path: apps/b-helmfile.yaml # no selector, so all releases are used - selectors: [] +selectors: [] - path: apps/c-helmfile.yaml # parent selector to be used or cli selector for the initial helmfile - selectorsInherited: true + selectorsInherited: true ``` + * When a selector is specified, only this selector applies and the parents or CLI selectors are ignored. * When not selector is specified there are 2 modes for the selector inheritance because we would like to change the current inheritance behavior (see [issue #344](https://github.com/roboll/helmfile/issues/344) ). * Legacy mode, sub-helmfiles without selectors inherit selectors from their parent helmfile. The initial helmfiles inherit from the command line selectors. diff --git a/cmd/cmd.go b/cmd/cmd.go index 5b855b86..c0bb90d1 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -25,7 +25,7 @@ func VisitAllDesiredStates(c *cli.Context, converge func(*state.HelmState, helme return converge(st, helm, ctx) } - err = a.VisitDesiredStates(fileOrDir, a.Selectors, convergeWithHelmBinary) + err = a.VisitDesiredStates(fileOrDir, app.LoadOpts{Selectors: a.Selectors}, convergeWithHelmBinary) return toCliError(c, err) } diff --git a/environment/environment.go b/environment/environment.go index 049c72ab..f10b5252 100644 --- a/environment/environment.go +++ b/environment/environment.go @@ -1,8 +1,45 @@ package environment +import ( + "encoding/json" + "github.com/imdario/mergo" +) + type Environment struct { Name string Values map[string]interface{} } var EmptyEnvironment Environment + +func (e Environment) DeepCopy() Environment { + bytes, err := json.Marshal(e.Values) + if err != nil { + panic(err) + } + var values map[string]interface{} + if err := json.Unmarshal(bytes, &values); err != nil { + panic(err) + } + return Environment{ + Name: e.Name, + Values: values, + } +} + +func (e *Environment) Merge(other *Environment) (*Environment, error) { + if e == nil { + if other != nil { + copy := other.DeepCopy() + return ©, nil + } + return nil, nil + } + copy := e.DeepCopy() + if other != nil { + if err := mergo.Merge(©, other, mergo.WithOverride); err != nil { + return nil, err + } + } + return ©, nil +} diff --git a/pkg/app/app.go b/pkg/app/app.go index 776c3094..b0d7a9b1 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -83,7 +83,7 @@ func (a *App) within(dir string, do func() error) error { return appErr } -func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error { +func (a *App) visitStateFiles(fileOrDir string, do func(string, string) error) error { desiredStateFiles, err := a.findDesiredStateFiles(fileOrDir) if err != nil { return appError("", err) @@ -103,7 +103,12 @@ func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error { a.Logger.Debugf("processing file \"%s\" in directory \"%s\"", file, dir) err := a.within(dir, func() error { - return do(file) + absd, err := a.abs(dir) + if err != nil { + return err + } + + return do(file, absd) }) if err != nil { return appError(fmt.Sprintf("in %s/%s", dir, file), err) @@ -113,7 +118,7 @@ func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error { return nil } -func (a *App) loadDesiredStateFromYaml(file string) (*state.HelmState, error) { +func (a *App) loadDesiredStateFromYaml(file string, opts ...LoadOpts) (*state.HelmState, error) { ld := &desiredStateLoader{ readFile: a.readFile, fileExists: a.fileExists, @@ -126,14 +131,20 @@ func (a *App) loadDesiredStateFromYaml(file string) (*state.HelmState, error) { KubeContext: a.KubeContext, glob: a.glob, } - return ld.Load(file) + + var op LoadOpts + if len(opts) > 0 { + op = opts[0] + } + + return ld.Load(file, op) } -func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error { +func (a *App) VisitDesiredStates(fileOrDir string, opts LoadOpts, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error { noMatchInHelmfiles := true - err := a.visitStateFiles(fileOrDir, func(f string) error { - st, err := a.loadDesiredStateFromYaml(f) + err := a.visitStateFiles(fileOrDir, func(f, d string) error { + st, err := a.loadDesiredStateFromYaml(f, opts) sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) @@ -169,16 +180,22 @@ func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge f return ctx.wrapErrs(err) } } - st.Selectors = selector + st.Selectors = opts.Selectors if len(st.Helmfiles) > 0 { noMatchInSubHelmfiles := true for i, m := range st.Helmfiles { + optsForNestedState := LoadOpts{ + CalleePath: filepath.Join(d, f), + Environment: m.Environment, + } //assign parent selector to sub helm selector in legacy mode or do not inherit in experimental mode if (m.Selectors == nil && !isExplicitSelectorInheritanceEnabled()) || m.SelectorsInherited { - m.Selectors = selector + optsForNestedState.Selectors = opts.Selectors + } else { + optsForNestedState.Selectors = m.Selectors } - if err := a.VisitDesiredStates(m.Path, m.Selectors, converge); err != nil { + if err := a.VisitDesiredStates(m.Path, optsForNestedState, converge); err != nil { switch err.(type) { case *NoMatchingHelmfileError: @@ -213,8 +230,9 @@ func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge f } func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) []error) error { + opts := LoadOpts{Selectors: a.Selectors} - err := a.VisitDesiredStates(fileOrDir, a.Selectors, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) { + err := a.VisitDesiredStates(fileOrDir, opts, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) { if len(st.Selectors) > 0 { err := st.FilterReleases() if err != nil { diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 568acd64..97c23c64 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -156,7 +156,7 @@ releases: t.Fatal("expected error did not occur") } - expected := "in ./helmfile.yaml: failed to read helmfile.yaml: environment values file matching \"env.*.yaml\" does not exist" + expected := "in ./helmfile.yaml: failed to read helmfile.yaml: environment values file matching \"env.*.yaml\" does not exist in \".\"" if err.Error() != expected { t.Errorf("unexpected error: expected=%s, got=%v", expected, err) } @@ -659,6 +659,122 @@ func runFilterSubHelmFilesTests(testcases []struct { } +func TestVisitDesiredStatesWithReleasesFiltered_EmbeddedNestedStateAdditionalEnvValues(t *testing.T) { + files := map[string]string{ + "/path/to/helmfile.yaml": ` +helmfiles: +- path: helmfile.d/a*.yaml + environment: + values: + - env.values.yaml +- helmfile.d/b*.yaml +- path: helmfile.d/c*.yaml + environment: + values: + - env.values.yaml + - tillerNs: INLINE_TILLER_NS_3 +`, + "/path/to/helmfile.d/a1.yaml": ` +environments: + default: + values: + - tillerNs: INLINE_TILLER_NS + ns: INLINE_NS +releases: +- name: foo + chart: stable/zipkin + tillerNamespace: {{ .Environment.Values.tillerNs }} + namespace: {{ .Environment.Values.ns }} +`, + "/path/to/helmfile.d/b.yaml": ` +environments: + default: + values: + - tillerNs: INLINE_TILLER_NS + ns: INLINE_NS +releases: +- name: bar + chart: stable/grafana + tillerNamespace: {{ .Environment.Values.tillerNs }} + namespace: {{ .Environment.Values.ns }} +`, + "/path/to/helmfile.d/c.yaml": ` +environments: + default: + values: + - tillerNs: INLINE_TILLER_NS + ns: INLINE_NS +releases: +- name: baz + chart: stable/envoy + tillerNamespace: {{ .Environment.Values.tillerNs }} + namespace: {{ .Environment.Values.ns }} +`, + "/path/to/env.values.yaml": ` +tillerNs: INLINE_TILLER_NS_2 +`, + } + + app := appWithFs(&App{ + KubeContext: "default", + Logger: helmexec.NewLogger(os.Stderr, "debug"), + Namespace: "", + Selectors: []string{}, + Env: "default", + }, files) + + processed := []state.ReleaseSpec{} + + collectReleases := func(st *state.HelmState, helm helmexec.Interface) []error { + for _, r := range st.Releases { + processed = append(processed, r) + } + return []error{} + } + + err := app.VisitDesiredStatesWithReleasesFiltered( + "helmfile.yaml", collectReleases, + ) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + type release struct { + chart string + tillerNs string + ns string + } + + expectedReleases := map[string]release{ + "foo": {"stable/zipkin", "INLINE_TILLER_NS_2", "INLINE_NS"}, + "bar": {"stable/grafana", "INLINE_TILLER_NS", "INLINE_NS"}, + "baz": {"stable/envoy", "INLINE_TILLER_NS_3", "INLINE_NS"}, + } + + for name := range processed { + actual := processed[name] + t.Run(actual.Name, func(t *testing.T) { + expected, ok := expectedReleases[actual.Name] + if !ok { + t.Fatalf("unexpected release processed: %v", actual) + } + + if expected.chart != actual.Chart { + t.Errorf("unexpected chart: expected=%s, got=%s", expected.chart, actual.Chart) + } + + if expected.tillerNs != actual.TillerNamespace { + t.Errorf("unexpected tiller namespace: expected=%s, got=%s", expected.tillerNs, actual.TillerNamespace) + } + + if expected.ns != actual.Namespace { + t.Errorf("unexpected namespace: expected=%s, got=%s", expected.ns, actual.Namespace) + } + }) + } +} + // See https://github.com/roboll/helmfile/issues/312 func TestVisitDesiredStatesWithReleasesFiltered_ReverseOrder(t *testing.T) { files := map[string]string{ @@ -897,25 +1013,23 @@ helmDefaults: if st.HelmDefaults.TillerNamespace != "TILLER_NS" { t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace) } - - if st.Releases[0].Name != "myrelease0" { - t.Errorf("unexpected releases[0].name: expected=myrelease0, got=%s", st.Releases[0].Name) + firstRelease := st.Releases[0] + if firstRelease.Name != "myrelease1" { + t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", firstRelease.Name) } - if st.Releases[1].Name != "myrelease1" { - t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", st.Releases[1].Name) + secondRelease := st.Releases[1] + if secondRelease.Name != "myrelease1" { + t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", secondRelease.Name) } - if st.Releases[2].Name != "myrelease1" { - t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", st.Releases[2].Name) + if secondRelease.Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { + t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", firstRelease.Values[0]) } - if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { - t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0]) - } - if *st.Releases[2].MissingFileHandler != "Warn" { - t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler) + if *secondRelease.MissingFileHandler != "Warn" { + t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *firstRelease.MissingFileHandler) } - if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { - t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0]) + if secondRelease.Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { + t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", firstRelease.Values[0]) } if st.HelmDefaults.KubeContext != "FOO" { @@ -1114,24 +1228,23 @@ helmDefaults: t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace) } - if st.Releases[0].Name != "myrelease0" { - t.Errorf("unexpected releases[0].name: expected=myrelease0, got=%s", st.Releases[0].Name) + firstRelease := st.Releases[0] + if firstRelease.Name != "myrelease1" { + t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", firstRelease.Name) } - if st.Releases[1].Name != "myrelease1" { - t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", st.Releases[1].Name) + secondRelease := st.Releases[1] + if secondRelease.Name != "myrelease1" { + t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", secondRelease.Name) } - if st.Releases[2].Name != "myrelease1" { - t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", st.Releases[2].Name) + if secondRelease.Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { + t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", firstRelease.Values[0]) } - if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { - t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0]) - } - if *st.Releases[2].MissingFileHandler != "Warn" { - t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler) + if *secondRelease.MissingFileHandler != "Warn" { + t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *firstRelease.MissingFileHandler) } - if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { - t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0]) + if secondRelease.Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { + t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", firstRelease.Values[0]) } if st.HelmDefaults.KubeContext != "FOO" { @@ -1188,10 +1301,58 @@ releases: if st.Releases[1].Name != "myrelease2" { t.Errorf("unexpected releases[0].name: expected=myrelease2, got=%s", st.Releases[1].Name) } - if st.Releases[2].Name != "myrelease1" { - t.Errorf("unexpected releases[0].name: expected=myrelease1, got=%s", st.Releases[2].Name) - } - if st.Releases[3].Name != "myrelease0" { - t.Errorf("unexpected releases[0].name: expected=myrelease0, got=%s", st.Releases[3].Name) + + if len(st.Releases) != 2 { + t.Errorf("unexpected number of releases: expected=2, got=%d", len(st.Releases)) + } +} + +// See https://github.com/roboll/helmfile/issues/615 +func TestLoadDesiredStateFromYaml_MultiPartTemplate_NoMergeArrayInEnvVal(t *testing.T) { + statePath := "/path/to/helmfile.yaml" + stateContent := ` +environments: + default: + values: + - foo: ["foo"] +--- +environments: + default: + values: + - foo: ["FOO"] + - 1.yaml +--- +environments: + default: + values: + - 2.yaml +--- +releases: +- name: {{ .Environment.Values.foo | quote }} + chart: {{ .Environment.Values.bar | quote }} +` + testFs := state.NewTestFs(map[string]string{ + statePath: stateContent, + "/path/to/1.yaml": `bar: ["bar"]`, + "/path/to/2.yaml": `bar: ["BAR"]`, + }) + app := &App{ + readFile: testFs.ReadFile, + glob: testFs.Glob, + abs: testFs.Abs, + Env: "default", + Logger: helmexec.NewLogger(os.Stderr, "debug"), + Reverse: true, + } + st, err := app.loadDesiredStateFromYaml(statePath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if st.Releases[0].Name != "[FOO]" { + t.Errorf("unexpected releases[0].name: expected=FOO, got=%s", st.Releases[0].Name) + } + if st.Releases[0].Chart != "[BAR]" { + t.Errorf("unexpected releases[0].chart: expected=BAR, got=%s", st.Releases[0].Chart) } } diff --git a/pkg/app/desired_state_file_loader.go b/pkg/app/desired_state_file_loader.go index 40eac2ec..862cde35 100644 --- a/pkg/app/desired_state_file_loader.go +++ b/pkg/app/desired_state_file_loader.go @@ -27,8 +27,36 @@ type desiredStateLoader struct { logger *zap.SugaredLogger } -func (ld *desiredStateLoader) Load(f string) (*state.HelmState, error) { - st, err := ld.loadFile(nil, filepath.Dir(f), filepath.Base(f), true) +type LoadOpts struct { + Selectors []string + Environment state.SubhelmfileEnvironmentSpec + CalleePath string +} + +func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, error) { + var overrodeEnv *environment.Environment + + args := opts.Environment.OverrideValues + + if len(args) > 0 { + if opts.CalleePath == "" { + return nil, fmt.Errorf("bug: opts.CalleePath was nil: f=%s, opts=%v", f, opts) + } + storage := state.NewStorage(opts.CalleePath, ld.logger, ld.glob) + envld := state.NewEnvironmentValuesLoader(storage, ld.readFile) + handler := state.MissingFileHandlerError + vals, err := envld.LoadEnvironmentValues(&handler, args) + if err != nil { + return nil, err + } + + overrodeEnv = &environment.Environment{ + Name: ld.env, + Values: vals, + } + } + + st, err := ld.loadFileWithOverrides(nil, overrodeEnv, filepath.Dir(f), filepath.Base(f), true) if err != nil { return nil, err } @@ -59,6 +87,9 @@ func (ld *desiredStateLoader) Load(f string) (*state.HelmState, error) { } func (ld *desiredStateLoader) loadFile(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) { + return ld.loadFileWithOverrides(inheritedEnv, nil, baseDir, file, evaluateBases) +} +func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) { var f string if filepath.IsAbs(file) { f = file @@ -78,6 +109,7 @@ func (ld *desiredStateLoader) loadFile(inheritedEnv *environment.Environment, ba if !experimentalModeEnabled() || ext == ".gotmpl" { self, err = ld.renderAndLoad( inheritedEnv, + overrodeEnv, baseDir, f, fileBytes, @@ -90,6 +122,7 @@ func (ld *desiredStateLoader) loadFile(inheritedEnv *environment.Environment, ba file, evaluateBases, inheritedEnv, + overrodeEnv, ) } @@ -102,33 +135,27 @@ func (a *desiredStateLoader) underlying() *state.StateCreator { return c } -func (a *desiredStateLoader) load(yaml []byte, baseDir, file string, evaluateBases bool, env *environment.Environment) (*state.HelmState, error) { - st, err := a.underlying().ParseAndLoad(yaml, baseDir, file, a.env, evaluateBases, env) +func (a *desiredStateLoader) load(yaml []byte, baseDir, file string, evaluateBases bool, env, overrodeEnv *environment.Environment) (*state.HelmState, error) { + merged, err := env.Merge(overrodeEnv) if err != nil { return nil, err } - helmfiles := []state.SubHelmfileSpec{} - for _, hf := range st.Helmfiles { - matches, err := st.ExpandPaths(hf.Path) - if err != nil { - return nil, err - } - if len(matches) == 0 { - return nil, fmt.Errorf("no file matching %s found", hf.Path) - } - for _, match := range matches { - newHelmfile := hf - newHelmfile.Path = match - helmfiles = append(helmfiles, newHelmfile) - } + st, err := a.underlying().ParseAndLoad(yaml, baseDir, file, a.env, evaluateBases, merged) + if err != nil { + return nil, err + } + + helmfiles, err := st.ExpandedHelmfiles() + if err != nil { + return nil, err } st.Helmfiles = helmfiles return st, nil } -func (ld *desiredStateLoader) renderAndLoad(env *environment.Environment, baseDir, filename string, content []byte, evaluateBases bool) (*state.HelmState, error) { +func (ld *desiredStateLoader) renderAndLoad(env, overrodeEnv *environment.Environment, baseDir, filename string, content []byte, evaluateBases bool) (*state.HelmState, error) { parts := bytes.Split(content, []byte("\n---\n")) var finalState *state.HelmState @@ -139,13 +166,13 @@ func (ld *desiredStateLoader) renderAndLoad(env *environment.Environment, baseDi id := fmt.Sprintf("%s.part.%d", filename, i) - if env == nil { + if env == nil && overrodeEnv == nil { yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part) if err != nil { return nil, fmt.Errorf("error during %s parsing: %v", id, err) } } else { - yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part, *env) + yamlBuf, err = ld.renderTemplatesToYamlWithEnv(baseDir, id, part, env, overrodeEnv) if err != nil { return nil, fmt.Errorf("error during %s parsing: %v", id, err) } @@ -157,6 +184,7 @@ func (ld *desiredStateLoader) renderAndLoad(env *environment.Environment, baseDi filename, evaluateBases, env, + overrodeEnv, ) if err != nil { return nil, err @@ -165,7 +193,7 @@ func (ld *desiredStateLoader) renderAndLoad(env *environment.Environment, baseDi if finalState == nil { finalState = currentState } else { - if err := mergo.Merge(finalState, currentState, mergo.WithAppendSlice); err != nil { + if err := mergo.Merge(finalState, currentState, mergo.WithOverride); err != nil { return nil, err } } diff --git a/pkg/app/two_pass_renderer.go b/pkg/app/two_pass_renderer.go index 26095842..60090d77 100644 --- a/pkg/app/two_pass_renderer.go +++ b/pkg/app/two_pass_renderer.go @@ -18,8 +18,8 @@ func prependLineNumbers(text string) string { return buf.String() } -func (r *desiredStateLoader) renderEnvironment(firstPassEnv environment.Environment, baseDir, filename string, content []byte) environment.Environment { - tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace} +func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) *environment.Environment { + tmplData := state.EnvironmentTemplateData{Environment: *firstPassEnv, Namespace: r.namespace} firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData) // parse as much as we can, tolerate errors, this is a preparse @@ -34,7 +34,7 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv environment.Environm c := r.underlying() c.Strict = false // create preliminary state, as we may have an environment. Tolerate errors. - prestate, err := c.ParseAndLoad(yamlBuf.Bytes(), baseDir, filename, r.env, false, &firstPassEnv) + prestate, err := c.ParseAndLoad(yamlBuf.Bytes(), baseDir, filename, r.env, false, firstPassEnv) if err != nil && r.logger != nil { switch err.(type) { case *state.StateLoadError: @@ -44,36 +44,55 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv environment.Environm } if prestate != nil { - firstPassEnv = prestate.Env + firstPassEnv = &prestate.Env } return firstPassEnv } -func (r *desiredStateLoader) renderTemplatesToYaml(baseDir, filename string, content []byte, context ...environment.Environment) (*bytes.Buffer, error) { - var env environment.Environment - - if len(context) > 0 { - env = context[0] - } else { - env = environment.Environment{Name: r.env, Values: map[string]interface{}(nil)} - } - - return r.twoPassRenderTemplateToYaml(env, baseDir, filename, content) +type RenderOpts struct { } -func (r *desiredStateLoader) twoPassRenderTemplateToYaml(initEnv environment.Environment, baseDir, filename string, content []byte) (*bytes.Buffer, error) { +func (r *desiredStateLoader) renderTemplatesToYaml(baseDir, filename string, content []byte) (*bytes.Buffer, error) { + env := &environment.Environment{Name: r.env, Values: map[string]interface{}(nil)} + + return r.renderTemplatesToYamlWithEnv(baseDir, filename, content, env, nil) +} + +func (r *desiredStateLoader) renderTemplatesToYamlWithEnv(baseDir, filename string, content []byte, inherited, overrode *environment.Environment) (*bytes.Buffer, error) { + return r.twoPassRenderTemplateToYaml(inherited, overrode, baseDir, filename, content) +} + +func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *environment.Environment, baseDir, filename string, content []byte) (*bytes.Buffer, error) { // try a first pass render. This will always succeed, but can produce a limited env if r.logger != nil { - r.logger.Debugf("first-pass rendering input of \"%s\": %v", filename, initEnv) + r.logger.Debugf("first-pass rendering starting for \"%s\": inherited=%v, overrode=%v", filename, inherited, overrode) } - firstPassEnv := r.renderEnvironment(initEnv, baseDir, filename, content) + initEnv, err := inherited.Merge(overrode) + if err != nil { + return nil, err + } if r.logger != nil { - r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, firstPassEnv) + r.logger.Debugf("first-pass uses: %v", initEnv) } - tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace} + renderedEnv := r.renderEnvironment(initEnv, baseDir, filename, content) + + if r.logger != nil { + r.logger.Debugf("first-pass produced: %v", initEnv) + } + + finalEnv, err := renderedEnv.Merge(overrode) + if err != nil { + return nil, err + } + + if r.logger != nil { + r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, *finalEnv) + } + + tmplData := state.EnvironmentTemplateData{Environment: *finalEnv, Namespace: r.namespace} secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData) yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content) if err != nil { diff --git a/state/create.go b/state/create.go index 299a5552..b2f48457 100644 --- a/state/create.go +++ b/state/create.go @@ -4,17 +4,13 @@ import ( "bytes" "errors" "fmt" - "io" - "os" - "path/filepath" - "sort" - "github.com/imdario/mergo" "github.com/roboll/helmfile/environment" "github.com/roboll/helmfile/helmexec" - "github.com/roboll/helmfile/tmpl" "go.uber.org/zap" "gopkg.in/yaml.v2" + "io" + "os" ) type StateLoadError struct { @@ -164,68 +160,19 @@ func (c *StateCreator) loadBases(envValues *environment.Environment, st *HelmSta return layers[0], nil } -func (st *HelmState) ExpandPaths(globPattern string) ([]string, error) { - result := []string{} - absPathPattern := st.normalizePath(globPattern) - matches, err := st.glob(absPathPattern) - if err != nil { - return nil, fmt.Errorf("failed processing %s: %v", globPattern, err) - } - - sort.Strings(matches) - - result = append(result, matches...) - return result, nil -} - func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, readFile func(string) ([]byte, error), glob func(string) ([]string, error)) (*environment.Environment, error) { envVals := map[string]interface{}{} envSpec, ok := st.Environments[name] if ok { - for _, v := range envSpec.Values { - switch typedValue := v.(type) { - case string: - urlOrPath := typedValue - resolved, skipped, err := st.resolveFile(envSpec.MissingFileHandler, "environment values", urlOrPath) - if err != nil { - return nil, err - } - if skipped { - continue - } - - for _, envvalFullPath := range resolved { - tmplData := EnvironmentTemplateData{Environment: environment.EmptyEnvironment, Namespace: ""} - r := tmpl.NewFileRenderer(readFile, filepath.Dir(envvalFullPath), tmplData) - bytes, err := r.RenderToBytes(envvalFullPath) - if err != nil { - return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", envvalFullPath, err) - } - m := map[string]interface{}{} - if err := yaml.Unmarshal(bytes, &m); err != nil { - return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", envvalFullPath, err) - } - if err := mergo.Merge(&envVals, &m, mergo.WithOverride); err != nil { - return nil, fmt.Errorf("failed to load \"%s\": %v", envvalFullPath, err) - } - } - case map[interface{}]interface{}: - m := map[string]interface{}{} - for k, v := range typedValue { - switch typedKey := k.(type) { - case string: - m[typedKey] = v - default: - return nil, fmt.Errorf("unexpected type of key in inline environment values %v: expected string, got %T", typedValue, typedKey) - } - } - if err := mergo.Merge(&envVals, &m, mergo.WithOverride); err != nil { - return nil, fmt.Errorf("failed to merge %v: %v", typedValue, err) - } - continue - default: - return nil, fmt.Errorf("unexpected type of values entry: %T", typedValue) - } + envValues := append([]interface{}{}, envSpec.Values...) + ld := &EnvironmentValuesLoader{ + storage: st.storage(), + readFile: st.readFile, + } + var err error + envVals, err = ld.LoadEnvironmentValues(envSpec.MissingFileHandler, envValues) + if err != nil { + return nil, err } if len(envSpec.Secrets) > 0 { @@ -233,7 +180,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, var envSecretFiles []string for _, urlOrPath := range envSpec.Secrets { - resolved, skipped, err := st.resolveFile(envSpec.MissingFileHandler, "environment values", urlOrPath) + resolved, skipped, err := st.storage().resolveFile(envSpec.MissingFileHandler, "environment values", urlOrPath) if err != nil { return nil, err } @@ -281,7 +228,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, if ctxEnv != nil { intEnv := *ctxEnv - if err := mergo.Merge(&intEnv, newEnv, mergo.WithAppendSlice); err != nil { + if err := mergo.Merge(&intEnv, newEnv, mergo.WithOverride); err != nil { return nil, fmt.Errorf("error while merging environment values for \"%s\": %v", name, err) } diff --git a/state/environment_values_loader.go b/state/environment_values_loader.go new file mode 100644 index 00000000..befecc51 --- /dev/null +++ b/state/environment_values_loader.go @@ -0,0 +1,75 @@ +package state + +import ( + "fmt" + "github.com/imdario/mergo" + "github.com/roboll/helmfile/environment" + "github.com/roboll/helmfile/tmpl" + "gopkg.in/yaml.v2" + "path/filepath" +) + +type EnvironmentValuesLoader struct { + storage *Storage + + readFile func(string) ([]byte, error) +} + +func NewEnvironmentValuesLoader(storage *Storage, readFile func(string) ([]byte, error)) *EnvironmentValuesLoader { + return &EnvironmentValuesLoader{ + storage: storage, + readFile: readFile, + } +} + +func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *string, envValues []interface{}) (map[string]interface{}, error) { + envVals := map[string]interface{}{} + + for _, v := range envValues { + switch typedValue := v.(type) { + case string: + urlOrPath := typedValue + resolved, skipped, err := ld.storage.resolveFile(missingFileHandler, "environment values", urlOrPath) + if err != nil { + return nil, err + } + if skipped { + continue + } + + for _, envvalFullPath := range resolved { + tmplData := EnvironmentTemplateData{Environment: environment.EmptyEnvironment, Namespace: ""} + r := tmpl.NewFileRenderer(ld.readFile, filepath.Dir(envvalFullPath), tmplData) + bytes, err := r.RenderToBytes(envvalFullPath) + if err != nil { + return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", envvalFullPath, err) + } + m := map[string]interface{}{} + if err := yaml.Unmarshal(bytes, &m); err != nil { + return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", envvalFullPath, err) + } + if err := mergo.Merge(&envVals, &m, mergo.WithOverride); err != nil { + return nil, fmt.Errorf("failed to load \"%s\": %v", envvalFullPath, err) + } + } + case map[interface{}]interface{}: + m := map[string]interface{}{} + for k, v := range typedValue { + switch typedKey := k.(type) { + case string: + m[typedKey] = v + default: + return nil, fmt.Errorf("unexpected type of key in inline environment values %v: expected string, got %T", typedValue, typedKey) + } + } + if err := mergo.Merge(&envVals, &m, mergo.WithOverride); err != nil { + return nil, fmt.Errorf("failed to merge %v: %v", typedValue, err) + } + continue + default: + return nil, fmt.Errorf("unexpected type of values entry: %T", typedValue) + } + } + + return envVals, nil +} diff --git a/state/state.go b/state/state.go index ba1ce682..8587ac6e 100644 --- a/state/state.go +++ b/state/state.go @@ -15,8 +15,6 @@ import ( "regexp" - "net/url" - "github.com/roboll/helmfile/environment" "github.com/roboll/helmfile/event" "github.com/roboll/helmfile/tmpl" @@ -63,6 +61,12 @@ type SubHelmfileSpec struct { Path string //path or glob pattern for the sub helmfiles Selectors []string //chosen selectors for the sub helmfiles SelectorsInherited bool //do the sub helmfiles inherits from parent selectors + + Environment SubhelmfileEnvironmentSpec +} + +type SubhelmfileEnvironmentSpec struct { + OverrideValues []interface{} `yaml:"values"` } // HelmSpec to defines helmDefault values @@ -1040,21 +1044,6 @@ func (st *HelmState) BuildDeps(helm helmexec.Interface) []error { return nil } -// JoinBase returns an absolute path in the form basePath/relative -func (st *HelmState) JoinBase(relPath string) string { - return filepath.Join(st.basePath, relPath) -} - -// normalizes relative path to absolute one -func (st *HelmState) normalizePath(path string) string { - u, _ := url.Parse(path) - if u.Scheme != "" || filepath.IsAbs(path) { - return path - } else { - return st.JoinBase(path) - } -} - // normalizeChart allows for the distinction between a file path reference and repository references. // - Any single (or double character) followed by a `/` will be considered a local file reference and // be constructed relative to the `base path`. @@ -1253,13 +1242,42 @@ func (st *HelmState) RenderValuesFileToBytes(path string) ([]byte, error) { return r.RenderToBytes(path) } +func (st *HelmState) storage() *Storage { + return &Storage{ + FilePath: st.FilePath, + basePath: st.basePath, + glob: st.glob, + logger: st.logger, + } +} + +func (st *HelmState) ExpandedHelmfiles() ([]SubHelmfileSpec, error) { + helmfiles := []SubHelmfileSpec{} + for _, hf := range st.Helmfiles { + matches, err := st.storage().ExpandPaths(hf.Path) + if err != nil { + return nil, err + } + if len(matches) == 0 { + return nil, fmt.Errorf("no file matching %s found", hf.Path) + } + for _, match := range matches { + newHelmfile := hf + newHelmfile.Path = match + helmfiles = append(helmfiles, newHelmfile) + } + } + + return helmfiles, nil +} + func (st *HelmState) generateTemporaryValuesFiles(values []interface{}, missingFileHandler *string) ([]string, error) { generatedFiles := []string{} for _, value := range values { switch typedValue := value.(type) { case string: - paths, skip, err := st.resolveFile(missingFileHandler, "values", typedValue) + paths, skip, err := st.storage().resolveFile(missingFileHandler, "values", typedValue) if err != nil { return nil, err } @@ -1317,7 +1335,7 @@ func (st *HelmState) namespaceAndValuesFlags(helm helmexec.Interface, release *R for _, v := range release.Values { switch typedValue := v.(type) { case string: - path := st.normalizePath(release.ValuesPathPrefix + typedValue) + path := st.storage().normalizePath(release.ValuesPathPrefix + typedValue) values = append(values, path) default: values = append(values, v) @@ -1336,7 +1354,7 @@ func (st *HelmState) namespaceAndValuesFlags(helm helmexec.Interface, release *R release.generatedValues = append(release.generatedValues, generatedFiles...) for _, value := range release.Secrets { - paths, skip, err := st.resolveFile(release.MissingFileHandler, "secrets", release.ValuesPathPrefix+value) + paths, skip, err := st.storage().resolveFile(release.MissingFileHandler, "secrets", release.ValuesPathPrefix+value) if err != nil { return nil, err } @@ -1363,7 +1381,7 @@ func (st *HelmState) namespaceAndValuesFlags(helm helmexec.Interface, release *R if set.Value != "" { flags = append(flags, "--set", fmt.Sprintf("%s=%s", escape(set.Name), escape(set.Value))) } else if set.File != "" { - flags = append(flags, "--set-file", fmt.Sprintf("%s=%s", escape(set.Name), st.normalizePath(set.File))) + flags = append(flags, "--set-file", fmt.Sprintf("%s=%s", escape(set.Name), st.storage().normalizePath(set.File))) } else if len(set.Values) > 0 { items := make([]string, len(set.Values)) for i, raw := range set.Values { @@ -1405,49 +1423,6 @@ func (st *HelmState) namespaceAndValuesFlags(helm helmexec.Interface, release *R return flags, nil } -func (st *HelmState) resolveFile(missingFileHandler *string, tpe, path string) ([]string, bool, error) { - title := fmt.Sprintf("%s file", tpe) - - files, err := st.ExpandPaths(path) - if err != nil { - return nil, false, err - } - - var handlerId string - - if missingFileHandler != nil { - handlerId = *missingFileHandler - } else { - handlerId = MissingFileHandlerError - } - - if len(files) == 0 { - switch handlerId { - case MissingFileHandlerError: - return nil, false, fmt.Errorf("%s matching \"%s\" does not exist", title, path) - case MissingFileHandlerWarn: - st.logger.Warnf("skipping missing %s matching \"%s\"", title, path) - return nil, true, nil - case MissingFileHandlerInfo: - st.logger.Infof("skipping missing %s matching \"%s\"", title, path) - return nil, true, nil - case MissingFileHandlerDebug: - st.logger.Debugf("skipping missing %s matching \"%s\"", title, path) - return nil, true, nil - default: - available := []string{ - MissingFileHandlerError, - MissingFileHandlerWarn, - MissingFileHandlerInfo, - MissingFileHandlerDebug, - } - return nil, false, fmt.Errorf("invalid missing file handler \"%s\" while processing \"%s\" in \"%s\": it must be one of %s", handlerId, path, st.FilePath, available) - } - } - - return files, false, nil -} - // DisplayAffectedReleases logs the upgraded, deleted and in error releases func (ar *AffectedReleases) DisplayAffectedReleases(logger *zap.SugaredLogger) { if ar.Upgraded != nil { @@ -1501,6 +1476,8 @@ func (hf *SubHelmfileSpec) UnmarshalYAML(unmarshal func(interface{}) error) erro Path string `yaml:"path"` Selectors []string `yaml:"selectors"` SelectorsInherited bool `yaml:"selectorsInherited"` + + Environment SubhelmfileEnvironmentSpec `yaml:"environment"` } if err := unmarshal(&subHelmfileSpecTmp); err != nil { return err @@ -1508,6 +1485,7 @@ func (hf *SubHelmfileSpec) UnmarshalYAML(unmarshal func(interface{}) error) erro hf.Path = subHelmfileSpecTmp.Path hf.Selectors = subHelmfileSpecTmp.Selectors hf.SelectorsInherited = subHelmfileSpecTmp.SelectorsInherited + hf.Environment = subHelmfileSpecTmp.Environment } //since we cannot make sur the "console" string can be red after the "path" we must check we don't have //a SubHelmfileSpec with only selector and no path diff --git a/state/state_test.go b/state/state_test.go index 8faa348d..0a11dd70 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -1001,7 +1001,7 @@ func TestHelmState_SyncReleases_MissingValuesFileForUndesiredRelease(t *testing. Values: []interface{}{"noexistent.values.yaml"}, }, listResult: ``, - expectedError: `failed processing release foo: values file matching "noexistent.values.yaml" does not exist`, + expectedError: `failed processing release foo: values file matching "noexistent.values.yaml" does not exist in "."`, }, { name: "should fail upgrading due to missing values file", @@ -1012,7 +1012,7 @@ func TestHelmState_SyncReleases_MissingValuesFileForUndesiredRelease(t *testing. }, listResult: `NAME REVISION UPDATED STATUS CHART APP VERSION NAMESPACE foo 1 Wed Apr 17 17:39:04 2019 DEPLOYED foo-bar-2.0.4 0.1.0 default`, - expectedError: `failed processing release foo: values file matching "noexistent.values.yaml" does not exist`, + expectedError: `failed processing release foo: values file matching "noexistent.values.yaml" does not exist in "."`, }, { name: "should uninstall even when there is a missing values file", @@ -1031,6 +1031,7 @@ func TestHelmState_SyncReleases_MissingValuesFileForUndesiredRelease(t *testing. tt := tests[i] t.Run(tt.name, func(t *testing.T) { state := &HelmState{ + basePath: ".", Releases: []ReleaseSpec{tt.release}, logger: logger, } diff --git a/state/storage.go b/state/storage.go new file mode 100644 index 00000000..4d73f899 --- /dev/null +++ b/state/storage.go @@ -0,0 +1,99 @@ +package state + +import ( + "fmt" + "go.uber.org/zap" + "net/url" + "path/filepath" + "sort" +) + +type Storage struct { + logger *zap.SugaredLogger + + FilePath string + + basePath string + glob func(string) ([]string, error) +} + +func NewStorage(forFile string, logger *zap.SugaredLogger, glob func(string) ([]string, error)) *Storage { + return &Storage{ + FilePath: forFile, + basePath: filepath.Dir(forFile), + logger: logger, + glob: glob, + } +} + +func (st *Storage) resolveFile(missingFileHandler *string, tpe, path string) ([]string, bool, error) { + title := fmt.Sprintf("%s file", tpe) + + files, err := st.ExpandPaths(path) + if err != nil { + return nil, false, err + } + + var handlerId string + + if missingFileHandler != nil { + handlerId = *missingFileHandler + } else { + handlerId = MissingFileHandlerError + } + + if len(files) == 0 { + switch handlerId { + case MissingFileHandlerError: + return nil, false, fmt.Errorf("%s matching \"%s\" does not exist in \"%s\"", title, path, st.basePath) + case MissingFileHandlerWarn: + st.logger.Warnf("skipping missing %s matching \"%s\"", title, path) + return nil, true, nil + case MissingFileHandlerInfo: + st.logger.Infof("skipping missing %s matching \"%s\"", title, path) + return nil, true, nil + case MissingFileHandlerDebug: + st.logger.Debugf("skipping missing %s matching \"%s\"", title, path) + return nil, true, nil + default: + available := []string{ + MissingFileHandlerError, + MissingFileHandlerWarn, + MissingFileHandlerInfo, + MissingFileHandlerDebug, + } + return nil, false, fmt.Errorf("invalid missing file handler \"%s\" while processing \"%s\" in \"%s\": it must be one of %s", handlerId, path, st.FilePath, available) + } + } + + return files, false, nil +} + +func (st *Storage) ExpandPaths(globPattern string) ([]string, error) { + result := []string{} + absPathPattern := st.normalizePath(globPattern) + matches, err := st.glob(absPathPattern) + if err != nil { + return nil, fmt.Errorf("failed processing %s: %v", globPattern, err) + } + + sort.Strings(matches) + + result = append(result, matches...) + return result, nil +} + +// normalizes relative path to absolute one +func (st *Storage) normalizePath(path string) string { + u, _ := url.Parse(path) + if u.Scheme != "" || filepath.IsAbs(path) { + return path + } else { + return st.JoinBase(path) + } +} + +// JoinBase returns an absolute path in the form basePath/relative +func (st *Storage) JoinBase(relPath string) string { + return filepath.Join(st.basePath, relPath) +}