From 1db205de48ce6f7aac5a56732b6914af7a736744 Mon Sep 17 00:00:00 2001 From: Yusuke KUOKA Date: Mon, 13 May 2019 15:39:29 +0900 Subject: [PATCH 1/2] feat: "bases" for easier layerina This adds the new configuration key `baeses` to your helmfile.yaml files, so that you can layer them without the `readFile` template function, which was a bit unintuitive. Please see https://github.com/roboll/helmfile/issues/388#issuecomment-491710348 for more context --- docs/writing-helmfile.md | 8 +- pkg/app/app.go | 128 ++++++----------------- pkg/app/app_test.go | 101 +++++++++++++++++-- pkg/app/constants.go | 8 +- pkg/app/desired_state_file_loader.go | 145 +++++++++++++++++++++++++++ pkg/app/two_pass_renderer.go | 33 ++---- pkg/app/two_pass_renderer_test.go | 33 +++--- state/create.go | 20 +--- state/create_test.go | 14 ++- state/state.go | 8 +- 10 files changed, 330 insertions(+), 168 deletions(-) create mode 100644 pkg/app/desired_state_file_loader.go diff --git a/docs/writing-helmfile.md b/docs/writing-helmfile.md index 23602a77..2067577c 100644 --- a/docs/writing-helmfile.md +++ b/docs/writing-helmfile.md @@ -100,10 +100,10 @@ Use Layering to extract the common parts into a dedicated *library helmfile*s, s Let's assume that your `helmfile.yaml` looks like: ``` -{ readFile "commons.yaml" }} ---- -{{ readFile "environments.yaml" }} ---- +bases: +- commons.yaml +- environments.yaml + releases: - name: myapp chart: mychart diff --git a/pkg/app/app.go b/pkg/app/app.go index ad61ce7f..33511a80 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -7,6 +7,7 @@ import ( "os" "os/signal" "strings" + "syscall" "github.com/roboll/helmfile/helmexec" "github.com/roboll/helmfile/state" @@ -14,7 +15,6 @@ import ( "path/filepath" "sort" - "syscall" ) type App struct { @@ -111,34 +111,42 @@ func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error { return nil } +func (a *App) loadDesiredStateFromYaml(file string) (*state.HelmState, error) { + ld := &desiredStateLoader{ + readFile: a.readFile, + env: a.Env, + namespace: a.Namespace, + logger: a.Logger, + abs: a.abs, + + Reverse: a.Reverse, + KubeContext: a.KubeContext, + glob: a.glob, + } + return ld.Load(file) +} + func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error { noMatchInHelmfiles := true err := a.visitStateFiles(fileOrDir, func(f string) error { - content, err := a.readFile(f) - if err != nil { - return err - } - // render template, in two runs - r := &twoPassRenderer{ - reader: a.readFile, - env: a.Env, - namespace: a.Namespace, - filename: f, - logger: a.Logger, - abs: a.abs, - } - yamlBuf, err := r.renderTemplate(content) - if err != nil { - return fmt.Errorf("error during %s parsing: %v", f, err) - } + st, err := a.loadDesiredStateFromYaml(f) - st, err := a.loadDesiredStateFromYaml( - yamlBuf.Bytes(), - f, - a.Namespace, - a.Env, - ) + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-sigs + + errs := []error{fmt.Errorf("Received [%s] to shutdown ", sig)} + _ = context{a, st}.clean(errs) + // See http://tldp.org/LDP/abs/html/exitcodes.html + switch sig { + case syscall.SIGINT: + os.Exit(130) + case syscall.SIGTERM: + os.Exit(143) + } + }() ctx := context{a, st} @@ -313,78 +321,6 @@ func directoryExistsAt(path string) bool { return err == nil && fileInfo.Mode().IsDir() } -func (a *App) loadDesiredStateFromYaml(yaml []byte, file string, namespace string, env string) (*state.HelmState, error) { - c := state.NewCreator(a.Logger, a.readFile, a.abs) - st, err := c.CreateFromYaml(yaml, file, env) - if err != nil { - return nil, err - } - - helmfiles := []state.SubHelmfileSpec{} - for _, hf := range st.Helmfiles { - globPattern := hf.Path - var absPathPattern string - if filepath.IsAbs(globPattern) { - absPathPattern = globPattern - } else { - absPathPattern = st.JoinBase(globPattern) - } - matches, err := a.glob(absPathPattern) - if err != nil { - return nil, fmt.Errorf("failed processing %s: %v", globPattern, err) - } - sort.Strings(matches) - for _, match := range matches { - newHelmfile := hf - newHelmfile.Path = match - helmfiles = append(helmfiles, newHelmfile) - } - - } - st.Helmfiles = helmfiles - - if a.Reverse { - rev := func(i, j int) bool { - return j < i - } - sort.Slice(st.Releases, rev) - sort.Slice(st.Helmfiles, rev) - } - - if a.KubeContext != "" { - if st.HelmDefaults.KubeContext != "" { - log.Printf("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.") - os.Exit(1) - } - st.HelmDefaults.KubeContext = a.KubeContext - } - if namespace != "" { - if st.Namespace != "" { - log.Printf("err: Cannot use option --namespace and set attribute namespace.") - os.Exit(1) - } - st.Namespace = namespace - } - - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - go func() { - sig := <-sigs - - errs := []error{fmt.Errorf("Received [%s] to shutdown ", sig)} - _ = context{a, st}.clean(errs) - // See http://tldp.org/LDP/abs/html/exitcodes.html - switch sig { - case syscall.SIGINT: - os.Exit(130) - case syscall.SIGTERM: - os.Exit(143) - } - }() - - return st, nil -} - type Error struct { msg string diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 5335e317..14a6de63 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -2,7 +2,6 @@ package app import ( "fmt" - "io/ioutil" "os" "path/filepath" "reflect" @@ -83,7 +82,14 @@ func (f *testFs) readFile(filename string) ([]byte, error) { return []byte(str), nil } -func (f *testFs) glob(pattern string) ([]string, error) { +func (f *testFs) glob(relPattern string) ([]string, error) { + var pattern string + if relPattern[0] == '/' { + pattern = relPattern + } else { + pattern = filepath.Join(f.wd, relPattern) + } + matches := []string{} for name, _ := range f.files { matched, err := filepath.Match(pattern, name) @@ -95,7 +101,7 @@ func (f *testFs) glob(pattern string) ([]string, error) { } } if len(matches) == 0 { - return []string(nil), fmt.Errorf("no file matched: %s", pattern) + return []string(nil), fmt.Errorf("no file matched %s for files: %v", pattern, f.files) } return matches, nil } @@ -640,15 +646,98 @@ func TestLoadDesiredStateFromYaml_DuplicateReleaseName(t *testing.T) { labels: stage: post `) + readFile := func(filename string) ([]byte, error) { + if filename != yamlFile { + return nil, fmt.Errorf("unexpected filename: %s", filename) + } + return yamlContent, nil + } app := &App{ - readFile: ioutil.ReadFile, + readFile: readFile, glob: filepath.Glob, abs: filepath.Abs, KubeContext: "default", + Env: "default", Logger: helmexec.NewLogger(os.Stderr, "debug"), } - _, err := app.loadDesiredStateFromYaml(yamlContent, yamlFile, "default", "default") + _, err := app.loadDesiredStateFromYaml(yamlFile) if err != nil { - t.Error("unexpected error") + t.Errorf("unexpected error: %v", err) + } +} + +func TestLoadDesiredStateFromYaml_Bases(t *testing.T) { + yamlFile := "/path/to/yaml/file" + yamlContent := []byte(`bases: +- ../base.yaml +- ../base.gotmpl + +{{ readFile "templates.yaml" }} + +releases: +- name: myrelease1 + chart: mychart1 + labels: + stage: pre + foo: bar +- name: myrelease1 + chart: mychart2 + labels: + stage: post + <<: *default +`) + files := map[string][]byte{ + yamlFile: yamlContent, + "/path/to/base.yaml": []byte(`environments: + default: + values: + - environments/default/1.yaml +`), + "/path/to/yaml/environments/default/1.yaml": []byte(`foo: FOO`), + "/path/to/base.gotmpl": []byte(`environments: + default: + values: + - environments/default/2.yaml + +helmDefaults: + tillerNamespace: {{ .Environment.Values.tillerNs }} +`), +"/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`), +"/path/to/yaml/templates.yaml": []byte(`templates: + default: &default + missingFileHandler: Warn + values: ["` + "{{`" + `{{.Release.Name}}` + "`}}" + `/values.yaml"] +`), + } + readFile := func(filename string) ([]byte, error) { + content, ok := files[filename] + if !ok { + return nil, fmt.Errorf("unexpected filename: %s", filename) + } + return content, nil + } + app := &App{ + readFile: readFile, + glob: filepath.Glob, + abs: filepath.Abs, + KubeContext: "default", + Env: "default", + Logger: helmexec.NewLogger(os.Stderr, "debug"), + } + st, err := app.loadDesiredStateFromYaml(yamlFile) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if st.HelmDefaults.TillerNamespace != "TILLER_NS" { + t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace) + } + + if *st.Releases[1].MissingFileHandler != "Warn" { + t.Errorf("unexpected releases[0].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler) + } + + if st.Releases[1].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { + t.Errorf("unexpected releases[0].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0]) } } diff --git a/pkg/app/constants.go b/pkg/app/constants.go index 472c93aa..ddb1e29f 100644 --- a/pkg/app/constants.go +++ b/pkg/app/constants.go @@ -13,6 +13,10 @@ const ( ExperimentalSelectorExplicit = "explicit-selector-inheritance" // value to remove default selector inheritance to sub-helmfiles and use the explicit one ) -func isExplicitSelectorInheritanceEnabled() bool { - return os.Getenv(ExperimentalEnvVar) == "true" || strings.Contains(os.Getenv(ExperimentalEnvVar), ExperimentalSelectorExplicit) +func experimentalModeEnabled() bool { + return os.Getenv(ExperimentalEnvVar) == "true" +} + +func isExplicitSelectorInheritanceEnabled() bool { + return experimentalModeEnabled() || strings.Contains(os.Getenv(ExperimentalEnvVar), ExperimentalSelectorExplicit) } diff --git a/pkg/app/desired_state_file_loader.go b/pkg/app/desired_state_file_loader.go new file mode 100644 index 00000000..dda9e3e6 --- /dev/null +++ b/pkg/app/desired_state_file_loader.go @@ -0,0 +1,145 @@ +package app + +import ( + "fmt" + "github.com/imdario/mergo" + "github.com/roboll/helmfile/state" + "go.uber.org/zap" + "log" + "os" + "path/filepath" + "sort" +) + +type desiredStateLoader struct { + KubeContext string + Reverse bool + + env string + namespace string + + readFile func(string) ([]byte, error) + abs func(string) (string, error) + glob func(string) ([]string, error) + + logger *zap.SugaredLogger +} + +func (ld *desiredStateLoader) Load(f string) (*state.HelmState, error) { + return ld.load(filepath.Dir(f), filepath.Base(f), true) +} + +func (ld *desiredStateLoader) load(baseDir, file string, evaluateBases bool) (*state.HelmState, error) { + var f string + if filepath.IsAbs(file) { + f = file + } else { + f = filepath.Join(baseDir, file) + } + + fileBytes, err := ld.readFile(f) + if err != nil { + return nil, err + } + + ext := filepath.Ext(f) + + var yamlBytes []byte + if !experimentalModeEnabled() || ext == ".gotmpl" { + yamlBuf, err := ld.renderTemplateToYaml(baseDir, f, fileBytes) + if err != nil { + return nil, fmt.Errorf("error during %s parsing: %v", f, err) + } + yamlBytes = yamlBuf.Bytes() + } else { + yamlBytes = fileBytes + } + + self, err := ld.loadYaml( + yamlBytes, + baseDir, + file, + ) + + if err != nil { + return nil, err + } + + if !evaluateBases { + return self, nil + } + + layers := []*state.HelmState{} + for _, b := range self.Bases { + base, err := ld.load(baseDir, b, false) + if err != nil { + return nil, err + } + layers = append(layers, base) + } + layers = append(layers, self) + + for i := 1; i < len(layers); i++ { + if err := mergo.Merge(layers[0], layers[i], mergo.WithAppendSlice); err != nil { + return nil, err + } + } + + return layers[0], nil +} + +func (a *desiredStateLoader) loadYaml(yaml []byte, baseDir, file string) (*state.HelmState, error) { + c := state.NewCreator(a.logger, a.readFile, a.abs) + st, err := c.ParseAndLoadEnv(yaml, baseDir, file, a.env) + if err != nil { + return nil, err + } + + helmfiles := []state.SubHelmfileSpec{} + for _, hf := range st.Helmfiles { + globPattern := hf.Path + var absPathPattern string + if filepath.IsAbs(globPattern) { + absPathPattern = globPattern + } else { + absPathPattern = st.JoinBase(globPattern) + } + matches, err := a.glob(absPathPattern) + if err != nil { + return nil, fmt.Errorf("failed processing %s: %v", globPattern, err) + } + sort.Strings(matches) + for _, match := range matches { + newHelmfile := hf + newHelmfile.Path = match + helmfiles = append(helmfiles, newHelmfile) + } + + } + st.Helmfiles = helmfiles + + if a.Reverse { + rev := func(i, j int) bool { + return j < i + } + sort.Slice(st.Releases, rev) + sort.Slice(st.Helmfiles, rev) + } + + if a.KubeContext != "" { + if st.HelmDefaults.KubeContext != "" { + log.Printf("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.") + os.Exit(1) + } + st.HelmDefaults.KubeContext = a.KubeContext + } + if a.namespace != "" { + if st.Namespace != "" { + log.Printf("err: Cannot use option --namespace and set attribute namespace.") + os.Exit(1) + } + st.Namespace = a.namespace + } + + return st, nil +} diff --git a/pkg/app/two_pass_renderer.go b/pkg/app/two_pass_renderer.go index 4bfaec50..291f0a29 100644 --- a/pkg/app/two_pass_renderer.go +++ b/pkg/app/two_pass_renderer.go @@ -6,8 +6,6 @@ import ( "github.com/roboll/helmfile/environment" "github.com/roboll/helmfile/state" "github.com/roboll/helmfile/tmpl" - "go.uber.org/zap" - "path/filepath" "strings" ) @@ -20,39 +18,30 @@ func prependLineNumbers(text string) string { return buf.String() } -type twoPassRenderer struct { - reader func(string) ([]byte, error) - env string - namespace string - filename string - logger *zap.SugaredLogger - abs func(string) (string, error) -} - -func (r *twoPassRenderer) renderEnvironment(content []byte) environment.Environment { +func (r *desiredStateLoader) renderEnvironment(baseDir, filename string, content []byte) environment.Environment { firstPassEnv := environment.Environment{Name: r.env, Values: map[string]interface{}(nil)} tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace} - firstPassRenderer := tmpl.NewFirstPassRenderer(filepath.Dir(r.filename), tmplData) + firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData) // parse as much as we can, tolerate errors, this is a preparse yamlBuf, err := firstPassRenderer.RenderTemplateContentToBuffer(content) if err != nil && r.logger != nil { - r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content))) + r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", filename, prependLineNumbers(string(content))) if yamlBuf == nil { // we have a template syntax error, let the second parse report r.logger.Debugf("template syntax error: %v", err) return firstPassEnv } } - c := state.NewCreator(r.logger, r.reader, r.abs) + c := state.NewCreator(r.logger, r.readFile, r.abs) c.Strict = false // create preliminary state, as we may have an environment. Tolerate errors. - prestate, err := c.CreateFromYaml(yamlBuf.Bytes(), r.filename, r.env) + prestate, err := c.ParseAndLoadEnv(yamlBuf.Bytes(), baseDir, filename, r.env) if err != nil && r.logger != nil { switch err.(type) { case *state.StateLoadError: r.logger.Infof("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err) } - r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String())) + r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String())) } if prestate != nil { firstPassEnv = prestate.Env @@ -60,21 +49,21 @@ func (r *twoPassRenderer) renderEnvironment(content []byte) environment.Environm return firstPassEnv } -func (r *twoPassRenderer) renderTemplate(content []byte) (*bytes.Buffer, error) { +func (r *desiredStateLoader) renderTemplateToYaml(baseDir, filename string, content []byte) (*bytes.Buffer, error) { // try a first pass render. This will always succeed, but can produce a limited env - firstPassEnv := r.renderEnvironment(content) + firstPassEnv := r.renderEnvironment(baseDir, filename, content) tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace} - secondPassRenderer := tmpl.NewFileRenderer(r.reader, filepath.Dir(r.filename), tmplData) + secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData) yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content) if err != nil { if r.logger != nil { - r.logger.Debugf("second-pass rendering failed, input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content))) + r.logger.Debugf("second-pass rendering failed, input of \"%s\":\n%s", filename, prependLineNumbers(string(content))) } return nil, err } if r.logger != nil { - r.logger.Debugf("second-pass rendering result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String())) + r.logger.Debugf("second-pass rendering result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String())) } return yamlBuf, nil } diff --git a/pkg/app/two_pass_renderer_test.go b/pkg/app/two_pass_renderer_test.go index 917b6a41..bf8072fe 100644 --- a/pkg/app/two_pass_renderer_test.go +++ b/pkg/app/two_pass_renderer_test.go @@ -12,12 +12,11 @@ import ( "gopkg.in/yaml.v2" ) -func makeRenderer(readFile func(string) ([]byte, error), env string) *twoPassRenderer { - return &twoPassRenderer{ - reader: readFile, +func makeLoader(readFile func(string) ([]byte, error), env string) *desiredStateLoader { + return &desiredStateLoader{ + readFile: readFile, env: env, namespace: "namespace", - filename: "", logger: helmexec.NewLogger(os.Stdout, "debug"), abs: filepath.Abs, } @@ -51,8 +50,8 @@ releases: return []byte(""), nil } - r := makeRenderer(fileReader, "staging") - yamlBuf, err := r.renderTemplate(yamlContent) + r := makeLoader(fileReader, "staging") + yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -102,9 +101,9 @@ releases: return defaultValuesYaml, nil } - r := makeRenderer(fileReader, "staging") + r := makeLoader(fileReader, "staging") // test the double rendering - yamlBuf, err := r.renderTemplate(yamlContent) + yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -151,9 +150,9 @@ releases: return defaultValuesYaml, nil } - r := makeRenderer(fileReader, "staging") + r := makeLoader(fileReader, "staging") // test the double rendering - _, err := r.renderTemplate(yamlContent) + _, err := r.renderTemplateToYaml("", "", yamlContent) if !strings.Contains(err.Error(), "stringTemplate:8") { t.Fatalf("error should contain a stringTemplate error (reference to unknow key) %v", err) @@ -190,8 +189,8 @@ releases: return defaultValuesYamlGotmpl, nil } - r := makeRenderer(fileReader, "staging") - rendered, _ := r.renderTemplate(yamlContent) + r := makeLoader(fileReader, "staging") + rendered, _ := r.renderTemplateToYaml("", "", yamlContent) var state state.HelmState yaml.Unmarshal(rendered.Bytes(), &state) @@ -217,8 +216,8 @@ func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) { return defaultValuesYaml, nil } - r := makeRenderer(fileReader, "staging") - yamlBuf, err := r.renderTemplate(yamlContent) + r := makeLoader(fileReader, "staging") + yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -231,7 +230,7 @@ func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) { } } -func TestReadFromYaml_HelfileShouldBeResilentToTemplateErrors(t *testing.T) { +func TestReadFromYaml_HelmfileShouldBeResilentToTemplateErrors(t *testing.T) { yamlContent := []byte(`environments: staging: production: @@ -248,8 +247,8 @@ releases: return yamlContent, nil } - r := makeRenderer(fileReader, "staging") - _, err := r.renderTemplate(yamlContent) + r := makeLoader(fileReader, "staging") + _, err := r.renderTemplateToYaml("", "", yamlContent) if err == nil { t.Fatalf("wanted error, none returned") } diff --git a/state/create.go b/state/create.go index b2d2e4ef..b68b0c00 100644 --- a/state/create.go +++ b/state/create.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "os" "path/filepath" @@ -33,16 +32,6 @@ func (e *UndefinedEnvError) Error() string { return e.msg } -func createFromYaml(content []byte, file string, env string, logger *zap.SugaredLogger) (*HelmState, error) { - c := &creator{ - logger, - ioutil.ReadFile, - filepath.Abs, - true, - } - return c.CreateFromYaml(content, file, env) -} - type creator struct { logger *zap.SugaredLogger readFile func(string) ([]byte, error) @@ -60,15 +49,12 @@ func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error) } } -func (c *creator) CreateFromYaml(content []byte, file string, env string) (*HelmState, error) { +// Parses YAML into HelmState, while loading environment values files relative to the `cwd` +func (c *creator) ParseAndLoadEnv(content []byte, baseDir, file string, env string) (*HelmState, error) { var state HelmState - basePath, err := c.abs(filepath.Dir(file)) - if err != nil { - return nil, &StateLoadError{fmt.Sprintf("failed to read %s", file), err} - } state.FilePath = file - state.basePath = basePath + state.basePath = baseDir decoder := yaml.NewDecoder(bytes.NewReader(content)) if !c.Strict { diff --git a/state/create_test.go b/state/create_test.go index d99d9b62..9dfac579 100644 --- a/state/create_test.go +++ b/state/create_test.go @@ -2,6 +2,8 @@ package state import ( "fmt" + "go.uber.org/zap" + "io/ioutil" "path/filepath" "reflect" "testing" @@ -10,6 +12,16 @@ import ( "gotest.tools/assert/cmp" ) +func createFromYaml(content []byte, file string, env string, logger *zap.SugaredLogger) (*HelmState, error) { + c := &creator{ + logger, + ioutil.ReadFile, + filepath.Abs, + true, + } + return c.ParseAndLoadEnv(content, filepath.Dir(file), file, env) +} + func TestReadFromYaml(t *testing.T) { yamlFile := "example/path/to/yaml/file" yamlContent := []byte(`releases: @@ -101,7 +113,7 @@ bar: {{ readFile "bar.txt" }} return nil, fmt.Errorf("unexpected filename: %s", filename) } - state, err := NewCreator(logger, readFile, filepath.Abs).CreateFromYaml(yamlContent, yamlFile, "production") + state, err := NewCreator(logger, readFile, filepath.Abs).ParseAndLoadEnv(yamlContent, filepath.Dir(yamlFile), yamlFile, "production") if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/state/state.go b/state/state.go index e7fc5cb2..6a869faa 100644 --- a/state/state.go +++ b/state/state.go @@ -27,9 +27,11 @@ import ( // HelmState structure for the helmfile type HelmState struct { - basePath string - Environments map[string]EnvironmentSpec - FilePath string + basePath string + Environments map[string]EnvironmentSpec + FilePath string + + Bases []string `yaml:"bases"` HelmDefaults HelmSpec `yaml:"helmDefaults"` Helmfiles []SubHelmfileSpec `yaml:"helmfiles"` DeprecatedContext string `yaml:"context"` From aef366660bb1edd981ef6542c322494c6dc0cc89 Mon Sep 17 00:00:00 2001 From: Yusuke KUOKA Date: Mon, 13 May 2019 21:47:39 +0900 Subject: [PATCH 2/2] feat: split-render-merge helmfile.yaml parts This splits your helmfile.yaml by the YAML document separator "---" before evaluating go template expressions as outlined in https://github.com/roboll/helmfile/issues/388#issuecomment-491710348 --- docs/writing-helmfile.md | 9 +- pkg/app/app_test.go | 105 +++++++++++++++++++- pkg/app/desired_state_file_loader.go | 142 +++++++++++++++++++-------- pkg/app/two_pass_renderer.go | 40 +++++++- pkg/app/two_pass_renderer_test.go | 12 +-- 5 files changed, 251 insertions(+), 57 deletions(-) diff --git a/docs/writing-helmfile.md b/docs/writing-helmfile.md index 2067577c..e4eab93d 100644 --- a/docs/writing-helmfile.md +++ b/docs/writing-helmfile.md @@ -125,23 +125,26 @@ environments: production: ``` -At run time, template expressions in your `helmfile.yaml` are executed: +At run time, `bases` in your `helmfile.yaml` are evaluated to produce: ```yaml +# commons.yaml releases: - name: metricbaet chart: stable/metricbeat --- +# environments.yaml environments: development: production: --- +# helmfile.yaml releases: - name: myapp chart: mychart ``` -Resulting YAML documents are merged in the order of occurrence, +Finally the resulting YAML documents are merged in the order of occurrence, so that your `helmfile.yaml` becomes: ```yaml @@ -159,3 +162,5 @@ releases: Great! Now, repeat the above steps for each your `helmfile.yaml`, so that all your helmfiles becomes DRY. + +Please also see [the discussion in the issue 388](https://github.com/roboll/helmfile/issues/388#issuecomment-491710348) for more advanced layering examples. diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 14a6de63..52705cd9 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -702,8 +702,8 @@ releases: helmDefaults: tillerNamespace: {{ .Environment.Values.tillerNs }} `), -"/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`), -"/path/to/yaml/templates.yaml": []byte(`templates: + "/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`), + "/path/to/yaml/templates.yaml": []byte(`templates: default: &default missingFileHandler: Warn values: ["` + "{{`" + `{{.Release.Name}}` + "`}}" + `/values.yaml"] @@ -741,3 +741,104 @@ helmDefaults: t.Errorf("unexpected releases[0].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0]) } } + +func TestLoadDesiredStateFromYaml_MultiPartTemplate(t *testing.T) { + yamlFile := "/path/to/yaml/file" + yamlContent := []byte(`bases: +- ../base.yaml +--- +bases: +- ../base.gotmpl +--- +helmDefaults: + kubeContext: {{ .Environment.Values.foo }} +--- +releases: +- name: myrelease0 + chart: mychart0 +--- + +{{ readFile "templates.yaml" }} + +releases: +- name: myrelease1 + chart: mychart1 + labels: + stage: pre + foo: bar +- name: myrelease1 + chart: mychart2 + labels: + stage: post + <<: *default +`) + files := map[string][]byte{ + yamlFile: yamlContent, + "/path/to/base.yaml": []byte(`environments: + default: + values: + - environments/default/1.yaml +`), + "/path/to/yaml/environments/default/1.yaml": []byte(`foo: FOO`), + "/path/to/base.gotmpl": []byte(`environments: + default: + values: + - environments/default/2.yaml + +helmDefaults: + tillerNamespace: {{ .Environment.Values.tillerNs }} +`), + "/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`), + "/path/to/yaml/templates.yaml": []byte(`templates: + default: &default + missingFileHandler: Warn + values: ["` + "{{`" + `{{.Release.Name}}` + "`}}" + `/values.yaml"] +`), + } + readFile := func(filename string) ([]byte, error) { + content, ok := files[filename] + if !ok { + return nil, fmt.Errorf("unexpected filename: %s", filename) + } + return content, nil + } + app := &App{ + readFile: readFile, + glob: filepath.Glob, + abs: filepath.Abs, + Env: "default", + Logger: helmexec.NewLogger(os.Stderr, "debug"), + } + st, err := app.loadDesiredStateFromYaml(yamlFile) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if st.HelmDefaults.TillerNamespace != "TILLER_NS" { + t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace) + } + + if st.Releases[0].Name != "myrelease0" { + t.Errorf("unexpected releases[0].name: expected=myrelease0, got=%s", st.Releases[0].Name) + } + if st.Releases[1].Name != "myrelease1" { + t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", st.Releases[1].Name) + } + if st.Releases[2].Name != "myrelease1" { + t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", st.Releases[2].Name) + } + if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { + t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0]) + } + if *st.Releases[2].MissingFileHandler != "Warn" { + t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler) + } + + if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" { + t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0]) + } + + if st.HelmDefaults.KubeContext != "FOO" { + t.Errorf("unexpected helmDefaults.kubeContext: expected=FOO, got=%s", st.HelmDefaults.KubeContext) + } +} diff --git a/pkg/app/desired_state_file_loader.go b/pkg/app/desired_state_file_loader.go index dda9e3e6..c8010d6a 100644 --- a/pkg/app/desired_state_file_loader.go +++ b/pkg/app/desired_state_file_loader.go @@ -1,8 +1,11 @@ package app import ( + "bytes" + "errors" "fmt" "github.com/imdario/mergo" + "github.com/roboll/helmfile/environment" "github.com/roboll/helmfile/state" "go.uber.org/zap" "log" @@ -26,10 +29,10 @@ type desiredStateLoader struct { } func (ld *desiredStateLoader) Load(f string) (*state.HelmState, error) { - return ld.load(filepath.Dir(f), filepath.Base(f), true) + return ld.loadFile(filepath.Dir(f), filepath.Base(f), true) } -func (ld *desiredStateLoader) load(baseDir, file string, evaluateBases bool) (*state.HelmState, error) { +func (ld *desiredStateLoader) loadFile(baseDir, file string, evaluateBases bool) (*state.HelmState, error) { var f string if filepath.IsAbs(file) { f = file @@ -44,51 +47,28 @@ func (ld *desiredStateLoader) load(baseDir, file string, evaluateBases bool) (*s ext := filepath.Ext(f) - var yamlBytes []byte + var self *state.HelmState + if !experimentalModeEnabled() || ext == ".gotmpl" { - yamlBuf, err := ld.renderTemplateToYaml(baseDir, f, fileBytes) - if err != nil { - return nil, fmt.Errorf("error during %s parsing: %v", f, err) - } - yamlBytes = yamlBuf.Bytes() + self, err = ld.renderAndLoad( + baseDir, + f, + fileBytes, + evaluateBases, + ) } else { - yamlBytes = fileBytes + self, err = ld.load( + fileBytes, + baseDir, + file, + evaluateBases, + ) } - self, err := ld.loadYaml( - yamlBytes, - baseDir, - file, - ) - - if err != nil { - return nil, err - } - - if !evaluateBases { - return self, nil - } - - layers := []*state.HelmState{} - for _, b := range self.Bases { - base, err := ld.load(baseDir, b, false) - if err != nil { - return nil, err - } - layers = append(layers, base) - } - layers = append(layers, self) - - for i := 1; i < len(layers); i++ { - if err := mergo.Merge(layers[0], layers[i], mergo.WithAppendSlice); err != nil { - return nil, err - } - } - - return layers[0], nil + return self, err } -func (a *desiredStateLoader) loadYaml(yaml []byte, baseDir, file string) (*state.HelmState, error) { +func (a *desiredStateLoader) load(yaml []byte, baseDir, file string, evaluateBases bool) (*state.HelmState, error) { c := state.NewCreator(a.logger, a.readFile, a.abs) st, err := c.ParseAndLoadEnv(yaml, baseDir, file, a.env) if err != nil { @@ -141,5 +121,83 @@ func (a *desiredStateLoader) loadYaml(yaml []byte, baseDir, file string) (*state st.Namespace = a.namespace } - return st, nil + if err != nil { + return nil, err + } + + if !evaluateBases { + if len(st.Bases) > 0 { + return nil, errors.New("nested `base` helmfile is unsupported. please submit a feature request if you need this!") + } + + return st, nil + } + + layers := []*state.HelmState{} + for _, b := range st.Bases { + base, err := a.loadFile(baseDir, b, false) + if err != nil { + return nil, err + } + layers = append(layers, base) + } + layers = append(layers, st) + + for i := 1; i < len(layers); i++ { + if err := mergo.Merge(layers[0], layers[i], mergo.WithAppendSlice); err != nil { + return nil, err + } + } + + return layers[0], nil +} + +func (ld *desiredStateLoader) renderAndLoad(baseDir, filename string, content []byte, evaluateBases bool) (*state.HelmState, error) { + parts := bytes.Split(content, []byte("\n---\n")) + + var finalState *state.HelmState + var env *environment.Environment + + for i, part := range parts { + var yamlBuf *bytes.Buffer + var err error + + id := fmt.Sprintf("%s.part.%d", filename, i) + + if env == nil { + yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part) + if err != nil { + return nil, fmt.Errorf("error during %s parsing: %v", id, err) + } + } else { + yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part, *env) + if err != nil { + return nil, fmt.Errorf("error during %s parsing: %v", id, err) + } + } + + currentState, err := ld.load( + yamlBuf.Bytes(), + baseDir, + filename, + evaluateBases, + ) + if err != nil { + return nil, err + } + + if finalState == nil { + finalState = currentState + } else { + if err := mergo.Merge(finalState, currentState, mergo.WithAppendSlice); err != nil { + return nil, err + } + } + + env = &finalState.Env + + ld.logger.Debugf("merged environment: %v", env) + } + + return finalState, nil } diff --git a/pkg/app/two_pass_renderer.go b/pkg/app/two_pass_renderer.go index 291f0a29..8c372b4b 100644 --- a/pkg/app/two_pass_renderer.go +++ b/pkg/app/two_pass_renderer.go @@ -3,6 +3,7 @@ package app import ( "bytes" "fmt" + "github.com/imdario/mergo" "github.com/roboll/helmfile/environment" "github.com/roboll/helmfile/state" "github.com/roboll/helmfile/tmpl" @@ -18,8 +19,7 @@ func prependLineNumbers(text string) string { return buf.String() } -func (r *desiredStateLoader) renderEnvironment(baseDir, filename string, content []byte) environment.Environment { - firstPassEnv := environment.Environment{Name: r.env, Values: map[string]interface{}(nil)} +func (r *desiredStateLoader) renderEnvironment(firstPassEnv environment.Environment, baseDir, filename string, content []byte) environment.Environment { tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace} firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData) @@ -43,15 +43,45 @@ func (r *desiredStateLoader) renderEnvironment(baseDir, filename string, content } r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String())) } + if prestate != nil { - firstPassEnv = prestate.Env + intEnv := environment.Environment{Name: firstPassEnv.Name} + if err := mergo.Merge(&intEnv, &firstPassEnv, mergo.WithAppendSlice); err != nil { + r.logger.Debugf("error in first-pass rendering: result of \"%s\": %v", filename, err) + return firstPassEnv + } + if err := mergo.Merge(&intEnv, &prestate.Env, mergo.WithAppendSlice); err != nil { + r.logger.Debugf("error in first-pass rendering: result of \"%s\": %v", filename, err) + return firstPassEnv + } + firstPassEnv = intEnv } return firstPassEnv } -func (r *desiredStateLoader) renderTemplateToYaml(baseDir, filename string, content []byte) (*bytes.Buffer, error) { +func (r *desiredStateLoader) renderTemplatesToYaml(baseDir, filename string, content []byte, context ...environment.Environment) (*bytes.Buffer, error) { + var env environment.Environment + + if len(context) > 0 { + env = context[0] + } else { + env = environment.Environment{Name: r.env, Values: map[string]interface{}(nil)} + } + + return r.twoPassRenderTemplateToYaml(env, baseDir, filename, content) +} + +func (r *desiredStateLoader) twoPassRenderTemplateToYaml(initEnv environment.Environment, baseDir, filename string, content []byte) (*bytes.Buffer, error) { // try a first pass render. This will always succeed, but can produce a limited env - firstPassEnv := r.renderEnvironment(baseDir, filename, content) + if r.logger != nil { + r.logger.Debugf("first-pass rendering input of \"%s\": %v", filename, initEnv) + } + + firstPassEnv := r.renderEnvironment(initEnv, baseDir, filename, content) + + if r.logger != nil { + r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, firstPassEnv) + } tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace} secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData) diff --git a/pkg/app/two_pass_renderer_test.go b/pkg/app/two_pass_renderer_test.go index bf8072fe..e042c1c4 100644 --- a/pkg/app/two_pass_renderer_test.go +++ b/pkg/app/two_pass_renderer_test.go @@ -51,7 +51,7 @@ releases: } r := makeLoader(fileReader, "staging") - yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent) + yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -103,7 +103,7 @@ releases: r := makeLoader(fileReader, "staging") // test the double rendering - yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent) + yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -152,7 +152,7 @@ releases: r := makeLoader(fileReader, "staging") // test the double rendering - _, err := r.renderTemplateToYaml("", "", yamlContent) + _, err := r.renderTemplatesToYaml("", "", yamlContent) if !strings.Contains(err.Error(), "stringTemplate:8") { t.Fatalf("error should contain a stringTemplate error (reference to unknow key) %v", err) @@ -190,7 +190,7 @@ releases: } r := makeLoader(fileReader, "staging") - rendered, _ := r.renderTemplateToYaml("", "", yamlContent) + rendered, _ := r.renderTemplatesToYaml("", "", yamlContent) var state state.HelmState yaml.Unmarshal(rendered.Bytes(), &state) @@ -217,7 +217,7 @@ func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) { } r := makeLoader(fileReader, "staging") - yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent) + yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -248,7 +248,7 @@ releases: } r := makeLoader(fileReader, "staging") - _, err := r.renderTemplateToYaml("", "", yamlContent) + _, err := r.renderTemplatesToYaml("", "", yamlContent) if err == nil { t.Fatalf("wanted error, none returned") }