public_git
/
helmfile
mirror of https://github.com/helmfile/helmfile.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: specify env values from the parent to the nested state (#622) 

* feat: specify env values from the parent to the nested state

Adds the `helmfiles[].environment.values` that accepts a mix of file pathes and inline dictes:

```yaml
helmfiles:
- path: path/to/nested/helmfile.yaml
  environment:
    values:
    - key1: val1
    - values.yaml
```

The values files are loaded in the context of the parent state file. For example, in case the above state file is located at `/path/to/helmfile.yaml`,
`values.yaml` is located at `/path/to/values.yaml` instead of `/path/to/nested/values.yaml`.

Resolves #523

* fix: multiple "bases" declarations yields duplicate releases

Fixes #615

* fix regression in double-rendering with env value overrides

The latest commit broke any state files like the below to NOT pass env value overrides at all:

```
helmfiles:
- path: nested/state.yaml
  environment:
    values:
    - overrides.yaml
```

This fixes the issue.
This commit is contained in:
KUOKA Yusuke 2019-05-29 19:08:51 +09:00 committed by GitHub
parent 681c866ce1
commit 1226ea6d1a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 611 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
README.md
View File

@ -142,7 +142,28 @@ releases:
wait: true
#
# Advanced Configuration: Helmfile Environments
# Advanced Configuration: Nested States
#
helmfiles:
- # Path to the helmfile state file being processed BEFORE releases in this state file
path: path/to/subhelmfile.yaml
# Label selector used for filtering releases in the nested state.
# For example, `name=prometheus` in this context is equivalent to processing the nested state like
# helmfile -f path/to/subhelmfile.yaml -l name=prometheus sync
selectors:
- name=prometheus
environment:
values:
# Environment values files merged into the nested state
- additiona.values.yaml
# Inline environment values merged into the nested state
- key1: val1
- # All the nested state files under `helmfiles:` is processed in the order of definition.
# So it can be used for preparation for your main `releases`. An example would be creating CRDs required by `reelases` in the parent state file.
path: path/to/mycrd.helmfile.yaml
#
# Advanced Configuration: Environments
#
# The list of environments managed by helmfile.
@ -703,19 +724,22 @@ Just run `helmfile sync` inside `myteam/`, and you are done.
All the files are sorted alphabetically per group = array item inside `helmfiles:`, so that you have granular control over ordering, too.
#### selectors
When composing helmfiles you can use selectors from the command line as well as explicit selectors inside the parent helmfile to filter the releases to be used.
```yaml
helmfiles:
- apps/*/helmfile.yaml
- path: apps/a-helmfile.yaml
selectors: # list of selectors
- name=prometheus
- tier=frontend
selectors: # list of selectors
- name=prometheus
- tier=frontend
- path: apps/b-helmfile.yaml # no selector, so all releases are used
selectors: []
selectors: []
- path: apps/c-helmfile.yaml # parent selector to be used or cli selector for the initial helmfile
selectorsInherited: true
selectorsInherited: true
```
* When a selector is specified, only this selector applies and the parents or CLI selectors are ignored.
* When not selector is specified there are 2 modes for the selector inheritance because we would like to change the current inheritance behavior (see [issue #344](https://github.com/roboll/helmfile/issues/344) ).
* Legacy mode, sub-helmfiles without selectors inherit selectors from their parent helmfile. The initial helmfiles inherit from the command line selectors.

2
cmd/cmd.go
View File

@ -25,7 +25,7 @@ func VisitAllDesiredStates(c *cli.Context, converge func(*state.HelmState, helme
return converge(st, helm, ctx)
}
err = a.VisitDesiredStates(fileOrDir, a.Selectors, convergeWithHelmBinary)
err = a.VisitDesiredStates(fileOrDir, app.LoadOpts{Selectors: a.Selectors}, convergeWithHelmBinary)
return toCliError(c, err)
}

37
environment/environment.go
View File

@ -1,8 +1,45 @@
package environment
import (
"encoding/json"
"github.com/imdario/mergo"
)
type Environment struct {
Name string
Values map[string]interface{}
}
var EmptyEnvironment Environment
func (e Environment) DeepCopy() Environment {
bytes, err := json.Marshal(e.Values)
if err != nil {
panic(err)
}
var values map[string]interface{}
if err := json.Unmarshal(bytes, &values); err != nil {
panic(err)
}
return Environment{
Name: e.Name,
Values: values,
}
}
func (e *Environment) Merge(other *Environment) (*Environment, error) {
if e == nil {
if other != nil {
copy := other.DeepCopy()
return &copy, nil
}
return nil, nil
}
copy := e.DeepCopy()
if other != nil {
if err := mergo.Merge(&copy, other, mergo.WithOverride); err != nil {
return nil, err
}
}
return &copy, nil
}

40
pkg/app/app.go
View File

@ -83,7 +83,7 @@ func (a *App) within(dir string, do func() error) error {
return appErr
}
func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error {
func (a *App) visitStateFiles(fileOrDir string, do func(string, string) error) error {
desiredStateFiles, err := a.findDesiredStateFiles(fileOrDir)
if err != nil {
return appError("", err)
@ -103,7 +103,12 @@ func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error {
a.Logger.Debugf("processing file \"%s\" in directory \"%s\"", file, dir)
err := a.within(dir, func() error {
return do(file)
absd, err := a.abs(dir)
if err != nil {
return err
}
return do(file, absd)
})
if err != nil {
return appError(fmt.Sprintf("in %s/%s", dir, file), err)
@ -113,7 +118,7 @@ func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error {
return nil
}
func (a *App) loadDesiredStateFromYaml(file string) (*state.HelmState, error) {
func (a *App) loadDesiredStateFromYaml(file string, opts ...LoadOpts) (*state.HelmState, error) {
ld := &desiredStateLoader{
readFile: a.readFile,
fileExists: a.fileExists,
@ -126,14 +131,20 @@ func (a *App) loadDesiredStateFromYaml(file string) (*state.HelmState, error) {
KubeContext: a.KubeContext,
glob: a.glob,
}
return ld.Load(file)
var op LoadOpts
if len(opts) > 0 {
op = opts[0]
}
return ld.Load(file, op)
}
func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error {
func (a *App) VisitDesiredStates(fileOrDir string, opts LoadOpts, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error {
noMatchInHelmfiles := true
err := a.visitStateFiles(fileOrDir, func(f string) error {
st, err := a.loadDesiredStateFromYaml(f)
err := a.visitStateFiles(fileOrDir, func(f, d string) error {
st, err := a.loadDesiredStateFromYaml(f, opts)
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
@ -169,16 +180,22 @@ func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge f
return ctx.wrapErrs(err)
}
}
st.Selectors = selector
st.Selectors = opts.Selectors
if len(st.Helmfiles) > 0 {
noMatchInSubHelmfiles := true
for i, m := range st.Helmfiles {
optsForNestedState := LoadOpts{
CalleePath: filepath.Join(d, f),
Environment: m.Environment,
}
//assign parent selector to sub helm selector in legacy mode or do not inherit in experimental mode
if (m.Selectors == nil && !isExplicitSelectorInheritanceEnabled()) || m.SelectorsInherited {
m.Selectors = selector
optsForNestedState.Selectors = opts.Selectors
} else {
optsForNestedState.Selectors = m.Selectors
}
if err := a.VisitDesiredStates(m.Path, m.Selectors, converge); err != nil {
if err := a.VisitDesiredStates(m.Path, optsForNestedState, converge); err != nil {
switch err.(type) {
case *NoMatchingHelmfileError:
@ -213,8 +230,9 @@ func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge f
}
func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) []error) error {
opts := LoadOpts{Selectors: a.Selectors}
err := a.VisitDesiredStates(fileOrDir, a.Selectors, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
err := a.VisitDesiredStates(fileOrDir, opts, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
if len(st.Selectors) > 0 {
err := st.FilterReleases()
if err != nil {

227
pkg/app/app_test.go
View File

@ -156,7 +156,7 @@ releases:
t.Fatal("expected error did not occur")
}
expected := "in ./helmfile.yaml: failed to read helmfile.yaml: environment values file matching \"env.*.yaml\" does not exist"
expected := "in ./helmfile.yaml: failed to read helmfile.yaml: environment values file matching \"env.*.yaml\" does not exist in \".\""
if err.Error() != expected {
t.Errorf("unexpected error: expected=%s, got=%v", expected, err)
}
@ -659,6 +659,122 @@ func runFilterSubHelmFilesTests(testcases []struct {
}
func TestVisitDesiredStatesWithReleasesFiltered_EmbeddedNestedStateAdditionalEnvValues(t *testing.T) {
files := map[string]string{
"/path/to/helmfile.yaml": `
helmfiles:
- path: helmfile.d/a*.yaml
environment:
values:
- env.values.yaml
- helmfile.d/b*.yaml
- path: helmfile.d/c*.yaml
environment:
values:
- env.values.yaml
- tillerNs: INLINE_TILLER_NS_3
`,
"/path/to/helmfile.d/a1.yaml": `
environments:
default:
values:
- tillerNs: INLINE_TILLER_NS
ns: INLINE_NS
releases:
- name: foo
chart: stable/zipkin
tillerNamespace: {{ .Environment.Values.tillerNs }}
namespace: {{ .Environment.Values.ns }}
`,
"/path/to/helmfile.d/b.yaml": `
environments:
default:
values:
- tillerNs: INLINE_TILLER_NS
ns: INLINE_NS
releases:
- name: bar
chart: stable/grafana
tillerNamespace: {{ .Environment.Values.tillerNs }}
namespace: {{ .Environment.Values.ns }}
`,
"/path/to/helmfile.d/c.yaml": `
environments:
default:
values:
- tillerNs: INLINE_TILLER_NS
ns: INLINE_NS
releases:
- name: baz
chart: stable/envoy
tillerNamespace: {{ .Environment.Values.tillerNs }}
namespace: {{ .Environment.Values.ns }}
`,
"/path/to/env.values.yaml": `
tillerNs: INLINE_TILLER_NS_2
`,
}
app := appWithFs(&App{
KubeContext: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
Namespace: "",
Selectors: []string{},
Env: "default",
}, files)
processed := []state.ReleaseSpec{}
collectReleases := func(st *state.HelmState, helm helmexec.Interface) []error {
for _, r := range st.Releases {
processed = append(processed, r)
}
return []error{}
}
err := app.VisitDesiredStatesWithReleasesFiltered(
"helmfile.yaml", collectReleases,
)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
type release struct {
chart string
tillerNs string
ns string
}
expectedReleases := map[string]release{
"foo": {"stable/zipkin", "INLINE_TILLER_NS_2", "INLINE_NS"},
"bar": {"stable/grafana", "INLINE_TILLER_NS", "INLINE_NS"},
"baz": {"stable/envoy", "INLINE_TILLER_NS_3", "INLINE_NS"},
}
for name := range processed {
actual := processed[name]
t.Run(actual.Name, func(t *testing.T) {
expected, ok := expectedReleases[actual.Name]
if !ok {
t.Fatalf("unexpected release processed: %v", actual)
}
if expected.chart != actual.Chart {
t.Errorf("unexpected chart: expected=%s, got=%s", expected.chart, actual.Chart)
}
if expected.tillerNs != actual.TillerNamespace {
t.Errorf("unexpected tiller namespace: expected=%s, got=%s", expected.tillerNs, actual.TillerNamespace)
}
if expected.ns != actual.Namespace {
t.Errorf("unexpected namespace: expected=%s, got=%s", expected.ns, actual.Namespace)
}
})
}
}
// See https://github.com/roboll/helmfile/issues/312
func TestVisitDesiredStatesWithReleasesFiltered_ReverseOrder(t *testing.T) {
files := map[string]string{
@ -897,25 +1013,23 @@ helmDefaults:
if st.HelmDefaults.TillerNamespace != "TILLER_NS" {
t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace)
}
if st.Releases[0].Name != "myrelease0" {
t.Errorf("unexpected releases[0].name: expected=myrelease0, got=%s", st.Releases[0].Name)
firstRelease := st.Releases[0]
if firstRelease.Name != "myrelease1" {
t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", firstRelease.Name)
}
if st.Releases[1].Name != "myrelease1" {
t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", st.Releases[1].Name)
secondRelease := st.Releases[1]
if secondRelease.Name != "myrelease1" {
t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", secondRelease.Name)
}
if st.Releases[2].Name != "myrelease1" {
t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", st.Releases[2].Name)
if secondRelease.Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", firstRelease.Values[0])
}
if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
}
if *st.Releases[2].MissingFileHandler != "Warn" {
t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler)
if *secondRelease.MissingFileHandler != "Warn" {
t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *firstRelease.MissingFileHandler)
}
if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
if secondRelease.Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", firstRelease.Values[0])
}
if st.HelmDefaults.KubeContext != "FOO" {
@ -1114,24 +1228,23 @@ helmDefaults:
t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace)
}
if st.Releases[0].Name != "myrelease0" {
t.Errorf("unexpected releases[0].name: expected=myrelease0, got=%s", st.Releases[0].Name)
firstRelease := st.Releases[0]
if firstRelease.Name != "myrelease1" {
t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", firstRelease.Name)
}
if st.Releases[1].Name != "myrelease1" {
t.Errorf("unexpected releases[1].name: expected=myrelease1, got=%s", st.Releases[1].Name)
secondRelease := st.Releases[1]
if secondRelease.Name != "myrelease1" {
t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", secondRelease.Name)
}
if st.Releases[2].Name != "myrelease1" {
t.Errorf("unexpected releases[2].name: expected=myrelease1, got=%s", st.Releases[2].Name)
if secondRelease.Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", firstRelease.Values[0])
}
if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
}
if *st.Releases[2].MissingFileHandler != "Warn" {
t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler)
if *secondRelease.MissingFileHandler != "Warn" {
t.Errorf("unexpected releases[2].missingFileHandler: expected=Warn, got=%s", *firstRelease.MissingFileHandler)
}
if st.Releases[2].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
if secondRelease.Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
t.Errorf("unexpected releases[2].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", firstRelease.Values[0])
}
if st.HelmDefaults.KubeContext != "FOO" {
@ -1188,10 +1301,58 @@ releases:
if st.Releases[1].Name != "myrelease2" {
t.Errorf("unexpected releases[0].name: expected=myrelease2, got=%s", st.Releases[1].Name)
}
if st.Releases[2].Name != "myrelease1" {
t.Errorf("unexpected releases[0].name: expected=myrelease1, got=%s", st.Releases[2].Name)
}
if st.Releases[3].Name != "myrelease0" {
t.Errorf("unexpected releases[0].name: expected=myrelease0, got=%s", st.Releases[3].Name)
if len(st.Releases) != 2 {
t.Errorf("unexpected number of releases: expected=2, got=%d", len(st.Releases))
}
}
// See https://github.com/roboll/helmfile/issues/615
func TestLoadDesiredStateFromYaml_MultiPartTemplate_NoMergeArrayInEnvVal(t *testing.T) {
statePath := "/path/to/helmfile.yaml"
stateContent := `
environments:
default:
values:
- foo: ["foo"]
---
environments:
default:
values:
- foo: ["FOO"]
- 1.yaml
---
environments:
default:
values:
- 2.yaml
---
releases:
- name: {{ .Environment.Values.foo | quote }}
chart: {{ .Environment.Values.bar | quote }}
`
testFs := state.NewTestFs(map[string]string{
statePath: stateContent,
"/path/to/1.yaml": `bar: ["bar"]`,
"/path/to/2.yaml": `bar: ["BAR"]`,
})
app := &App{
readFile: testFs.ReadFile,
glob: testFs.Glob,
abs: testFs.Abs,
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
Reverse: true,
}
st, err := app.loadDesiredStateFromYaml(statePath)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if st.Releases[0].Name != "[FOO]" {
t.Errorf("unexpected releases[0].name: expected=FOO, got=%s", st.Releases[0].Name)
}
if st.Releases[0].Chart != "[BAR]" {
t.Errorf("unexpected releases[0].chart: expected=BAR, got=%s", st.Releases[0].Chart)
}
}

72
pkg/app/desired_state_file_loader.go
View File

@ -27,8 +27,36 @@ type desiredStateLoader struct {
logger *zap.SugaredLogger
}
func (ld *desiredStateLoader) Load(f string) (*state.HelmState, error) {
st, err := ld.loadFile(nil, filepath.Dir(f), filepath.Base(f), true)
type LoadOpts struct {
Selectors []string
Environment state.SubhelmfileEnvironmentSpec
CalleePath string
}
func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, error) {
var overrodeEnv *environment.Environment
args := opts.Environment.OverrideValues
if len(args) > 0 {
if opts.CalleePath == "" {
return nil, fmt.Errorf("bug: opts.CalleePath was nil: f=%s, opts=%v", f, opts)
}
storage := state.NewStorage(opts.CalleePath, ld.logger, ld.glob)
envld := state.NewEnvironmentValuesLoader(storage, ld.readFile)
handler := state.MissingFileHandlerError
vals, err := envld.LoadEnvironmentValues(&handler, args)
if err != nil {
return nil, err
}
overrodeEnv = &environment.Environment{
Name: ld.env,
Values: vals,
}
}
st, err := ld.loadFileWithOverrides(nil, overrodeEnv, filepath.Dir(f), filepath.Base(f), true)
if err != nil {
return nil, err
}
@ -59,6 +87,9 @@ func (ld *desiredStateLoader) Load(f string) (*state.HelmState, error) {
}
func (ld *desiredStateLoader) loadFile(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
return ld.loadFileWithOverrides(inheritedEnv, nil, baseDir, file, evaluateBases)
}
func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
var f string
if filepath.IsAbs(file) {
f = file
@ -78,6 +109,7 @@ func (ld *desiredStateLoader) loadFile(inheritedEnv *environment.Environment, ba
if !experimentalModeEnabled() || ext == ".gotmpl" {
self, err = ld.renderAndLoad(
inheritedEnv,
overrodeEnv,
baseDir,
f,
fileBytes,
@ -90,6 +122,7 @@ func (ld *desiredStateLoader) loadFile(inheritedEnv *environment.Environment, ba
file,
evaluateBases,
inheritedEnv,
overrodeEnv,
)
}
@ -102,33 +135,27 @@ func (a *desiredStateLoader) underlying() *state.StateCreator {
return c
}
func (a *desiredStateLoader) load(yaml []byte, baseDir, file string, evaluateBases bool, env *environment.Environment) (*state.HelmState, error) {
st, err := a.underlying().ParseAndLoad(yaml, baseDir, file, a.env, evaluateBases, env)
func (a *desiredStateLoader) load(yaml []byte, baseDir, file string, evaluateBases bool, env, overrodeEnv *environment.Environment) (*state.HelmState, error) {
merged, err := env.Merge(overrodeEnv)
if err != nil {
return nil, err
}
helmfiles := []state.SubHelmfileSpec{}
for _, hf := range st.Helmfiles {
matches, err := st.ExpandPaths(hf.Path)
if err != nil {
return nil, err
}
if len(matches) == 0 {
return nil, fmt.Errorf("no file matching %s found", hf.Path)
}
for _, match := range matches {
newHelmfile := hf
newHelmfile.Path = match
helmfiles = append(helmfiles, newHelmfile)
}
st, err := a.underlying().ParseAndLoad(yaml, baseDir, file, a.env, evaluateBases, merged)
if err != nil {
return nil, err
}
helmfiles, err := st.ExpandedHelmfiles()
if err != nil {
return nil, err
}
st.Helmfiles = helmfiles
return st, nil
}
func (ld *desiredStateLoader) renderAndLoad(env *environment.Environment, baseDir, filename string, content []byte, evaluateBases bool) (*state.HelmState, error) {
func (ld *desiredStateLoader) renderAndLoad(env, overrodeEnv *environment.Environment, baseDir, filename string, content []byte, evaluateBases bool) (*state.HelmState, error) {
parts := bytes.Split(content, []byte("\n---\n"))
var finalState *state.HelmState
@ -139,13 +166,13 @@ func (ld *desiredStateLoader) renderAndLoad(env *environment.Environment, baseDi
id := fmt.Sprintf("%s.part.%d", filename, i)
if env == nil {
if env == nil && overrodeEnv == nil {
yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part)
if err != nil {
return nil, fmt.Errorf("error during %s parsing: %v", id, err)
}
} else {
yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part, *env)
yamlBuf, err = ld.renderTemplatesToYamlWithEnv(baseDir, id, part, env, overrodeEnv)
if err != nil {
return nil, fmt.Errorf("error during %s parsing: %v", id, err)
}
@ -157,6 +184,7 @@ func (ld *desiredStateLoader) renderAndLoad(env *environment.Environment, baseDi
filename,
evaluateBases,
env,
overrodeEnv,
)
if err != nil {
return nil, err
@ -165,7 +193,7 @@ func (ld *desiredStateLoader) renderAndLoad(env *environment.Environment, baseDi
if finalState == nil {
finalState = currentState
} else {
if err := mergo.Merge(finalState, currentState, mergo.WithAppendSlice); err != nil {
if err := mergo.Merge(finalState, currentState, mergo.WithOverride); err != nil {
return nil, err
}
}

57
pkg/app/two_pass_renderer.go
View File

@ -18,8 +18,8 @@ func prependLineNumbers(text string) string {
return buf.String()
}
func (r *desiredStateLoader) renderEnvironment(firstPassEnv environment.Environment, baseDir, filename string, content []byte) environment.Environment {
tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) *environment.Environment {
tmplData := state.EnvironmentTemplateData{Environment: *firstPassEnv, Namespace: r.namespace}
firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData)
// parse as much as we can, tolerate errors, this is a preparse
@ -34,7 +34,7 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv environment.Environm
c := r.underlying()
c.Strict = false
// create preliminary state, as we may have an environment. Tolerate errors.
prestate, err := c.ParseAndLoad(yamlBuf.Bytes(), baseDir, filename, r.env, false, &firstPassEnv)
prestate, err := c.ParseAndLoad(yamlBuf.Bytes(), baseDir, filename, r.env, false, firstPassEnv)
if err != nil && r.logger != nil {
switch err.(type) {
case *state.StateLoadError:
@ -44,36 +44,55 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv environment.Environm
}
if prestate != nil {
firstPassEnv = prestate.Env
firstPassEnv = &prestate.Env
}
return firstPassEnv
}
func (r *desiredStateLoader) renderTemplatesToYaml(baseDir, filename string, content []byte, context ...environment.Environment) (*bytes.Buffer, error) {
var env environment.Environment
if len(context) > 0 {
env = context[0]
} else {
env = environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
}
return r.twoPassRenderTemplateToYaml(env, baseDir, filename, content)
type RenderOpts struct {
}
func (r *desiredStateLoader) twoPassRenderTemplateToYaml(initEnv environment.Environment, baseDir, filename string, content []byte) (*bytes.Buffer, error) {
func (r *desiredStateLoader) renderTemplatesToYaml(baseDir, filename string, content []byte) (*bytes.Buffer, error) {
env := &environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
return r.renderTemplatesToYamlWithEnv(baseDir, filename, content, env, nil)
}
func (r *desiredStateLoader) renderTemplatesToYamlWithEnv(baseDir, filename string, content []byte, inherited, overrode *environment.Environment) (*bytes.Buffer, error) {
return r.twoPassRenderTemplateToYaml(inherited, overrode, baseDir, filename, content)
}
func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *environment.Environment, baseDir, filename string, content []byte) (*bytes.Buffer, error) {
// try a first pass render. This will always succeed, but can produce a limited env
if r.logger != nil {
r.logger.Debugf("first-pass rendering input of \"%s\": %v", filename, initEnv)
r.logger.Debugf("first-pass rendering starting for \"%s\": inherited=%v, overrode=%v", filename, inherited, overrode)
}
firstPassEnv := r.renderEnvironment(initEnv, baseDir, filename, content)
initEnv, err := inherited.Merge(overrode)
if err != nil {
return nil, err
}
if r.logger != nil {
r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, firstPassEnv)
r.logger.Debugf("first-pass uses: %v", initEnv)
}
tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
renderedEnv := r.renderEnvironment(initEnv, baseDir, filename, content)
if r.logger != nil {
r.logger.Debugf("first-pass produced: %v", initEnv)
}
finalEnv, err := renderedEnv.Merge(overrode)
if err != nil {
return nil, err
}
if r.logger != nil {
r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, *finalEnv)
}
tmplData := state.EnvironmentTemplateData{Environment: *finalEnv, Namespace: r.namespace}
secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData)
yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content)
if err != nil {

79
state/create.go
View File

@ -4,17 +4,13 @@ import (
"bytes"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"github.com/imdario/mergo"
"github.com/roboll/helmfile/environment"
"github.com/roboll/helmfile/helmexec"
"github.com/roboll/helmfile/tmpl"
"go.uber.org/zap"
"gopkg.in/yaml.v2"
"io"
"os"
)
type StateLoadError struct {
@ -164,68 +160,19 @@ func (c *StateCreator) loadBases(envValues *environment.Environment, st *HelmSta
return layers[0], nil
}
func (st *HelmState) ExpandPaths(globPattern string) ([]string, error) {
result := []string{}
absPathPattern := st.normalizePath(globPattern)
matches, err := st.glob(absPathPattern)
if err != nil {
return nil, fmt.Errorf("failed processing %s: %v", globPattern, err)
}
sort.Strings(matches)
result = append(result, matches...)
return result, nil
}
func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, readFile func(string) ([]byte, error), glob func(string) ([]string, error)) (*environment.Environment, error) {
envVals := map[string]interface{}{}
envSpec, ok := st.Environments[name]
if ok {
for _, v := range envSpec.Values {
switch typedValue := v.(type) {
case string:
urlOrPath := typedValue
resolved, skipped, err := st.resolveFile(envSpec.MissingFileHandler, "environment values", urlOrPath)
if err != nil {
return nil, err
}
if skipped {
continue
}
for _, envvalFullPath := range resolved {
tmplData := EnvironmentTemplateData{Environment: environment.EmptyEnvironment, Namespace: ""}
r := tmpl.NewFileRenderer(readFile, filepath.Dir(envvalFullPath), tmplData)
bytes, err := r.RenderToBytes(envvalFullPath)
if err != nil {
return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", envvalFullPath, err)
}
m := map[string]interface{}{}
if err := yaml.Unmarshal(bytes, &m); err != nil {
return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", envvalFullPath, err)
}
if err := mergo.Merge(&envVals, &m, mergo.WithOverride); err != nil {
return nil, fmt.Errorf("failed to load \"%s\": %v", envvalFullPath, err)
}
}
case map[interface{}]interface{}:
m := map[string]interface{}{}
for k, v := range typedValue {
switch typedKey := k.(type) {
case string:
m[typedKey] = v
default:
return nil, fmt.Errorf("unexpected type of key in inline environment values %v: expected string, got %T", typedValue, typedKey)
}
}
if err := mergo.Merge(&envVals, &m, mergo.WithOverride); err != nil {
return nil, fmt.Errorf("failed to merge %v: %v", typedValue, err)
}
continue
default:
return nil, fmt.Errorf("unexpected type of values entry: %T", typedValue)
}
envValues := append([]interface{}{}, envSpec.Values...)
ld := &EnvironmentValuesLoader{
storage: st.storage(),
readFile: st.readFile,
}
var err error
envVals, err = ld.LoadEnvironmentValues(envSpec.MissingFileHandler, envValues)
if err != nil {
return nil, err
}
if len(envSpec.Secrets) > 0 {
@ -233,7 +180,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
var envSecretFiles []string
for _, urlOrPath := range envSpec.Secrets {
resolved, skipped, err := st.resolveFile(envSpec.MissingFileHandler, "environment values", urlOrPath)
resolved, skipped, err := st.storage().resolveFile(envSpec.MissingFileHandler, "environment values", urlOrPath)
if err != nil {
return nil, err
}
@ -281,7 +228,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
if ctxEnv != nil {
intEnv := *ctxEnv
if err := mergo.Merge(&intEnv, newEnv, mergo.WithAppendSlice); err != nil {
if err := mergo.Merge(&intEnv, newEnv, mergo.WithOverride); err != nil {
return nil, fmt.Errorf("error while merging environment values for \"%s\": %v", name, err)
}

75
state/environment_values_loader.go Normal file
View File

@ -0,0 +1,75 @@
package state
import (
"fmt"
"github.com/imdario/mergo"
"github.com/roboll/helmfile/environment"
"github.com/roboll/helmfile/tmpl"
"gopkg.in/yaml.v2"
"path/filepath"
)
type EnvironmentValuesLoader struct {
storage *Storage
readFile func(string) ([]byte, error)
}
func NewEnvironmentValuesLoader(storage *Storage, readFile func(string) ([]byte, error)) *EnvironmentValuesLoader {
return &EnvironmentValuesLoader{
storage: storage,
readFile: readFile,
}
}
func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *string, envValues []interface{}) (map[string]interface{}, error) {
envVals := map[string]interface{}{}
for _, v := range envValues {
switch typedValue := v.(type) {
case string:
urlOrPath := typedValue
resolved, skipped, err := ld.storage.resolveFile(missingFileHandler, "environment values", urlOrPath)
if err != nil {
return nil, err
}
if skipped {
continue
}
for _, envvalFullPath := range resolved {
tmplData := EnvironmentTemplateData{Environment: environment.EmptyEnvironment, Namespace: ""}
r := tmpl.NewFileRenderer(ld.readFile, filepath.Dir(envvalFullPath), tmplData)
bytes, err := r.RenderToBytes(envvalFullPath)
if err != nil {
return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", envvalFullPath, err)
}
m := map[string]interface{}{}
if err := yaml.Unmarshal(bytes, &m); err != nil {
return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", envvalFullPath, err)
}
if err := mergo.Merge(&envVals, &m, mergo.WithOverride); err != nil {
return nil, fmt.Errorf("failed to load \"%s\": %v", envvalFullPath, err)
}
}
case map[interface{}]interface{}:
m := map[string]interface{}{}
for k, v := range typedValue {
switch typedKey := k.(type) {
case string:
m[typedKey] = v
default:
return nil, fmt.Errorf("unexpected type of key in inline environment values %v: expected string, got %T", typedValue, typedKey)
}
}
if err := mergo.Merge(&envVals, &m, mergo.WithOverride); err != nil {
return nil, fmt.Errorf("failed to merge %v: %v", typedValue, err)
}
continue
default:
return nil, fmt.Errorf("unexpected type of values entry: %T", typedValue)
}
}
return envVals, nil
}

106
state/state.go
View File

@ -15,8 +15,6 @@ import (
"regexp"
"net/url"
"github.com/roboll/helmfile/environment"
"github.com/roboll/helmfile/event"
"github.com/roboll/helmfile/tmpl"
@ -63,6 +61,12 @@ type SubHelmfileSpec struct {
Path string //path or glob pattern for the sub helmfiles
Selectors []string //chosen selectors for the sub helmfiles
SelectorsInherited bool //do the sub helmfiles inherits from parent selectors
Environment SubhelmfileEnvironmentSpec
}
type SubhelmfileEnvironmentSpec struct {
OverrideValues []interface{} `yaml:"values"`
}
// HelmSpec to defines helmDefault values
@ -1040,21 +1044,6 @@ func (st *HelmState) BuildDeps(helm helmexec.Interface) []error {
return nil
}
// JoinBase returns an absolute path in the form basePath/relative
func (st *HelmState) JoinBase(relPath string) string {
return filepath.Join(st.basePath, relPath)
}
// normalizes relative path to absolute one
func (st *HelmState) normalizePath(path string) string {
u, _ := url.Parse(path)
if u.Scheme != "" || filepath.IsAbs(path) {
return path
} else {
return st.JoinBase(path)
}
}
// normalizeChart allows for the distinction between a file path reference and repository references.
// - Any single (or double character) followed by a `/` will be considered a local file reference and
// be constructed relative to the `base path`.
@ -1253,13 +1242,42 @@ func (st *HelmState) RenderValuesFileToBytes(path string) ([]byte, error) {
return r.RenderToBytes(path)
}
func (st *HelmState) storage() *Storage {
return &Storage{
FilePath: st.FilePath,
basePath: st.basePath,
glob: st.glob,
logger: st.logger,
}
}
func (st *HelmState) ExpandedHelmfiles() ([]SubHelmfileSpec, error) {
helmfiles := []SubHelmfileSpec{}
for _, hf := range st.Helmfiles {
matches, err := st.storage().ExpandPaths(hf.Path)
if err != nil {
return nil, err
}
if len(matches) == 0 {
return nil, fmt.Errorf("no file matching %s found", hf.Path)
}
for _, match := range matches {
newHelmfile := hf
newHelmfile.Path = match
helmfiles = append(helmfiles, newHelmfile)
}
}
return helmfiles, nil
}
func (st *HelmState) generateTemporaryValuesFiles(values []interface{}, missingFileHandler *string) ([]string, error) {
generatedFiles := []string{}
for _, value := range values {
switch typedValue := value.(type) {
case string:
paths, skip, err := st.resolveFile(missingFileHandler, "values", typedValue)
paths, skip, err := st.storage().resolveFile(missingFileHandler, "values", typedValue)
if err != nil {
return nil, err
}
@ -1317,7 +1335,7 @@ func (st *HelmState) namespaceAndValuesFlags(helm helmexec.Interface, release *R
for _, v := range release.Values {
switch typedValue := v.(type) {
case string:
path := st.normalizePath(release.ValuesPathPrefix + typedValue)
path := st.storage().normalizePath(release.ValuesPathPrefix + typedValue)
values = append(values, path)
default:
values = append(values, v)
@ -1336,7 +1354,7 @@ func (st *HelmState) namespaceAndValuesFlags(helm helmexec.Interface, release *R
release.generatedValues = append(release.generatedValues, generatedFiles...)
for _, value := range release.Secrets {
paths, skip, err := st.resolveFile(release.MissingFileHandler, "secrets", release.ValuesPathPrefix+value)
paths, skip, err := st.storage().resolveFile(release.MissingFileHandler, "secrets", release.ValuesPathPrefix+value)
if err != nil {
return nil, err
}
@ -1363,7 +1381,7 @@ func (st *HelmState) namespaceAndValuesFlags(helm helmexec.Interface, release *R
if set.Value != "" {
flags = append(flags, "--set", fmt.Sprintf("%s=%s", escape(set.Name), escape(set.Value)))
} else if set.File != "" {
flags = append(flags, "--set-file", fmt.Sprintf("%s=%s", escape(set.Name), st.normalizePath(set.File)))
flags = append(flags, "--set-file", fmt.Sprintf("%s=%s", escape(set.Name), st.storage().normalizePath(set.File)))
} else if len(set.Values) > 0 {
items := make([]string, len(set.Values))
for i, raw := range set.Values {
@ -1405,49 +1423,6 @@ func (st *HelmState) namespaceAndValuesFlags(helm helmexec.Interface, release *R
return flags, nil
}
func (st *HelmState) resolveFile(missingFileHandler *string, tpe, path string) ([]string, bool, error) {
title := fmt.Sprintf("%s file", tpe)
files, err := st.ExpandPaths(path)
if err != nil {
return nil, false, err
}
var handlerId string
if missingFileHandler != nil {
handlerId = *missingFileHandler
} else {
handlerId = MissingFileHandlerError
}
if len(files) == 0 {
switch handlerId {
case MissingFileHandlerError:
return nil, false, fmt.Errorf("%s matching \"%s\" does not exist", title, path)
case MissingFileHandlerWarn:
st.logger.Warnf("skipping missing %s matching \"%s\"", title, path)
return nil, true, nil
case MissingFileHandlerInfo:
st.logger.Infof("skipping missing %s matching \"%s\"", title, path)
return nil, true, nil
case MissingFileHandlerDebug:
st.logger.Debugf("skipping missing %s matching \"%s\"", title, path)
return nil, true, nil
default:
available := []string{
MissingFileHandlerError,
MissingFileHandlerWarn,
MissingFileHandlerInfo,
MissingFileHandlerDebug,
}
return nil, false, fmt.Errorf("invalid missing file handler \"%s\" while processing \"%s\" in \"%s\": it must be one of %s", handlerId, path, st.FilePath, available)
}
}
return files, false, nil
}
// DisplayAffectedReleases logs the upgraded, deleted and in error releases
func (ar *AffectedReleases) DisplayAffectedReleases(logger *zap.SugaredLogger) {
if ar.Upgraded != nil {
@ -1501,6 +1476,8 @@ func (hf *SubHelmfileSpec) UnmarshalYAML(unmarshal func(interface{}) error) erro
Path string `yaml:"path"`
Selectors []string `yaml:"selectors"`
SelectorsInherited bool `yaml:"selectorsInherited"`
Environment SubhelmfileEnvironmentSpec `yaml:"environment"`
}
if err := unmarshal(&subHelmfileSpecTmp); err != nil {
return err
@ -1508,6 +1485,7 @@ func (hf *SubHelmfileSpec) UnmarshalYAML(unmarshal func(interface{}) error) erro
hf.Path = subHelmfileSpecTmp.Path
hf.Selectors = subHelmfileSpecTmp.Selectors
hf.SelectorsInherited = subHelmfileSpecTmp.SelectorsInherited
hf.Environment = subHelmfileSpecTmp.Environment
}
//since we cannot make sur the "console" string can be red after the "path" we must check we don't have
//a SubHelmfileSpec with only selector and no path

5
state/state_test.go
View File

@ -1001,7 +1001,7 @@ func TestHelmState_SyncReleases_MissingValuesFileForUndesiredRelease(t *testing.
Values: []interface{}{"noexistent.values.yaml"},
},
listResult: ``,
expectedError: `failed processing release foo: values file matching "noexistent.values.yaml" does not exist`,
expectedError: `failed processing release foo: values file matching "noexistent.values.yaml" does not exist in "."`,
},
{
name: "should fail upgrading due to missing values file",
@ -1012,7 +1012,7 @@ func TestHelmState_SyncReleases_MissingValuesFileForUndesiredRelease(t *testing.
},
listResult: `NAME REVISION UPDATED STATUS CHART APP VERSION NAMESPACE
foo 1 Wed Apr 17 17:39:04 2019 DEPLOYED foo-bar-2.0.4 0.1.0 default`,
expectedError: `failed processing release foo: values file matching "noexistent.values.yaml" does not exist`,
expectedError: `failed processing release foo: values file matching "noexistent.values.yaml" does not exist in "."`,
},
{
name: "should uninstall even when there is a missing values file",
@ -1031,6 +1031,7 @@ func TestHelmState_SyncReleases_MissingValuesFileForUndesiredRelease(t *testing.
tt := tests[i]
t.Run(tt.name, func(t *testing.T) {
state := &HelmState{
basePath: ".",
Releases: []ReleaseSpec{tt.release},
logger: logger,
}

99
state/storage.go Normal file
View File

@ -0,0 +1,99 @@
package state
import (
"fmt"
"go.uber.org/zap"
"net/url"
"path/filepath"
"sort"
)
type Storage struct {
logger *zap.SugaredLogger
FilePath string
basePath string
glob func(string) ([]string, error)
}
func NewStorage(forFile string, logger *zap.SugaredLogger, glob func(string) ([]string, error)) *Storage {
return &Storage{
FilePath: forFile,
basePath: filepath.Dir(forFile),
logger: logger,
glob: glob,
}
}
func (st *Storage) resolveFile(missingFileHandler *string, tpe, path string) ([]string, bool, error) {
title := fmt.Sprintf("%s file", tpe)
files, err := st.ExpandPaths(path)
if err != nil {
return nil, false, err
}
var handlerId string
if missingFileHandler != nil {
handlerId = *missingFileHandler
} else {
handlerId = MissingFileHandlerError
}
if len(files) == 0 {
switch handlerId {
case MissingFileHandlerError:
return nil, false, fmt.Errorf("%s matching \"%s\" does not exist in \"%s\"", title, path, st.basePath)
case MissingFileHandlerWarn:
st.logger.Warnf("skipping missing %s matching \"%s\"", title, path)
return nil, true, nil
case MissingFileHandlerInfo:
st.logger.Infof("skipping missing %s matching \"%s\"", title, path)
return nil, true, nil
case MissingFileHandlerDebug:
st.logger.Debugf("skipping missing %s matching \"%s\"", title, path)
return nil, true, nil
default:
available := []string{
MissingFileHandlerError,
MissingFileHandlerWarn,
MissingFileHandlerInfo,
MissingFileHandlerDebug,
}
return nil, false, fmt.Errorf("invalid missing file handler \"%s\" while processing \"%s\" in \"%s\": it must be one of %s", handlerId, path, st.FilePath, available)
}
}
return files, false, nil
}
func (st *Storage) ExpandPaths(globPattern string) ([]string, error) {
result := []string{}
absPathPattern := st.normalizePath(globPattern)
matches, err := st.glob(absPathPattern)
if err != nil {
return nil, fmt.Errorf("failed processing %s: %v", globPattern, err)
}
sort.Strings(matches)
result = append(result, matches...)
return result, nil
}
// normalizes relative path to absolute one
func (st *Storage) normalizePath(path string) string {
u, _ := url.Parse(path)
if u.Scheme != "" || filepath.IsAbs(path) {
return path
} else {
return st.JoinBase(path)
}
}
// JoinBase returns an absolute path in the form basePath/relative
func (st *Storage) JoinBase(relPath string) string {
return filepath.Join(st.basePath, relPath)
}