public_git
/
helmfile
mirror of https://github.com/helmfile/helmfile.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: "bases" for easier layerina 

This adds the new configuration key `baeses` to your helmfile.yaml files, so that you can layer them without the `readFile` template function, which was a bit unintuitive.

Please see https://github.com/roboll/helmfile/issues/388#issuecomment-491710348 for more context
This commit is contained in:
Yusuke KUOKA 2019-05-13 15:39:29 +09:00
parent 4f83e69bf6
commit 1db205de48
10 changed files with 330 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/writing-helmfile.md
View File

@ -100,10 +100,10 @@ Use Layering to extract the common parts into a dedicated *library helmfile*s, s
Let's assume that your `helmfile.yaml` looks like:
```
{ readFile "commons.yaml" }}
---
{{ readFile "environments.yaml" }}
---
bases:
- commons.yaml
- environments.yaml
releases:
- name: myapp
chart: mychart

128
pkg/app/app.go
View File

@ -7,6 +7,7 @@ import (
"os"
"os/signal"
"strings"
"syscall"
"github.com/roboll/helmfile/helmexec"
"github.com/roboll/helmfile/state"
@ -14,7 +15,6 @@ import (
"path/filepath"
"sort"
"syscall"
)
type App struct {
@ -111,34 +111,42 @@ func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error {
return nil
}
func (a *App) loadDesiredStateFromYaml(file string) (*state.HelmState, error) {
ld := &desiredStateLoader{
readFile: a.readFile,
env: a.Env,
namespace: a.Namespace,
logger: a.Logger,
abs: a.abs,
Reverse: a.Reverse,
KubeContext: a.KubeContext,
glob: a.glob,
}
return ld.Load(file)
}
func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error {
noMatchInHelmfiles := true
err := a.visitStateFiles(fileOrDir, func(f string) error {
content, err := a.readFile(f)
if err != nil {
return err
}
// render template, in two runs
r := &twoPassRenderer{
reader: a.readFile,
env: a.Env,
namespace: a.Namespace,
filename: f,
logger: a.Logger,
abs: a.abs,
}
yamlBuf, err := r.renderTemplate(content)
if err != nil {
return fmt.Errorf("error during %s parsing: %v", f, err)
}
st, err := a.loadDesiredStateFromYaml(f)
st, err := a.loadDesiredStateFromYaml(
yamlBuf.Bytes(),
f,
a.Namespace,
a.Env,
)
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-sigs
errs := []error{fmt.Errorf("Received [%s] to shutdown ", sig)}
_ = context{a, st}.clean(errs)
// See http://tldp.org/LDP/abs/html/exitcodes.html
switch sig {
case syscall.SIGINT:
os.Exit(130)
case syscall.SIGTERM:
os.Exit(143)
}
}()
ctx := context{a, st}
@ -313,78 +321,6 @@ func directoryExistsAt(path string) bool {
return err == nil && fileInfo.Mode().IsDir()
}
func (a *App) loadDesiredStateFromYaml(yaml []byte, file string, namespace string, env string) (*state.HelmState, error) {
c := state.NewCreator(a.Logger, a.readFile, a.abs)
st, err := c.CreateFromYaml(yaml, file, env)
if err != nil {
return nil, err
}
helmfiles := []state.SubHelmfileSpec{}
for _, hf := range st.Helmfiles {
globPattern := hf.Path
var absPathPattern string
if filepath.IsAbs(globPattern) {
absPathPattern = globPattern
} else {
absPathPattern = st.JoinBase(globPattern)
}
matches, err := a.glob(absPathPattern)
if err != nil {
return nil, fmt.Errorf("failed processing %s: %v", globPattern, err)
}
sort.Strings(matches)
for _, match := range matches {
newHelmfile := hf
newHelmfile.Path = match
helmfiles = append(helmfiles, newHelmfile)
}
}
st.Helmfiles = helmfiles
if a.Reverse {
rev := func(i, j int) bool {
return j < i
}
sort.Slice(st.Releases, rev)
sort.Slice(st.Helmfiles, rev)
}
if a.KubeContext != "" {
if st.HelmDefaults.KubeContext != "" {
log.Printf("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.")
os.Exit(1)
}
st.HelmDefaults.KubeContext = a.KubeContext
}
if namespace != "" {
if st.Namespace != "" {
log.Printf("err: Cannot use option --namespace and set attribute namespace.")
os.Exit(1)
}
st.Namespace = namespace
}
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-sigs
errs := []error{fmt.Errorf("Received [%s] to shutdown ", sig)}
_ = context{a, st}.clean(errs)
// See http://tldp.org/LDP/abs/html/exitcodes.html
switch sig {
case syscall.SIGINT:
os.Exit(130)
case syscall.SIGTERM:
os.Exit(143)
}
}()
return st, nil
}
type Error struct {
msg string

101
pkg/app/app_test.go
View File

@ -2,7 +2,6 @@ package app
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
@ -83,7 +82,14 @@ func (f *testFs) readFile(filename string) ([]byte, error) {
return []byte(str), nil
}
func (f *testFs) glob(pattern string) ([]string, error) {
func (f *testFs) glob(relPattern string) ([]string, error) {
var pattern string
if relPattern[0] == '/' {
pattern = relPattern
} else {
pattern = filepath.Join(f.wd, relPattern)
}
matches := []string{}
for name, _ := range f.files {
matched, err := filepath.Match(pattern, name)
@ -95,7 +101,7 @@ func (f *testFs) glob(pattern string) ([]string, error) {
}
}
if len(matches) == 0 {
return []string(nil), fmt.Errorf("no file matched: %s", pattern)
return []string(nil), fmt.Errorf("no file matched %s for files: %v", pattern, f.files)
}
return matches, nil
}
@ -640,15 +646,98 @@ func TestLoadDesiredStateFromYaml_DuplicateReleaseName(t *testing.T) {
labels:
stage: post
`)
readFile := func(filename string) ([]byte, error) {
if filename != yamlFile {
return nil, fmt.Errorf("unexpected filename: %s", filename)
}
return yamlContent, nil
}
app := &App{
readFile: ioutil.ReadFile,
readFile: readFile,
glob: filepath.Glob,
abs: filepath.Abs,
KubeContext: "default",
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
_, err := app.loadDesiredStateFromYaml(yamlContent, yamlFile, "default", "default")
_, err := app.loadDesiredStateFromYaml(yamlFile)
if err != nil {
t.Error("unexpected error")
t.Errorf("unexpected error: %v", err)
}
}
func TestLoadDesiredStateFromYaml_Bases(t *testing.T) {
yamlFile := "/path/to/yaml/file"
yamlContent := []byte(`bases:
- ../base.yaml
- ../base.gotmpl
{{ readFile "templates.yaml" }}
releases:
- name: myrelease1
chart: mychart1
labels:
stage: pre
foo: bar
- name: myrelease1
chart: mychart2
labels:
stage: post
<<: *default
`)
files := map[string][]byte{
yamlFile: yamlContent,
"/path/to/base.yaml": []byte(`environments:
default:
values:
- environments/default/1.yaml
`),
"/path/to/yaml/environments/default/1.yaml": []byte(`foo: FOO`),
"/path/to/base.gotmpl": []byte(`environments:
default:
values:
- environments/default/2.yaml
helmDefaults:
tillerNamespace: {{ .Environment.Values.tillerNs }}
`),
"/path/to/yaml/environments/default/2.yaml": []byte(`tillerNs: TILLER_NS`),
"/path/to/yaml/templates.yaml": []byte(`templates:
default: &default
missingFileHandler: Warn
values: ["` + "{{`" + `{{.Release.Name}}` + "`}}" + `/values.yaml"]
`),
}
readFile := func(filename string) ([]byte, error) {
content, ok := files[filename]
if !ok {
return nil, fmt.Errorf("unexpected filename: %s", filename)
}
return content, nil
}
app := &App{
readFile: readFile,
glob: filepath.Glob,
abs: filepath.Abs,
KubeContext: "default",
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
st, err := app.loadDesiredStateFromYaml(yamlFile)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if st.HelmDefaults.TillerNamespace != "TILLER_NS" {
t.Errorf("unexpected helmDefaults.tillerNamespace: expected=TILLER_NS, got=%s", st.HelmDefaults.TillerNamespace)
}
if *st.Releases[1].MissingFileHandler != "Warn" {
t.Errorf("unexpected releases[0].missingFileHandler: expected=Warn, got=%s", *st.Releases[1].MissingFileHandler)
}
if st.Releases[1].Values[0] != "{{`{{.Release.Name}}`}}/values.yaml" {
t.Errorf("unexpected releases[0].missingFileHandler: expected={{`{{.Release.Name}}`}}/values.yaml, got=%s", st.Releases[1].Values[0])
}
}

8
pkg/app/constants.go
View File

@ -13,6 +13,10 @@ const (
ExperimentalSelectorExplicit = "explicit-selector-inheritance" // value to remove default selector inheritance to sub-helmfiles and use the explicit one
)
func isExplicitSelectorInheritanceEnabled() bool {
return os.Getenv(ExperimentalEnvVar) == "true" || strings.Contains(os.Getenv(ExperimentalEnvVar), ExperimentalSelectorExplicit)
func experimentalModeEnabled() bool {
return os.Getenv(ExperimentalEnvVar) == "true"
}
func isExplicitSelectorInheritanceEnabled() bool {
return experimentalModeEnabled() || strings.Contains(os.Getenv(ExperimentalEnvVar), ExperimentalSelectorExplicit)
}

145
pkg/app/desired_state_file_loader.go Normal file
View File

@ -0,0 +1,145 @@
package app
import (
"fmt"
"github.com/imdario/mergo"
"github.com/roboll/helmfile/state"
"go.uber.org/zap"
"log"
"os"
"path/filepath"
"sort"
)
type desiredStateLoader struct {
KubeContext string
Reverse bool
env string
namespace string
readFile func(string) ([]byte, error)
abs func(string) (string, error)
glob func(string) ([]string, error)
logger *zap.SugaredLogger
}
func (ld *desiredStateLoader) Load(f string) (*state.HelmState, error) {
return ld.load(filepath.Dir(f), filepath.Base(f), true)
}
func (ld *desiredStateLoader) load(baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
var f string
if filepath.IsAbs(file) {
f = file
} else {
f = filepath.Join(baseDir, file)
}
fileBytes, err := ld.readFile(f)
if err != nil {
return nil, err
}
ext := filepath.Ext(f)
var yamlBytes []byte
if !experimentalModeEnabled() || ext == ".gotmpl" {
yamlBuf, err := ld.renderTemplateToYaml(baseDir, f, fileBytes)
if err != nil {
return nil, fmt.Errorf("error during %s parsing: %v", f, err)
}
yamlBytes = yamlBuf.Bytes()
} else {
yamlBytes = fileBytes
}
self, err := ld.loadYaml(
yamlBytes,
baseDir,
file,
)
if err != nil {
return nil, err
}
if !evaluateBases {
return self, nil
}
layers := []*state.HelmState{}
for _, b := range self.Bases {
base, err := ld.load(baseDir, b, false)
if err != nil {
return nil, err
}
layers = append(layers, base)
}
layers = append(layers, self)
for i := 1; i < len(layers); i++ {
if err := mergo.Merge(layers[0], layers[i], mergo.WithAppendSlice); err != nil {
return nil, err
}
}
return layers[0], nil
}
func (a *desiredStateLoader) loadYaml(yaml []byte, baseDir, file string) (*state.HelmState, error) {
c := state.NewCreator(a.logger, a.readFile, a.abs)
st, err := c.ParseAndLoadEnv(yaml, baseDir, file, a.env)
if err != nil {
return nil, err
}
helmfiles := []state.SubHelmfileSpec{}
for _, hf := range st.Helmfiles {
globPattern := hf.Path
var absPathPattern string
if filepath.IsAbs(globPattern) {
absPathPattern = globPattern
} else {
absPathPattern = st.JoinBase(globPattern)
}
matches, err := a.glob(absPathPattern)
if err != nil {
return nil, fmt.Errorf("failed processing %s: %v", globPattern, err)
}
sort.Strings(matches)
for _, match := range matches {
newHelmfile := hf
newHelmfile.Path = match
helmfiles = append(helmfiles, newHelmfile)
}
}
st.Helmfiles = helmfiles
if a.Reverse {
rev := func(i, j int) bool {
return j < i
}
sort.Slice(st.Releases, rev)
sort.Slice(st.Helmfiles, rev)
}
if a.KubeContext != "" {
if st.HelmDefaults.KubeContext != "" {
log.Printf("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.")
os.Exit(1)
}
st.HelmDefaults.KubeContext = a.KubeContext
}
if a.namespace != "" {
if st.Namespace != "" {
log.Printf("err: Cannot use option --namespace and set attribute namespace.")
os.Exit(1)
}
st.Namespace = a.namespace
}
return st, nil
}

33
pkg/app/two_pass_renderer.go
View File

@ -6,8 +6,6 @@ import (
"github.com/roboll/helmfile/environment"
"github.com/roboll/helmfile/state"
"github.com/roboll/helmfile/tmpl"
"go.uber.org/zap"
"path/filepath"
"strings"
)
@ -20,39 +18,30 @@ func prependLineNumbers(text string) string {
return buf.String()
}
type twoPassRenderer struct {
reader func(string) ([]byte, error)
env string
namespace string
filename string
logger *zap.SugaredLogger
abs func(string) (string, error)
}
func (r *twoPassRenderer) renderEnvironment(content []byte) environment.Environment {
func (r *desiredStateLoader) renderEnvironment(baseDir, filename string, content []byte) environment.Environment {
firstPassEnv := environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
firstPassRenderer := tmpl.NewFirstPassRenderer(filepath.Dir(r.filename), tmplData)
firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData)
// parse as much as we can, tolerate errors, this is a preparse
yamlBuf, err := firstPassRenderer.RenderTemplateContentToBuffer(content)
if err != nil && r.logger != nil {
r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content)))
r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", filename, prependLineNumbers(string(content)))
if yamlBuf == nil { // we have a template syntax error, let the second parse report
r.logger.Debugf("template syntax error: %v", err)
return firstPassEnv
}
}
c := state.NewCreator(r.logger, r.reader, r.abs)
c := state.NewCreator(r.logger, r.readFile, r.abs)
c.Strict = false
// create preliminary state, as we may have an environment. Tolerate errors.
prestate, err := c.CreateFromYaml(yamlBuf.Bytes(), r.filename, r.env)
prestate, err := c.ParseAndLoadEnv(yamlBuf.Bytes(), baseDir, filename, r.env)
if err != nil && r.logger != nil {
switch err.(type) {
case *state.StateLoadError:
r.logger.Infof("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err)
}
r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String()))
r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String()))
}
if prestate != nil {
firstPassEnv = prestate.Env
@ -60,21 +49,21 @@ func (r *twoPassRenderer) renderEnvironment(content []byte) environment.Environm
return firstPassEnv
}
func (r *twoPassRenderer) renderTemplate(content []byte) (*bytes.Buffer, error) {
func (r *desiredStateLoader) renderTemplateToYaml(baseDir, filename string, content []byte) (*bytes.Buffer, error) {
// try a first pass render. This will always succeed, but can produce a limited env
firstPassEnv := r.renderEnvironment(content)
firstPassEnv := r.renderEnvironment(baseDir, filename, content)
tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
secondPassRenderer := tmpl.NewFileRenderer(r.reader, filepath.Dir(r.filename), tmplData)
secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData)
yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content)
if err != nil {
if r.logger != nil {
r.logger.Debugf("second-pass rendering failed, input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content)))
r.logger.Debugf("second-pass rendering failed, input of \"%s\":\n%s", filename, prependLineNumbers(string(content)))
}
return nil, err
}
if r.logger != nil {
r.logger.Debugf("second-pass rendering result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String()))
r.logger.Debugf("second-pass rendering result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String()))
}
return yamlBuf, nil
}

33
pkg/app/two_pass_renderer_test.go
View File

@ -12,12 +12,11 @@ import (
"gopkg.in/yaml.v2"
)
func makeRenderer(readFile func(string) ([]byte, error), env string) *twoPassRenderer {
return &twoPassRenderer{
reader: readFile,
func makeLoader(readFile func(string) ([]byte, error), env string) *desiredStateLoader {
return &desiredStateLoader{
readFile: readFile,
env: env,
namespace: "namespace",
filename: "",
logger: helmexec.NewLogger(os.Stdout, "debug"),
abs: filepath.Abs,
}
@ -51,8 +50,8 @@ releases:
return []byte(""), nil
}
r := makeRenderer(fileReader, "staging")
yamlBuf, err := r.renderTemplate(yamlContent)
r := makeLoader(fileReader, "staging")
yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
@ -102,9 +101,9 @@ releases:
return defaultValuesYaml, nil
}
r := makeRenderer(fileReader, "staging")
r := makeLoader(fileReader, "staging")
// test the double rendering
yamlBuf, err := r.renderTemplate(yamlContent)
yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -151,9 +150,9 @@ releases:
return defaultValuesYaml, nil
}
r := makeRenderer(fileReader, "staging")
r := makeLoader(fileReader, "staging")
// test the double rendering
_, err := r.renderTemplate(yamlContent)
_, err := r.renderTemplateToYaml("", "", yamlContent)
if !strings.Contains(err.Error(), "stringTemplate:8") {
t.Fatalf("error should contain a stringTemplate error (reference to unknow key) %v", err)
@ -190,8 +189,8 @@ releases:
return defaultValuesYamlGotmpl, nil
}
r := makeRenderer(fileReader, "staging")
rendered, _ := r.renderTemplate(yamlContent)
r := makeLoader(fileReader, "staging")
rendered, _ := r.renderTemplateToYaml("", "", yamlContent)
var state state.HelmState
yaml.Unmarshal(rendered.Bytes(), &state)
@ -217,8 +216,8 @@ func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) {
return defaultValuesYaml, nil
}
r := makeRenderer(fileReader, "staging")
yamlBuf, err := r.renderTemplate(yamlContent)
r := makeLoader(fileReader, "staging")
yamlBuf, err := r.renderTemplateToYaml("", "", yamlContent)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -231,7 +230,7 @@ func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) {
}
}
func TestReadFromYaml_HelfileShouldBeResilentToTemplateErrors(t *testing.T) {
func TestReadFromYaml_HelmfileShouldBeResilentToTemplateErrors(t *testing.T) {
yamlContent := []byte(`environments:
staging:
production:
@ -248,8 +247,8 @@ releases:
return yamlContent, nil
}
r := makeRenderer(fileReader, "staging")
_, err := r.renderTemplate(yamlContent)
r := makeLoader(fileReader, "staging")
_, err := r.renderTemplateToYaml("", "", yamlContent)
if err == nil {
t.Fatalf("wanted error, none returned")
}

20
state/create.go
View File

@ -4,7 +4,6 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
@ -33,16 +32,6 @@ func (e *UndefinedEnvError) Error() string {
return e.msg
}
func createFromYaml(content []byte, file string, env string, logger *zap.SugaredLogger) (*HelmState, error) {
c := &creator{
logger,
ioutil.ReadFile,
filepath.Abs,
true,
}
return c.CreateFromYaml(content, file, env)
}
type creator struct {
logger *zap.SugaredLogger
readFile func(string) ([]byte, error)
@ -60,15 +49,12 @@ func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error)
}
}
func (c *creator) CreateFromYaml(content []byte, file string, env string) (*HelmState, error) {
// Parses YAML into HelmState, while loading environment values files relative to the `cwd`
func (c *creator) ParseAndLoadEnv(content []byte, baseDir, file string, env string) (*HelmState, error) {
var state HelmState
basePath, err := c.abs(filepath.Dir(file))
if err != nil {
return nil, &StateLoadError{fmt.Sprintf("failed to read %s", file), err}
}
state.FilePath = file
state.basePath = basePath
state.basePath = baseDir
decoder := yaml.NewDecoder(bytes.NewReader(content))
if !c.Strict {

14
state/create_test.go
View File

@ -2,6 +2,8 @@ package state
import (
"fmt"
"go.uber.org/zap"
"io/ioutil"
"path/filepath"
"reflect"
"testing"
@ -10,6 +12,16 @@ import (
"gotest.tools/assert/cmp"
)
func createFromYaml(content []byte, file string, env string, logger *zap.SugaredLogger) (*HelmState, error) {
c := &creator{
logger,
ioutil.ReadFile,
filepath.Abs,
true,
}
return c.ParseAndLoadEnv(content, filepath.Dir(file), file, env)
}
func TestReadFromYaml(t *testing.T) {
yamlFile := "example/path/to/yaml/file"
yamlContent := []byte(`releases:
@ -101,7 +113,7 @@ bar: {{ readFile "bar.txt" }}
return nil, fmt.Errorf("unexpected filename: %s", filename)
}
state, err := NewCreator(logger, readFile, filepath.Abs).CreateFromYaml(yamlContent, yamlFile, "production")
state, err := NewCreator(logger, readFile, filepath.Abs).ParseAndLoadEnv(yamlContent, filepath.Dir(yamlFile), yamlFile, "production")
if err != nil {
t.Errorf("unexpected error: %v", err)
}

8
state/state.go
View File

@ -27,9 +27,11 @@ import (
// HelmState structure for the helmfile
type HelmState struct {
basePath string
Environments map[string]EnvironmentSpec
FilePath string
basePath string
Environments map[string]EnvironmentSpec
FilePath string
Bases []string `yaml:"bases"`
HelmDefaults HelmSpec `yaml:"helmDefaults"`
Helmfiles []SubHelmfileSpec `yaml:"helmfiles"`
DeprecatedContext string `yaml:"context"`