feat: Opt-out dep-builds and repo-updates (#463)

* Improve code organization

To make sure it is still readable after upcoming changes to helmfile

* feat: `helmfile deps` to update dependencies of all the local charts

Resolves #450

* feat: helmfile updates repos and build deps by default

But not update deps. Use `helmfile deps` to update deps, and provide `--skip-deps` to skip updating repos and builds deps in sync/diff/apply/template

Resolves #415

* Improve integration test coverage

pkg/app/app.go

@@ -0,0 +1,386 @@
+package app
+
+import (
+	"fmt"
+	"github.com/roboll/helmfile/helmexec"
+	"github.com/roboll/helmfile/state"
+	"go.uber.org/zap"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"sort"
+	"syscall"
+)
+
+type App struct {
+	KubeContext string
+	Logger      *zap.SugaredLogger
+	Reverse     bool
+	Env         string
+	Namespace   string
+	Selectors   []string
+
+	readFile          func(string) ([]byte, error)
+	glob              func(string) ([]string, error)
+	abs               func(string) (string, error)
+	fileExistsAt      func(string) bool
+	directoryExistsAt func(string) bool
+
+	getwd func() (string, error)
+	chdir func(string) error
+}
+
+func Init(app *App) *App {
+	app.readFile = ioutil.ReadFile
+	app.glob = filepath.Glob
+	app.abs = filepath.Abs
+	app.getwd = os.Getwd
+	app.chdir = os.Chdir
+	app.fileExistsAt = fileExistsAt
+	app.directoryExistsAt = directoryExistsAt
+	return app
+}
+
+func (a *App) within(dir string, do func() error) error {
+	if dir == "." {
+		return do()
+	}
+
+	prev, err := a.getwd()
+	if err != nil {
+		return fmt.Errorf("failed getting current working direcotyr: %v", err)
+	}
+
+	absDir, err := a.abs(dir)
+	if err != nil {
+		return err
+	}
+
+	a.Logger.Debugf("changing working directory to \"%s\"", absDir)
+
+	if err := a.chdir(absDir); err != nil {
+		return fmt.Errorf("failed changing working directory to \"%s\": %v", absDir, err)
+	}
+
+	appErr := do()
+
+	a.Logger.Debugf("changing working directory back to \"%s\"", prev)
+
+	if chdirBackErr := a.chdir(prev); chdirBackErr != nil {
+		if appErr != nil {
+			a.Logger.Warnf("%v", appErr)
+		}
+		return fmt.Errorf("failed chaging working directory back to \"%s\": %v", prev, chdirBackErr)
+	}
+
+	return appErr
+}
+
+func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error {
+	desiredStateFiles, err := a.findDesiredStateFiles(fileOrDir)
+	if err != nil {
+		return err
+	}
+
+	for _, relPath := range desiredStateFiles {
+		var file string
+		var dir string
+		if a.directoryExistsAt(relPath) {
+			file = relPath
+			dir = relPath
+		} else {
+			file = filepath.Base(relPath)
+			dir = filepath.Dir(relPath)
+		}
+
+		a.Logger.Debugf("processing file \"%s\" in directory \"%s\"", file, dir)
+
+		err := a.within(dir, func() error {
+			return do(file)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (a *App) VisitDesiredStates(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error {
+	noMatchInHelmfiles := true
+	err := a.visitStateFiles(fileOrDir, func(f string) error {
+		content, err := a.readFile(f)
+		if err != nil {
+			return err
+		}
+		// render template, in two runs
+		r := &twoPassRenderer{
+			reader:    a.readFile,
+			env:       a.Env,
+			namespace: a.Namespace,
+			filename:  f,
+			logger:    a.Logger,
+			abs:       a.abs,
+		}
+		yamlBuf, err := r.renderTemplate(content)
+		if err != nil {
+			return fmt.Errorf("error during %s parsing: %v", f, err)
+		}
+
+		st, err := a.loadDesiredStateFromYaml(
+			yamlBuf.Bytes(),
+			f,
+			a.Namespace,
+			a.Env,
+		)
+
+		helm := helmexec.New(a.Logger, a.KubeContext)
+
+		if err != nil {
+			switch stateLoadErr := err.(type) {
+			// Addresses https://github.com/roboll/helmfile/issues/279
+			case *state.StateLoadError:
+				switch stateLoadErr.Cause.(type) {
+				case *state.UndefinedEnvError:
+					return nil
+				default:
+					return err
+				}
+			default:
+				return err
+			}
+		}
+
+		errs := []error{}
+
+		if len(st.Helmfiles) > 0 {
+			noMatchInSubHelmfiles := true
+			for _, m := range st.Helmfiles {
+				if err := a.VisitDesiredStates(m, converge); err != nil {
+					switch err.(type) {
+					case *NoMatchingHelmfileError:
+
+					default:
+						return fmt.Errorf("failed processing %s: %v", m, err)
+					}
+				} else {
+					noMatchInSubHelmfiles = false
+				}
+			}
+			noMatchInHelmfiles = noMatchInHelmfiles && noMatchInSubHelmfiles
+		} else {
+			var err error
+			st, err = st.ExecuteTemplates()
+			if err != nil {
+				return fmt.Errorf("failed executing release templates in \"%s\": %v", f, err)
+			}
+
+			var processed bool
+			processed, errs = converge(st, helm)
+			noMatchInHelmfiles = noMatchInHelmfiles && !processed
+		}
+
+		return clean(st, errs)
+	})
+	if err != nil {
+		return err
+	}
+	if noMatchInHelmfiles {
+		return &NoMatchingHelmfileError{selectors: a.Selectors, env: a.Env}
+	}
+	return nil
+}
+
+func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) []error) error {
+	selectors := a.Selectors
+
+	err := a.VisitDesiredStates(fileOrDir, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
+		if len(selectors) > 0 {
+			err := st.FilterReleases(selectors)
+			if err != nil {
+				return false, []error{err}
+			}
+		}
+
+		releaseNameCounts := map[string]int{}
+		for _, r := range st.Releases {
+			releaseNameCounts[r.Name]++
+		}
+		for name, c := range releaseNameCounts {
+			if c > 1 {
+				return false, []error{fmt.Errorf("duplicate release \"%s\" found: there were %d releases named \"%s\" matching specified selector", name, c, name)}
+			}
+		}
+
+		errs := converge(st, helm)
+
+		processed := len(st.Releases) != 0 && len(errs) == 0
+
+		return processed, errs
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (a *App) findStateFilesInAbsPaths(specifiedPath string) ([]string, error) {
+	rels, err := a.findDesiredStateFiles(specifiedPath)
+	if err != nil {
+		return rels, err
+	}
+
+	files := make([]string, len(rels))
+	for i := range rels {
+		files[i], err = filepath.Abs(rels[i])
+		if err != nil {
+			return []string{}, err
+		}
+	}
+	return files, nil
+}
+
+func (a *App) findDesiredStateFiles(specifiedPath string) ([]string, error) {
+	var helmfileDir string
+	if specifiedPath != "" {
+		if a.fileExistsAt(specifiedPath) {
+			return []string{specifiedPath}, nil
+		} else if a.directoryExistsAt(specifiedPath) {
+			helmfileDir = specifiedPath
+		} else {
+			return []string{}, fmt.Errorf("specified state file %s is not found", specifiedPath)
+		}
+	} else {
+		var defaultFile string
+		if a.fileExistsAt(DefaultHelmfile) {
+			defaultFile = DefaultHelmfile
+		} else if a.fileExistsAt(DeprecatedHelmfile) {
+			log.Printf(
+				"warn: %s is being loaded: %s is deprecated in favor of %s. See https://github.com/roboll/helmfile/issues/25 for more information",
+				DeprecatedHelmfile,
+				DeprecatedHelmfile,
+				DefaultHelmfile,
+			)
+			defaultFile = DeprecatedHelmfile
+		}
+
+		if a.directoryExistsAt(DefaultHelmfileDirectory) {
+			if defaultFile != "" {
+				return []string{}, fmt.Errorf("configuration conlict error: you can have either %s or %s, but not both", defaultFile, DefaultHelmfileDirectory)
+			}
+
+			helmfileDir = DefaultHelmfileDirectory
+		} else if defaultFile != "" {
+			return []string{defaultFile}, nil
+		} else {
+			return []string{}, fmt.Errorf("no state file found. It must be named %s/*.yaml, %s, or %s, or otherwise specified with the --file flag", DefaultHelmfileDirectory, DefaultHelmfile, DeprecatedHelmfile)
+		}
+	}
+
+	files, err := a.glob(filepath.Join(helmfileDir, "*.yaml"))
+	if err != nil {
+		return []string{}, err
+	}
+	sort.Slice(files, func(i, j int) bool {
+		return files[i] < files[j]
+	})
+	return files, nil
+}
+
+func fileExistsAt(path string) bool {
+	fileInfo, err := os.Stat(path)
+	return err == nil && fileInfo.Mode().IsRegular()
+}
+
+func directoryExistsAt(path string) bool {
+	fileInfo, err := os.Stat(path)
+	return err == nil && fileInfo.Mode().IsDir()
+}
+
+func (a *App) loadDesiredStateFromYaml(yaml []byte, file string, namespace string, env string) (*state.HelmState, error) {
+	c := state.NewCreator(a.Logger, a.readFile, a.abs)
+	st, err := c.CreateFromYaml(yaml, file, env)
+	if err != nil {
+		return nil, err
+	}
+
+	helmfiles := []string{}
+	for _, globPattern := range st.Helmfiles {
+		helmfileRelativePattern := st.JoinBase(globPattern)
+		matches, err := a.glob(helmfileRelativePattern)
+		if err != nil {
+			return nil, fmt.Errorf("failed processing %s: %v", globPattern, err)
+		}
+		sort.Strings(matches)
+
+		helmfiles = append(helmfiles, matches...)
+	}
+	st.Helmfiles = helmfiles
+
+	if a.Reverse {
+		rev := func(i, j int) bool {
+			return j < i
+		}
+		sort.Slice(st.Releases, rev)
+		sort.Slice(st.Helmfiles, rev)
+	}
+
+	if a.KubeContext != "" {
+		if st.HelmDefaults.KubeContext != "" {
+			log.Printf("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.")
+			os.Exit(1)
+		}
+		st.HelmDefaults.KubeContext = a.KubeContext
+	}
+	if namespace != "" {
+		if st.Namespace != "" {
+			log.Printf("err: Cannot use option --namespace and set attribute namespace.")
+			os.Exit(1)
+		}
+		st.Namespace = namespace
+	}
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		sig := <-sigs
+
+		errs := []error{fmt.Errorf("Received [%s] to shutdown ", sig)}
+		_ = clean(st, errs)
+		// See http://tldp.org/LDP/abs/html/exitcodes.html
+		switch sig {
+		case syscall.SIGINT:
+			os.Exit(130)
+		case syscall.SIGTERM:
+			os.Exit(143)
+		}
+	}()
+
+	return st, nil
+}
+
+func clean(st *state.HelmState, errs []error) error {
+	if errs == nil {
+		errs = []error{}
+	}
+
+	cleanErrs := st.Clean()
+	if cleanErrs != nil {
+		errs = append(errs, cleanErrs...)
+	}
+
+	if errs != nil && len(errs) > 0 {
+		for _, err := range errs {
+			switch e := err.(type) {
+			case *state.ReleaseError:
+				fmt.Printf("err: release \"%s\" in \"%s\" failed: %v\n", e.Name, st.FilePath, e)
+			default:
+				fmt.Printf("err: %v\n", e)
+			}
+		}
+		return errs[0]
+	}
+	return nil
+}

pkg/app/app_test.go

@@ -0,0 +1,403 @@
+package app
+
+import (
+	"fmt"
+	"github.com/roboll/helmfile/helmexec"
+	"github.com/roboll/helmfile/state"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type testFs struct {
+	wd    string
+	dirs  map[string]bool
+	files map[string]string
+}
+
+func appWithFs(app *App, files map[string]string) *App {
+	fs := newTestFs(files)
+	return injectFs(app, fs)
+}
+
+func injectFs(app *App, fs *testFs) *App {
+	app.readFile = fs.readFile
+	app.glob = fs.glob
+	app.abs = fs.abs
+	app.getwd = fs.getwd
+	app.chdir = fs.chdir
+	app.fileExistsAt = fs.fileExistsAt
+	app.directoryExistsAt = fs.directoryExistsAt
+	return app
+}
+
+func newTestFs(files map[string]string) *testFs {
+	dirs := map[string]bool{}
+	for abs, _ := range files {
+		d := filepath.Dir(abs)
+		dirs[d] = true
+	}
+	return &testFs{
+		wd:    "/path/to",
+		dirs:  dirs,
+		files: files,
+	}
+}
+
+func (f *testFs) fileExistsAt(path string) bool {
+	var ok bool
+	if strings.Contains(path, "/") {
+		_, ok = f.files[path]
+	} else {
+		_, ok = f.files[filepath.Join(f.wd, path)]
+	}
+	return ok
+}
+
+func (f *testFs) directoryExistsAt(path string) bool {
+	var ok bool
+	if strings.Contains(path, "/") {
+		_, ok = f.dirs[path]
+	} else {
+		_, ok = f.dirs[filepath.Join(f.wd, path)]
+	}
+	return ok
+}
+
+func (f *testFs) readFile(filename string) ([]byte, error) {
+	var str string
+	var ok bool
+	if strings.Contains(filename, "/") {
+		str, ok = f.files[filename]
+	} else {
+		str, ok = f.files[filepath.Join(f.wd, filename)]
+	}
+	if !ok {
+		return []byte(nil), fmt.Errorf("no file found: %s", filename)
+	}
+	return []byte(str), nil
+}
+
+func (f *testFs) glob(pattern string) ([]string, error) {
+	matches := []string{}
+	for name, _ := range f.files {
+		matched, err := filepath.Match(pattern, name)
+		if err != nil {
+			return nil, err
+		}
+		if matched {
+			matches = append(matches, name)
+		}
+	}
+	if len(matches) == 0 {
+		return []string(nil), fmt.Errorf("no file matched: %s", pattern)
+	}
+	return matches, nil
+}
+
+func (f *testFs) abs(path string) (string, error) {
+	var p string
+	if path[0] == '/' {
+		p = path
+	} else {
+		p = filepath.Join(f.wd, path)
+	}
+	return filepath.Clean(p), nil
+}
+
+func (f *testFs) getwd() (string, error) {
+	return f.wd, nil
+}
+
+func (f *testFs) chdir(dir string) error {
+	if dir == "/path/to" || dir == "/path/to/helmfile.d" {
+		f.wd = dir
+		return nil
+	}
+	return fmt.Errorf("unexpected chdir \"%s\"", dir)
+}
+
+// See https://github.com/roboll/helmfile/issues/193
+func TestVisitDesiredStatesWithReleasesFiltered(t *testing.T) {
+	files := map[string]string{
+		"/path/to/helmfile.yaml": `
+helmfiles:
+- helmfile.d/a*.yaml
+- helmfile.d/b*.yaml
+`,
+		"/path/to/helmfile.d/a1.yaml": `
+releases:
+- name: zipkin
+  chart: stable/zipkin
+`,
+		"/path/to/helmfile.d/a2.yaml": `
+releases:
+- name: prometheus
+  chart: stable/prometheus
+`,
+		"/path/to/helmfile.d/b.yaml": `
+releases:
+- name: grafana
+  chart: stable/grafana
+`,
+	}
+	noop := func(st *state.HelmState, helm helmexec.Interface) []error {
+		return []error{}
+	}
+
+	testcases := []struct {
+		name      string
+		expectErr bool
+	}{
+		{name: "prometheus", expectErr: false},
+		{name: "zipkin", expectErr: false},
+		{name: "grafana", expectErr: false},
+		{name: "elasticsearch", expectErr: true},
+	}
+
+	for _, testcase := range testcases {
+		app := appWithFs(&App{
+			KubeContext: "default",
+			Logger:      helmexec.NewLogger(os.Stderr, "debug"),
+			Selectors:   []string{fmt.Sprintf("name=%s", testcase.name)},
+			Namespace:   "",
+			Env:         "default",
+		}, files)
+		err := app.VisitDesiredStatesWithReleasesFiltered(
+			"helmfile.yaml", noop,
+		)
+		if testcase.expectErr && err == nil {
+			t.Errorf("error expected but not happened for name=%s", testcase.name)
+		} else if !testcase.expectErr && err != nil {
+			t.Errorf("unexpected error for name=%s: %v", testcase.name, err)
+		}
+	}
+}
+
+// See https://github.com/roboll/helmfile/issues/320
+func TestVisitDesiredStatesWithReleasesFiltered_UndefinedEnv(t *testing.T) {
+	files := map[string]string{
+		"/path/to/helmfile.yaml": `
+environments:
+  prod:
+
+helmfiles:
+- helmfile.d/a*.yaml
+`,
+		"/path/to/helmfile.d/a1.yaml": `
+environments:
+  prod:
+
+releases:
+- name: zipkin
+  chart: stable/zipkin
+`,
+	}
+	noop := func(st *state.HelmState, helm helmexec.Interface) []error {
+		return []error{}
+	}
+
+	testcases := []struct {
+		name      string
+		expectErr bool
+	}{
+		{name: "undefined_env", expectErr: true},
+		{name: "default", expectErr: false},
+		{name: "prod", expectErr: false},
+	}
+
+	for _, testcase := range testcases {
+		app := appWithFs(&App{
+			KubeContext: "default",
+			Logger:      helmexec.NewLogger(os.Stderr, "debug"),
+			Namespace:   "",
+			Selectors:   []string{},
+			Env:         testcase.name,
+		}, files)
+		err := app.VisitDesiredStatesWithReleasesFiltered(
+			"helmfile.yaml", noop,
+		)
+		if testcase.expectErr && err == nil {
+			t.Errorf("error expected but not happened for environment=%s", testcase.name)
+		} else if !testcase.expectErr && err != nil {
+			t.Errorf("unexpected error for environment=%s: %v", testcase.name, err)
+		}
+	}
+}
+
+// See https://github.com/roboll/helmfile/issues/322
+func TestVisitDesiredStatesWithReleasesFiltered_Selectors(t *testing.T) {
+	files := map[string]string{
+		"/path/to/helmfile.yaml": `
+helmfiles:
+- helmfile.d/a*.yaml
+- helmfile.d/b*.yaml
+`,
+		"/path/to/helmfile.d/a1.yaml": `
+releases:
+- name: zipkin
+  chart: stable/zipkin
+`,
+		"/path/to/helmfile.d/a2.yaml": `
+releases:
+- name: prometheus
+  chart: stable/prometheus
+`,
+		"/path/to/helmfile.d/b.yaml": `
+releases:
+- name: grafana
+  chart: stable/grafana
+- name: foo
+  chart: charts/foo
+  labels:
+    duplicated: yes
+- name: foo
+  chart: charts/foo
+  labels:
+    duplicated: yes
+`,
+	}
+
+	testcases := []struct {
+		label         string
+		expectedCount int
+		expectErr     bool
+		errMsg        string
+	}{
+		{label: "name=prometheus", expectedCount: 1, expectErr: false},
+		{label: "name=", expectedCount: 0, expectErr: true, errMsg: "failed processing /path/to/helmfile.d/a1.yaml: Malformed label: name=. Expected label in form k=v or k!=v"},
+		{label: "name!=", expectedCount: 0, expectErr: true, errMsg: "failed processing /path/to/helmfile.d/a1.yaml: Malformed label: name!=. Expected label in form k=v or k!=v"},
+		{label: "name", expectedCount: 0, expectErr: true, errMsg: "failed processing /path/to/helmfile.d/a1.yaml: Malformed label: name. Expected label in form k=v or k!=v"},
+		// See https://github.com/roboll/helmfile/issues/193
+		{label: "duplicated=yes", expectedCount: 0, expectErr: true, errMsg: "failed processing /path/to/helmfile.d/b.yaml: duplicate release \"foo\" found: there were 2 releases named \"foo\" matching specified selector"},
+	}
+
+	for _, testcase := range testcases {
+		actual := []string{}
+
+		collectReleases := func(st *state.HelmState, helm helmexec.Interface) []error {
+			for _, r := range st.Releases {
+				actual = append(actual, r.Name)
+			}
+			return []error{}
+		}
+
+		app := appWithFs(&App{
+			KubeContext: "default",
+			Logger:      helmexec.NewLogger(os.Stderr, "debug"),
+			Namespace:   "",
+			Selectors:   []string{testcase.label},
+			Env:         "default",
+		}, files)
+
+		err := app.VisitDesiredStatesWithReleasesFiltered(
+			"helmfile.yaml", collectReleases,
+		)
+		if testcase.expectErr {
+			if err == nil {
+				t.Errorf("error expected but not happened for selector %s", testcase.label)
+			} else if err.Error() != testcase.errMsg {
+				t.Errorf("unexpected error message: expected=\"%s\", actual=\"%s\"", testcase.errMsg, err.Error())
+			}
+		} else if !testcase.expectErr && err != nil {
+			t.Errorf("unexpected error for selector %s: %v", testcase.label, err)
+		}
+		if len(actual) != testcase.expectedCount {
+			t.Errorf("unexpected release count for selector %s: expected=%d, actual=%d", testcase.label, testcase.expectedCount, len(actual))
+		}
+	}
+}
+
+// See https://github.com/roboll/helmfile/issues/312
+func TestVisitDesiredStatesWithReleasesFiltered_ReverseOrder(t *testing.T) {
+	files := map[string]string{
+		"/path/to/helmfile.yaml": `
+helmfiles:
+- helmfile.d/a*.yaml
+- helmfile.d/b*.yaml
+`,
+		"/path/to/helmfile.d/a1.yaml": `
+releases:
+- name: zipkin
+  chart: stable/zipkin
+`,
+		"/path/to/helmfile.d/a2.yaml": `
+releases:
+- name: prometheus
+  chart: stable/prometheus
+- name: elasticsearch
+  chart: stable/elasticsearch
+`,
+		"/path/to/helmfile.d/b.yaml": `
+releases:
+- name: grafana
+  chart: stable/grafana
+`,
+	}
+
+	expected := []string{"grafana", "elasticsearch", "prometheus", "zipkin"}
+
+	testcases := []struct {
+		reverse  bool
+		expected []string
+	}{
+		{reverse: false, expected: []string{"zipkin", "prometheus", "elasticsearch", "grafana"}},
+		{reverse: true, expected: []string{"grafana", "elasticsearch", "prometheus", "zipkin"}},
+	}
+	for _, testcase := range testcases {
+		actual := []string{}
+
+		collectReleases := func(st *state.HelmState, helm helmexec.Interface) []error {
+			for _, r := range st.Releases {
+				actual = append(actual, r.Name)
+			}
+			return []error{}
+		}
+		app := appWithFs(&App{
+			KubeContext: "default",
+			Logger:      helmexec.NewLogger(os.Stderr, "debug"),
+			Reverse:     testcase.reverse,
+			Namespace:   "",
+			Selectors:   []string{},
+			Env:         "default",
+		}, files)
+		err := app.VisitDesiredStatesWithReleasesFiltered(
+			"helmfile.yaml", collectReleases,
+		)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		if !reflect.DeepEqual(testcase.expected, actual) {
+			t.Errorf("releases did not match: expected=%v actual=%v", expected, actual)
+		}
+	}
+}
+
+func TestLoadDesiredStateFromYaml_DuplicateReleaseName(t *testing.T) {
+	yamlFile := "example/path/to/yaml/file"
+	yamlContent := []byte(`releases:
+- name: myrelease1
+  chart: mychart1
+  labels:
+    stage: pre
+    foo: bar
+- name: myrelease1
+  chart: mychart2
+  labels:
+    stage: post
+`)
+	app := &App{
+		readFile:    ioutil.ReadFile,
+		glob:        filepath.Glob,
+		abs:         filepath.Abs,
+		KubeContext: "default",
+		Logger:      helmexec.NewLogger(os.Stderr, "debug"),
+	}
+	_, err := app.loadDesiredStateFromYaml(yamlContent, yamlFile, "default", "default")
+	if err != nil {
+		t.Error("unexpected error")
+	}
+}

pkg/app/constants.go

@@ -0,0 +1,7 @@
+package app
+
+const (
+	DefaultHelmfile          = "helmfile.yaml"
+	DeprecatedHelmfile       = "charts.yaml"
+	DefaultHelmfileDirectory = "helmfile.d"
+)

pkg/app/context.go

@@ -0,0 +1,33 @@
+package app
+
+import "github.com/roboll/helmfile/state"
+
+type Context struct {
+	updatedRepos map[string]struct{}
+}
+
+func NewContext() Context {
+	return Context{
+		updatedRepos: map[string]struct{}{},
+	}
+}
+
+func (ctx Context) SyncReposOnce(st *state.HelmState, helm state.RepoUpdater) []error {
+	var errs []error
+
+	allUpdated := true
+	for _, r := range st.Repositories {
+		_, exists := ctx.updatedRepos[r.Name]
+		allUpdated = allUpdated && exists
+	}
+
+	if !allUpdated {
+		errs = st.SyncRepos(helm)
+
+		for _, r := range st.Repositories {
+			ctx.updatedRepos[r.Name] = struct{}{}
+		}
+	}
+
+	return errs
+}

pkg/app/errors.go

@@ -0,0 +1,19 @@
+package app
+
+import (
+	"fmt"
+	"strings"
+)
+
+type NoMatchingHelmfileError struct {
+	selectors []string
+	env       string
+}
+
+func (e *NoMatchingHelmfileError) Error() string {
+	return fmt.Sprintf(
+		"err: no releases found that matches specified selector(%s) and environment(%s), in any helmfile",
+		strings.Join(e.selectors, ", "),
+		e.env,
+	)
+}

pkg/app/two_pass_renderer.go

@@ -0,0 +1,80 @@
+package app
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/roboll/helmfile/environment"
+	"github.com/roboll/helmfile/state"
+	"github.com/roboll/helmfile/tmpl"
+	"go.uber.org/zap"
+	"path/filepath"
+	"strings"
+)
+
+func prependLineNumbers(text string) string {
+	buf := bytes.NewBufferString("")
+	lines := strings.Split(text, "\n")
+	for i, line := range lines {
+		buf.WriteString(fmt.Sprintf("%2d: %s\n", i, line))
+	}
+	return buf.String()
+}
+
+type twoPassRenderer struct {
+	reader    func(string) ([]byte, error)
+	env       string
+	namespace string
+	filename  string
+	logger    *zap.SugaredLogger
+	abs       func(string) (string, error)
+}
+
+func (r *twoPassRenderer) renderEnvironment(content []byte) environment.Environment {
+	firstPassEnv := environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
+	tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
+	firstPassRenderer := tmpl.NewFirstPassRenderer(filepath.Dir(r.filename), tmplData)
+
+	// parse as much as we can, tolerate errors, this is a preparse
+	yamlBuf, err := firstPassRenderer.RenderTemplateContentToBuffer(content)
+	if err != nil && r.logger != nil {
+		r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content)))
+		if yamlBuf == nil { // we have a template syntax error, let the second parse report
+			r.logger.Debugf("template syntax error: %v", err)
+			return firstPassEnv
+		}
+	}
+	c := state.NewCreator(r.logger, r.reader, r.abs)
+	c.Strict = false
+	// create preliminary state, as we may have an environment. Tolerate errors.
+	prestate, err := c.CreateFromYaml(yamlBuf.Bytes(), r.filename, r.env)
+	if err != nil && r.logger != nil {
+		switch err.(type) {
+		case *state.StateLoadError:
+			r.logger.Infof("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err)
+		}
+		r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String()))
+	}
+	if prestate != nil {
+		firstPassEnv = prestate.Env
+	}
+	return firstPassEnv
+}
+
+func (r *twoPassRenderer) renderTemplate(content []byte) (*bytes.Buffer, error) {
+	// try a first pass render. This will always succeed, but can produce a limited env
+	firstPassEnv := r.renderEnvironment(content)
+
+	tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
+	secondPassRenderer := tmpl.NewFileRenderer(r.reader, filepath.Dir(r.filename), tmplData)
+	yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content)
+	if err != nil {
+		if r.logger != nil {
+			r.logger.Debugf("second-pass rendering failed, input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content)))
+		}
+		return nil, err
+	}
+	if r.logger != nil {
+		r.logger.Debugf("second-pass rendering result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String()))
+	}
+	return yamlBuf, nil
+}

pkg/app/two_pass_renderer_test.go

@@ -0,0 +1,256 @@
+package app
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/roboll/helmfile/helmexec"
+	"github.com/roboll/helmfile/state"
+	"gopkg.in/yaml.v2"
+)
+
+func makeRenderer(readFile func(string) ([]byte, error), env string) *twoPassRenderer {
+	return &twoPassRenderer{
+		reader:    readFile,
+		env:       env,
+		namespace: "namespace",
+		filename:  "",
+		logger:    helmexec.NewLogger(os.Stdout, "debug"),
+		abs:       filepath.Abs,
+	}
+}
+
+func TestReadFromYaml_MakeEnvironmentHasNoSideEffects(t *testing.T) {
+
+	yamlContent := []byte(`
+environments:
+  staging:
+    values:
+    - default/values.yaml
+  production:
+
+releases:
+- name: {{ readFile "other/default/values.yaml" }}
+  chart: mychart1
+`)
+
+	fileReaderCalls := 0
+	// make a reader that returns a simulated context
+	fileReader := func(filename string) ([]byte, error) {
+		expectedFilename := filepath.Clean("default/values.yaml")
+		if !strings.HasSuffix(filename, expectedFilename) {
+			return nil, fmt.Errorf("unexpected filename: expected=%s, actual=%s", expectedFilename, filename)
+		}
+		fileReaderCalls++
+		if fileReaderCalls == 2 {
+			return []byte("SecondPass"), nil
+		}
+		return []byte(""), nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	yamlBuf, err := r.renderTemplate(yamlContent)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	var state state.HelmState
+	err = yaml.Unmarshal(yamlBuf.Bytes(), &state)
+
+	if fileReaderCalls > 2 {
+		t.Error("reader should be called only twice")
+	}
+
+	if state.Releases[0].Name != "SecondPass" {
+		t.Errorf("release name should have ben set as SecondPass")
+	}
+}
+
+func TestReadFromYaml_RenderTemplate(t *testing.T) {
+
+	defaultValuesYaml := []byte(`
+releaseName: "hello"
+conditionalReleaseTag: "yes"
+`)
+
+	yamlContent := []byte(`
+environments:
+  staging:
+    values:
+    - default/values.yaml
+  production:
+
+releases:
+- name: {{ .Environment.Values.releaseName }}
+  chart: mychart1
+
+{{ if (eq .Environment.Values.conditionalReleaseTag "yes") }}
+- name: conditionalRelease
+{{ end }}
+
+`)
+
+	// make a reader that returns a simulated context
+	fileReader := func(filename string) ([]byte, error) {
+		expectedFilename := filepath.Clean("default/values.yaml")
+		if !strings.HasSuffix(filename, expectedFilename) {
+			return nil, fmt.Errorf("unexpected filename: expected=%s, actual=%s", expectedFilename, filename)
+		}
+		return defaultValuesYaml, nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	// test the double rendering
+	yamlBuf, err := r.renderTemplate(yamlContent)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	var state state.HelmState
+	err = yaml.Unmarshal(yamlBuf.Bytes(), &state)
+
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	if len(state.Releases) != 2 {
+		t.Fatal("there should be 2 releases")
+	}
+
+	if state.Releases[0].Name != "hello" {
+		t.Errorf("release name should be hello")
+	}
+
+	if state.Releases[1].Name != "conditionalRelease" {
+		t.Error("conditional release should have been present")
+	}
+}
+
+func TestReadFromYaml_RenderTemplateWithValuesReferenceError(t *testing.T) {
+	defaultValuesYaml := []byte("")
+
+	yamlContent := []byte(`
+environments:
+  staging:
+    values:
+    - default/values.yaml
+  production:
+
+{{ if (eq .Environment.Values.releaseName "a") }} # line 8
+releases:
+- name: a
+	chart: mychart1
+{{ end }}
+`)
+
+	// make a reader that returns a simulated context
+	fileReader := func(filename string) ([]byte, error) {
+		return defaultValuesYaml, nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	// test the double rendering
+	_, err := r.renderTemplate(yamlContent)
+
+	if !strings.Contains(err.Error(), "stringTemplate:8") {
+		t.Fatalf("error should contain a stringTemplate error (reference to unknow key) %v", err)
+	}
+}
+
+// This test shows that a gotmpl reference will get rendered correctly
+// even if the pre-render disables the readFile and exec functions.
+// This does not apply to .gotmpl files, which is a nice side-effect.
+func TestReadFromYaml_RenderTemplateWithGotmpl(t *testing.T) {
+
+	defaultValuesYamlGotmpl := []byte(`
+releaseName: {{ readFile "nonIgnoredFile" }}
+`)
+
+	yamlContent := []byte(`
+environments:
+  staging:
+    values:
+    - values.yaml.gotmpl
+  production:
+
+{{ if (eq .Environment.Values.releaseName "release-a") }} # line 8
+releases:
+- name: a
+  chart: mychart1
+{{ end }}
+`)
+
+	fileReader := func(filename string) ([]byte, error) {
+		if strings.HasSuffix(filename, "nonIgnoredFile") {
+			return []byte("release-a"), nil
+		}
+		return defaultValuesYamlGotmpl, nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	rendered, _ := r.renderTemplate(yamlContent)
+
+	var state state.HelmState
+	yaml.Unmarshal(rendered.Bytes(), &state)
+
+	if len(state.Releases) != 1 {
+		t.Fatal("there should be 1 release")
+	}
+
+	if state.Releases[0].Name != "a" {
+		t.Fatal("release should have been declared")
+	}
+}
+
+func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) {
+	defaultValuesYaml := []byte(``)
+	yamlContent := []byte(`releases:
+- name: {{ .Namespace }}-myrelease
+  chart: mychart
+`)
+
+	// make a reader that returns a simulated context
+	fileReader := func(filename string) ([]byte, error) {
+		return defaultValuesYaml, nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	yamlBuf, err := r.renderTemplate(yamlContent)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	var state state.HelmState
+	err = yaml.Unmarshal(yamlBuf.Bytes(), &state)
+
+	if state.Releases[0].Name != "namespace-myrelease" {
+		t.Errorf("release name should be namespace-myrelease")
+	}
+}
+
+func TestReadFromYaml_HelfileShouldBeResilentToTemplateErrors(t *testing.T) {
+	yamlContent := []byte(`environments:
+  staging:
+	production:
+
+releases:
+{{ if (eq .Environment.Name "production" }}  # notice syntax error: unclosed left paren
+- name: prod-myrelease
+{{ else }}
+- name: myapp
+{{ end }}
+  chart: mychart
+`)
+	fileReader := func(filename string) ([]byte, error) {
+		return yamlContent, nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	_, err := r.renderTemplate(yamlContent)
+	if err == nil {
+		t.Fatalf("wanted error, none returned")
+	}
+}