do not pass --api-versions to "helm diff" (#1061)

Fixes #539

Dockerfile.helm3

@@ -11,10 +11,10 @@ FROM alpine:3.10
 
 RUN apk add --no-cache ca-certificates git bash curl jq
 
-ARG HELM_VERSION=v3.0.0
+ARG HELM_VERSION=v3.0.2
 ARG HELM_LOCATION="https://get.helm.sh"
 ARG HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
-ARG HELM_SHA256="10e1fdcca263062b1d7b2cb93a924be1ef3dd6c381263d8151dd1a20a3d8c0dc"
+ARG HELM_SHA256="c6b7aa7e4ffc66e8abb4be328f71d48c643cb8f398d95c74d075cfb348710e1d"
 RUN wget ${HELM_LOCATION}/${HELM_FILENAME} && \
     echo Verifying ${HELM_FILENAME}... && \
     sha256sum ${HELM_FILENAME} | grep -q "${HELM_SHA256}" && \

Makefile

@@ -26,15 +26,15 @@ integration:
 .PHONY: integration
 
 cross:
-	env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le !s390x' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS}
+	env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le !s390x' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X github.com/roboll/helmfile/pkg/app/version.Version=${TAG}' ${TARGETS}
 .PHONY: cross
 
 static-linux:
-	env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -o "dist/helmfile_linux_amd64" -ldflags '-X main.Version=${TAG}' ${TARGETS}
+	env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -o "dist/helmfile_linux_amd64" -ldflags '-X github.com/roboll/helmfile/pkg/app/version.Version=${TAG}' ${TARGETS}
 .PHONY: static-linux
 
 install:
-	env CGO_ENABLED=0 go install -ldflags '-X main.Version=${TAG}' ${TARGETS}
+	env CGO_ENABLED=0 go install -ldflags '-X github.com/roboll/helmfile/pkg/app/version.Version=${TAG}' ${TARGETS}
 .PHONY: install
 
 clean:

README.md

@@ -74,6 +74,7 @@ helmDefaults:
   tillerNamespace: tiller-namespace  #dedicated default key for tiller-namespace
   tillerless: false                  #dedicated default key for tillerless
   kubeContext: kube-context          #dedicated default key for kube-context (--kube-context)
+  cleanupOnFail: false               #dedicated default key for helm flag --cleanup-on-fail
   # additional and global args passed to helm
   args:
     - "--set k=v"
@@ -91,6 +92,8 @@ helmDefaults:
   tlsCert: "path/to/cert.pem"
   # path to TLS key file (default "$HELM_HOME/key.pem")
   tlsKey: "path/to/key.pem"
+  # limit the maximum number of revisions saved per release. Use 0 for no limit (default 10)
+  historyMax: 10
 
 # The desired states of Helm releases.
 #
@@ -157,6 +160,8 @@ releases:
     installed: true
     # restores previous state in case of failed release
     atomic: true
+    # when true, cleans up any new resources created during a failed release
+    cleanupOnFail: false
     # name of the tiller namespace
     tillerNamespace: vault
     # if true, will use the helm-tiller plugin
@@ -173,6 +178,8 @@ releases:
     # CAUTION: this doesn't work as expected for `tilerless: true`.
     # See https://github.com/roboll/helmfile/issues/642
     kubeContext: kube-context
+    # limit the maximum number of revisions saved per release. Use 0 for no limit (default 10)
+    historyMax: 10
 
   # Local chart example
   - name: grafana                            # name of this release
@@ -259,6 +266,18 @@ bases:
 - environments.yaml
 - defaults.yaml
 - templates.yaml
+
+#
+# Advanced Configuration: API Capabilities
+#
+# 'helmfile template' renders releases locally without querying an actual cluster,
+# and in this case `.Capabilities.APIVersions` cannot be populated.
+# When a chart queries for a specific CRD, this can lead to unexpected results.
+# 
+# Configure a fixed list of api versions to pass to 'helm template' via the --api-versions flag:
+apiVersions:
+- example/v1
+
 ```
 
 ## Templating
@@ -331,7 +350,7 @@ Congratulations! You now have your first Prometheus deployment running inside yo
 
 Iterate on the `helmfile.yaml` by referencing:
 
-- [Configuration syntax](#configuration-syntax)
+- [Configuration](#configuration)
 - [CLI reference](#cli-reference).
 - [Helmfile Best Practices Guide](https://github.com/roboll/helmfile/blob/master/docs/writing-helmfile.md)
 
@@ -732,7 +751,7 @@ or in the `releases` entries.
 `needs` controls the order of the installation/deletion of the release:
 
 ```yaml
-relesaes:
+releases:
 - name: somerelease
   needs:
   - [TILLER_NAMESPACE/][NAMESPACE/]anotherelease
@@ -767,7 +786,7 @@ For the following example, `helmfile [sync|apply]` installs releases in this ord
 
 Note that all the releases in a same group is installed concurrently. That is, myapp1 and myapp2 are installed concurrently.
 
-On `helmdile [delete|destroy]`, deleations happen in the reverse order.
+On `helmfile [delete|destroy]`, deletions happen in the reverse order.
 
 That is, `myapp1` and `myapp2` are deleted first, then `servicemesh`, and finally `logging`.
 

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/gosuri/uitable v0.0.3
 	github.com/hashicorp/go-getter v1.3.0
 	github.com/hashicorp/go-retryablehttp v0.6.3 // indirect
+	github.com/hashicorp/go-version v1.2.0
 	github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c // indirect
 	github.com/huandu/xstrings v1.2.0 // indirect
 	github.com/imdario/mergo v0.3.6
@@ -27,7 +28,7 @@ require (
 	github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939
 	github.com/urfave/cli v1.20.0
 	github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363
-	github.com/variantdev/vals v0.0.0-20191119133107-c90d858605ee
+	github.com/variantdev/vals v0.0.0-20191123015158-21b16052816c
 	go.mozilla.org/sops v0.0.0-20190912205235-14a22d7a7060 // indirect
 	go.opencensus.io v0.22.1 // indirect
 	go.uber.org/multierr v1.1.0

go.sum

@@ -732,6 +732,10 @@ github.com/variantdev/vals v0.0.0-20191117001227-ba4c7a29875a h1:Ce3KCr4PAC1qdEy
 github.com/variantdev/vals v0.0.0-20191117001227-ba4c7a29875a/go.mod h1:8CW8eonQlIJgAjF1fLfrkaBe16fGjwGf2PX52/Vualw=
 github.com/variantdev/vals v0.0.0-20191119133107-c90d858605ee h1:kiXVEsU6gfSuEumQ95YFvIIhwUomOiG238PP/Hz3tZc=
 github.com/variantdev/vals v0.0.0-20191119133107-c90d858605ee/go.mod h1:8CW8eonQlIJgAjF1fLfrkaBe16fGjwGf2PX52/Vualw=
+github.com/variantdev/vals v0.0.0-20191121120813-de375c5be687 h1:Cf/Krc6gd8Yd4EuRJ7aeaOnqpwvglGW/Mm6651+6W5M=
+github.com/variantdev/vals v0.0.0-20191121120813-de375c5be687/go.mod h1:6Ma8+ZOzCxDY5cBSaoRQqE3E5IBaD9BYLfdUMmSsgHg=
+github.com/variantdev/vals v0.0.0-20191123015158-21b16052816c h1:iuTwDEIPEAXk0jlO4QNhQR84QC3Wq/dIZ4PIOi1nuZM=
+github.com/variantdev/vals v0.0.0-20191123015158-21b16052816c/go.mod h1:6Ma8+ZOzCxDY5cBSaoRQqE3E5IBaD9BYLfdUMmSsgHg=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -937,6 +941,7 @@ google.golang.org/api v0.11.0 h1:n/qM3q0/rV2F0pox7o0CvNhlPvZAo7pLbef122cbLJ0=
 google.golang.org/api v0.11.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA=
 google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0 h1:uMf5uLi4eQMRrMKhCplNik4U4H8Z6C1br3zOtAa/aDE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=

main.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/roboll/helmfile/pkg/app/version"
+
 	"github.com/roboll/helmfile/pkg/app"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/maputil"
@@ -14,8 +16,6 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
-var Version string
-
 var logger *zap.SugaredLogger
 
 func configureLogging(c *cli.Context) error {
@@ -45,7 +45,7 @@ func main() {
 	cliApp := cli.NewApp()
 	cliApp.Name = "helmfile"
 	cliApp.Usage = ""
-	cliApp.Version = Version
+	cliApp.Version = version.Version
 	cliApp.Flags = []cli.Flag{
 		cli.StringFlag{
 			Name:  "helm-binary, b",
@@ -484,10 +484,10 @@ func NewUrfaveCliConfigImpl(c *cli.Context) (configImpl, error) {
 			ops := strings.Split(optsSet[i], ",")
 			for j := range ops {
 				op := strings.SplitN(ops[j], "=", 2)
-				k := strings.Split(op[0], ".")
+				k := maputil.ParseKey(op[0])
 				v := op[1]
 
-				set = maputil.Set(set, k, v)
+				maputil.Set(set, k, v)
 			}
 		}
 		conf.set = set

pkg/app/app_test.go

@@ -177,6 +177,42 @@ releases:
 	}
 }
 
+func TestVisitDesiredStatesWithReleasesFiltered_Issue1008_MissingNonDefaultEnvInBase(t *testing.T) {
+	files := map[string]string{
+		"/path/to/base.yaml": `
+helmDefaults:
+  wait: true
+`,
+		"/path/to/helmfile.yaml": `
+bases:
+- base.yaml
+environments:
+  test:
+releases:
+- name: zipkin
+  chart: stable/zipkin
+`,
+	}
+	fs := testhelper.NewTestFs(files)
+	app := &App{
+		KubeContext: "default",
+		Logger:      helmexec.NewLogger(os.Stderr, "debug"),
+		Namespace:   "",
+		Env:         "test",
+	}
+	app = injectFs(app, fs)
+	noop := func(st *state.HelmState, helm helmexec.Interface) []error {
+		return []error{}
+	}
+
+	err := app.VisitDesiredStatesWithReleasesFiltered(
+		"helmfile.yaml", noop,
+	)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+}
+
 func TestVisitDesiredStatesWithReleasesFiltered_MissingEnvValuesFileHandler(t *testing.T) {
 	testcases := []struct {
 		name        string
@@ -2056,6 +2092,64 @@ releases:
 	}
 }
 
+func TestTemplate_ApiVersions(t *testing.T) {
+	files := map[string]string{
+		"/path/to/helmfile.yaml": `
+apiVersions:
+- helmfile.test/v1
+- helmfile.test/v2
+releases:
+- name: myrelease1
+  chart: mychart1
+`,
+	}
+
+	var helm = &mockHelmExec{}
+	var wantReleases = []mockTemplates{
+		{name: "myrelease1", chart: "mychart1", flags: []string{"--api-versions", "helmfile.test/v1", "--api-versions", "helmfile.test/v2", "--namespace", "testNamespace", "--output-dir", "output/subdir/helmfile-[a-z0-9]{8}-myrelease1"}},
+	}
+
+	var buffer bytes.Buffer
+	logger := helmexec.NewLogger(&buffer, "debug")
+
+	valsRuntime, err := vals.New(vals.Options{CacheSize: 32})
+	if err != nil {
+		t.Errorf("unexpected error creating vals runtime: %v", err)
+	}
+
+	app := appWithFs(&App{
+		glob:        filepath.Glob,
+		abs:         filepath.Abs,
+		KubeContext: "default",
+		Env:         "default",
+		Logger:      logger,
+		helmExecer:  helm,
+		Namespace:   "testNamespace",
+		valsRuntime: valsRuntime,
+	}, files)
+	app.Template(configImpl{})
+
+	for i := range wantReleases {
+		if wantReleases[i].name != helm.templated[i].name {
+			t.Errorf("name = [%v], want %v", helm.templated[i].name, wantReleases[i].name)
+		}
+		if !strings.Contains(helm.templated[i].chart, wantReleases[i].chart) {
+			t.Errorf("chart = [%v], want %v", helm.templated[i].chart, wantReleases[i].chart)
+		}
+		for j := range wantReleases[i].flags {
+			if j == 7 {
+				matched, _ := regexp.Match(wantReleases[i].flags[j], []byte(helm.templated[i].flags[j]))
+				if !matched {
+					t.Errorf("HelmState.TemplateReleases() = [%v], want %v", helm.templated[i].flags[j], wantReleases[i].flags[j])
+				}
+			} else if wantReleases[i].flags[j] != helm.templated[i].flags[j] {
+				t.Errorf("HelmState.TemplateReleases() = [%v], want %v", helm.templated[i].flags[j], wantReleases[i].flags[j])
+			}
+		}
+
+	}
+}
+
 func TestApply(t *testing.T) {
 	testcases := []struct {
 		name        string
@@ -2149,18 +2243,18 @@ releases:
 			},
 			lists: map[exectest.ListKey]string{
 				// delete frontend-v1 and backend-v1
-				exectest.ListKey{Filter: "^frontend-v1$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^frontend-v1$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 frontend-v1 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	backend-3.1.0	3.1.0      	default
 `,
-				exectest.ListKey{Filter: "^backend-v1$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^backend-v1$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 backend-v1 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	backend-3.1.0	3.1.0      	default
 `,
 			},
 			// Disable concurrency to avoid in-deterministic result
 			concurrency: 1,
 			upgraded: []exectest.Release{
-				{Name: "front-proxy", Flags: []string{}},
 				{Name: "logging", Flags: []string{}},
+				{Name: "front-proxy", Flags: []string{}},
 				{Name: "database", Flags: []string{}},
 				{Name: "servicemesh", Flags: []string{}},
 				{Name: "anotherbackend", Flags: []string{}},
@@ -2314,7 +2408,7 @@ GROUP RELEASES
 2     backend-v1, backend-v2
 3     anotherbackend
 4     database, servicemesh
-5     front-proxy, logging
+5     logging, front-proxy
 
 processing releases in group 1/5: frontend-v1, frontend-v2, frontend-v3
 worker 1/1 started
@@ -2324,52 +2418,52 @@ worker 1/1 started
 worker 1/1 finished
 processing releases in group 3/5: anotherbackend
 processing releases in group 4/5: database, servicemesh
-processing releases in group 5/5: front-proxy, logging
+processing releases in group 5/5: logging, front-proxy
 processing 5 groups of releases in this order:
 GROUP RELEASES
-1     front-proxy, logging
+1     logging, front-proxy
 2     database, servicemesh
 3     anotherbackend
 4     backend-v1, backend-v2
 5     frontend-v1, frontend-v2, frontend-v3
 
-processing releases in group 1/5: front-proxy, logging
+processing releases in group 1/5: logging, front-proxy
 worker 1/1 started
 worker 1/1 finished
 worker 1/1 started
-getting deployed release version failed:unexpected list key: {^front-proxy$ --kube-contextdefault}
-getting deployed release version failed:unexpected list key: {^logging$ --kube-contextdefault}
+getting deployed release version failed:unexpected list key: {^logging$ --kube-contextdefault--deployed--failed--pending}
+getting deployed release version failed:unexpected list key: {^front-proxy$ --kube-contextdefault--deployed--failed--pending}
 worker 1/1 finished
 processing releases in group 2/5: database, servicemesh
 worker 1/1 started
 worker 1/1 finished
 worker 1/1 started
-getting deployed release version failed:unexpected list key: {^database$ --kube-contextdefault}
-getting deployed release version failed:unexpected list key: {^servicemesh$ --kube-contextdefault}
+getting deployed release version failed:unexpected list key: {^database$ --kube-contextdefault--deployed--failed--pending}
+getting deployed release version failed:unexpected list key: {^servicemesh$ --kube-contextdefault--deployed--failed--pending}
 worker 1/1 finished
 processing releases in group 3/5: anotherbackend
 worker 1/1 started
 worker 1/1 finished
 worker 1/1 started
-getting deployed release version failed:unexpected list key: {^anotherbackend$ --kube-contextdefault}
+getting deployed release version failed:unexpected list key: {^anotherbackend$ --kube-contextdefault--deployed--failed--pending}
 worker 1/1 finished
 processing releases in group 4/5: backend-v1, backend-v2
 worker 1/1 started
 worker 1/1 finished
 worker 1/1 started
-getting deployed release version failed:unexpected list key: {^backend-v2$ --kube-contextdefault}
+getting deployed release version failed:unexpected list key: {^backend-v2$ --kube-contextdefault--deployed--failed--pending}
 worker 1/1 finished
 processing releases in group 5/5: frontend-v1, frontend-v2, frontend-v3
 worker 1/1 started
 worker 1/1 finished
 worker 1/1 started
-getting deployed release version failed:unexpected list key: {^frontend-v3$ --kube-contextdefault}
+getting deployed release version failed:unexpected list key: {^frontend-v3$ --kube-contextdefault--deployed--failed--pending}
 worker 1/1 finished
 
 UPDATED RELEASES:
 NAME             CHART                   VERSION
-front-proxy      stable/envoy                   
 logging          charts/fluent-bit              
+front-proxy      stable/envoy                   
 database         charts/mysql                   
 servicemesh      charts/istio                   
 anotherbackend   charts/anotherbackend          
@@ -2405,8 +2499,8 @@ releases:
 				exectest.DiffKey{Name: "bar", Chart: "mychart2", Flags: "--kube-contextdefault--detailed-exitcode"}: nil,
 			},
 			lists: map[exectest.ListKey]string{
-				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault"}: ``,
-				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault--deployed--failed--pending"}: ``,
+				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 bar 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart2-3.1.0	3.1.0      	default
 `,
 			},
@@ -2439,8 +2533,8 @@ releases:
 			},
 			lists: map[exectest.ListKey]string{},
 			upgraded: []exectest.Release{
-				{Name: "bar", Flags: []string{}},
 				{Name: "baz", Flags: []string{}},
+				{Name: "bar", Flags: []string{}},
 				{Name: "foo", Flags: []string{}},
 			},
 			deleted:     []exectest.Release{},
@@ -2491,27 +2585,27 @@ Affected releases are:
 
 processing 2 groups of releases in this order:
 GROUP RELEASES
-1     bar, baz
+1     baz, bar
 2     foo
 
-processing releases in group 1/2: bar, baz
+processing releases in group 1/2: baz, bar
 worker 1/1 started
 worker 1/1 finished
 worker 1/1 started
-getting deployed release version failed:unexpected list key: {^bar$ --kube-contextdefault}
-getting deployed release version failed:unexpected list key: {^baz$ --kube-contextdefault}
+getting deployed release version failed:unexpected list key: {^baz$ --kube-contextdefault--deployed--failed--pending}
+getting deployed release version failed:unexpected list key: {^bar$ --kube-contextdefault--deployed--failed--pending}
 worker 1/1 finished
 processing releases in group 2/2: foo
 worker 1/1 started
 worker 1/1 finished
 worker 1/1 started
-getting deployed release version failed:unexpected list key: {^foo$ --kube-contextdefault}
+getting deployed release version failed:unexpected list key: {^foo$ --kube-contextdefault--deployed--failed--pending}
 worker 1/1 finished
 
 UPDATED RELEASES:
 NAME   CHART      VERSION
-bar    mychart2          
 baz    mychart3          
+bar    mychart2          
 foo    mychart1          
 
 `,
@@ -2775,13 +2869,13 @@ processing releases in group 1/2: tns1/ns1/foo
 worker 1/1 started
 worker 1/1 finished
 worker 1/1 started
-getting deployed release version failed:unexpected list key: {^foo$ --tiller-namespacetns1--kube-contextdefault}
+getting deployed release version failed:unexpected list key: {^foo$ --tiller-namespacetns1--kube-contextdefault--deployed--failed--pending}
 worker 1/1 finished
 processing releases in group 2/2: tns2/ns2/bar
 worker 1/1 started
 worker 1/1 finished
 worker 1/1 started
-getting deployed release version failed:unexpected list key: {^bar$ --tiller-namespacetns2--kube-contextdefault}
+getting deployed release version failed:unexpected list key: {^bar$ --tiller-namespacetns2--kube-contextdefault--deployed--failed--pending}
 worker 1/1 finished
 
 UPDATED RELEASES:
@@ -2815,10 +2909,10 @@ releases:
 				exectest.DiffKey{Name: "foo", Chart: "mychart1", Flags: "--kube-contextdefault--detailed-exitcode"}: helmexec.ExitError{Code: 2},
 			},
 			lists: map[exectest.ListKey]string{
-				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 foo 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart1-3.1.0	3.1.0      	default
 `,
-				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 bar 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart2-3.1.0	3.1.0      	default
 `,
 			},
@@ -2848,10 +2942,10 @@ releases:
 				exectest.DiffKey{Name: "foo", Chart: "mychart1", Flags: "--kube-contextdefault--detailed-exitcode"}: helmexec.ExitError{Code: 2},
 			},
 			lists: map[exectest.ListKey]string{
-				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 foo 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart1-3.1.0	3.1.0      	default
 `,
-				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 bar 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart2-3.1.0	3.1.0      	default
 `,
 			},
@@ -2883,10 +2977,10 @@ releases:
 				exectest.DiffKey{Name: "foo", Chart: "mychart1", Flags: "--kube-contextdefault--detailed-exitcode"}: helmexec.ExitError{Code: 2},
 			},
 			lists: map[exectest.ListKey]string{
-				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 foo 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart1-3.1.0	3.1.0      	default
 `,
-				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 bar 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart2-3.1.0	3.1.0      	default
 `,
 			},
@@ -2917,10 +3011,10 @@ releases:
 				exectest.DiffKey{Name: "foo", Chart: "mychart1", Flags: "--kube-contextdefault--detailed-exitcode"}: helmexec.ExitError{Code: 2},
 			},
 			lists: map[exectest.ListKey]string{
-				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 foo 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart1-3.1.0	3.1.0      	default
 `,
-				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 bar 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart2-3.1.0	3.1.0      	default
 `,
 			},
@@ -2951,10 +3045,10 @@ releases:
 				exectest.DiffKey{Name: "foo", Chart: "mychart1", Flags: "--kube-contextdefault--detailed-exitcode"}: helmexec.ExitError{Code: 2},
 			},
 			lists: map[exectest.ListKey]string{
-				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 foo 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart1-3.1.0	3.1.0      	default
 `,
-				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 bar 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart2-3.1.0	3.1.0      	default
 `,
 			},
@@ -2985,10 +3079,10 @@ releases:
 				exectest.DiffKey{Name: "foo", Chart: "mychart1", Flags: "--kube-contextdefault--detailed-exitcode"}: helmexec.ExitError{Code: 2},
 			},
 			lists: map[exectest.ListKey]string{
-				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^foo$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 foo 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart1-3.1.0	3.1.0      	default
 `,
-				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+				exectest.ListKey{Filter: "^bar$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
 bar 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mychart2-3.1.0	3.1.0      	default
 `,
 			},
@@ -3127,7 +3221,7 @@ processing releases in group 2/3: default/external-secrets
 worker 1/1 started
 worker 1/1 finished
 worker 1/1 started
-getting deployed release version failed:unexpected list key: {^external-secrets$ --kube-contextdefault}
+getting deployed release version failed:unexpected list key: {^external-secrets$ --kube-contextdefault--deployed--failed--pending}
 worker 1/1 finished
 processing releases in group 3/3: default/my-release
 1 release(s) matching app=test found in helmfile.yaml
@@ -3135,7 +3229,7 @@ processing releases in group 3/3: default/my-release
 worker 1/1 started
 worker 1/1 finished
 worker 1/1 started
-getting deployed release version failed:unexpected list key: {^my-release$ --kube-contextdefault}
+getting deployed release version failed:unexpected list key: {^my-release$ --kube-contextdefault--deployed--failed--pending}
 worker 1/1 finished
 
 UPDATED RELEASES:

pkg/app/desired_state_file_loader.go

@@ -126,7 +126,20 @@ func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *e
 		)
 	}
 
-	return self, err
+	if err != nil {
+		return nil, err
+	}
+
+	for i, h := range self.Helmfiles {
+		if h.Path == f {
+			return nil, fmt.Errorf("%s contains a recursion into the same sub-helmfile at helmfiles[%d]", f, i)
+		}
+		if h.Path == "." {
+			return nil, fmt.Errorf("%s contains a recursion into the the directory containing this helmfile at helmfiles[%d]", f, i)
+		}
+	}
+
+	return self, nil
 }
 
 func (a *desiredStateLoader) underlying() *state.StateCreator {

pkg/app/version/version.go

@@ -0,0 +1,3 @@
+package version
+
+var Version string

pkg/helmexec/context.go

@@ -8,15 +8,16 @@ import (
 type HelmContext struct {
 	Tillerless      bool
 	TillerNamespace string
+	HistoryMax      int
 	WorkerIndex     int
 }
 
-func (context *HelmContext) GetTillerlessArgs(helmBinary string) []string {
-	if context.Tillerless {
+func (context *HelmContext) GetTillerlessArgs(helm *execer) []string {
+	if context.Tillerless && !helm.IsHelm3() {
 		if context.TillerNamespace != "" {
-			return []string{"tiller", "run", context.TillerNamespace, "--", helmBinary}
+			return []string{"tiller", "run", context.TillerNamespace, "--", helm.helmBinary}
 		} else {
-			return []string{"tiller", "run", "--", helmBinary}
+			return []string{"tiller", "run", "--", helm.helmBinary}
 		}
 	} else {
 		return []string{}

pkg/helmexec/exec.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -122,8 +123,15 @@ func (helm *execer) UpdateDeps(chart string) error {
 
 func (helm *execer) SyncRelease(context HelmContext, name, chart string, flags ...string) error {
 	helm.logger.Infof("Upgrading release=%v, chart=%v", name, chart)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
+
+	if helm.IsHelm3() {
+		flags = append(flags, "--history-max", strconv.Itoa(context.HistoryMax))
+	} else {
+		env["HELM_TILLER_HISTORY_MAX"] = strconv.Itoa(context.HistoryMax)
+	}
+
 	out, err := helm.exec(append(append(preArgs, "upgrade", "--install", "--reset-values", name, chart), flags...), env)
 	helm.write(out)
 	return err
@@ -131,7 +139,7 @@ func (helm *execer) SyncRelease(context HelmContext, name, chart string, flags .
 
 func (helm *execer) ReleaseStatus(context HelmContext, name string, flags ...string) error {
 	helm.logger.Infof("Getting status %v", name)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
 	out, err := helm.exec(append(append(preArgs, "status", name), flags...), env)
 	helm.write(out)
@@ -140,7 +148,7 @@ func (helm *execer) ReleaseStatus(context HelmContext, name string, flags ...str
 
 func (helm *execer) List(context HelmContext, filter string, flags ...string) (string, error) {
 	helm.logger.Infof("Listing releases matching %v", filter)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
 	var args []string
 	if helm.IsHelm3() {
@@ -187,7 +195,7 @@ func (helm *execer) DecryptSecret(context HelmContext, name string, flags ...str
 		helm.decryptedSecretMutex.Unlock()
 
 		helm.logger.Infof("Decrypting secret %v", absPath)
-		preArgs := context.GetTillerlessArgs(helm.helmBinary)
+		preArgs := context.GetTillerlessArgs(helm)
 		env := context.getTillerlessEnv()
 		out, err := helm.exec(append(append(preArgs, "secrets", "dec", absPath), flags...), env)
 		helm.info(out)
@@ -249,7 +257,7 @@ func (helm *execer) TemplateRelease(name string, chart string, flags ...string)
 
 func (helm *execer) DiffRelease(context HelmContext, name, chart string, flags ...string) error {
 	helm.logger.Infof("Comparing release=%v, chart=%v", name, chart)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
 	out, err := helm.exec(append(append(preArgs, "diff", "upgrade", "--reset-values", "--allow-unreleased", name, chart), flags...), env)
 	// Do our best to write STDOUT only when diff existed
@@ -291,7 +299,7 @@ func (helm *execer) Fetch(chart string, flags ...string) error {
 
 func (helm *execer) DeleteRelease(context HelmContext, name string, flags ...string) error {
 	helm.logger.Infof("Deleting %v", name)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
 	out, err := helm.exec(append(append(preArgs, "delete", name), flags...), env)
 	helm.write(out)
@@ -300,14 +308,9 @@ func (helm *execer) DeleteRelease(context HelmContext, name string, flags ...str
 
 func (helm *execer) TestRelease(context HelmContext, name string, flags ...string) error {
 	helm.logger.Infof("Testing %v", name)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
-	var args []string
-	if helm.IsHelm3() {
-		args = []string{"test", "run", name}
-	} else {
-		args = []string{"test", name}
-	}
+	args := []string{"test", name}
 	out, err := helm.exec(append(append(preArgs, args...), flags...), env)
 	helm.write(out)
 	return err

pkg/maputil/maputil.go

@@ -1,6 +1,9 @@
 package maputil
 
-import "fmt"
+import (
+	"fmt"
+	"strings"
+)
 
 func CastKeysToStrings(s interface{}) (map[string]interface{}, error) {
 	new := map[string]interface{}{}
@@ -60,32 +63,129 @@ func recursivelyStringifyMapKey(v interface{}) (interface{}, error) {
 	return casted_v, nil
 }
 
-func Set(m map[string]interface{}, key []string, value string) map[string]interface{} {
+type arg interface {
+	getMap(map[string]interface{}) map[string]interface{}
+	set(map[string]interface{}, string)
+}
+
+type keyArg struct {
+	key string
+}
+
+func (a keyArg) getMap(m map[string]interface{}) map[string]interface{} {
+	_, ok := m[a.key]
+	if !ok {
+		m[a.key] = map[string]interface{}{}
+	}
+	switch t := m[a.key].(type) {
+	case map[string]interface{}:
+		return t
+	default:
+		panic(fmt.Errorf("unexpected type: %v(%T)", t, t))
+	}
+}
+
+func (a keyArg) set(m map[string]interface{}, value string) {
+	m[a.key] = value
+}
+
+type indexedKeyArg struct {
+	key   string
+	index int
+}
+
+func (a indexedKeyArg) getArray(m map[string]interface{}) []interface{} {
+	_, ok := m[a.key]
+	if !ok {
+		m[a.key] = make([]interface{}, a.index+1)
+	}
+	switch t := m[a.key].(type) {
+	case []interface{}:
+		if len(t) <= a.index {
+			t2 := make([]interface{}, a.index+1)
+			copy(t, t2)
+			t = t2
+		}
+		return t
+	default:
+		panic(fmt.Errorf("unexpected type: %v(%T)", t, t))
+	}
+}
+
+func (a indexedKeyArg) getMap(m map[string]interface{}) map[string]interface{} {
+	t := a.getArray(m)
+	if t[a.index] == nil {
+		t[a.index] = map[string]interface{}{}
+	}
+	switch t := t[a.index].(type) {
+	case map[string]interface{}:
+		return t
+	default:
+		panic(fmt.Errorf("unexpected type: %v(%T)", t, t))
+	}
+}
+
+func (a indexedKeyArg) set(m map[string]interface{}, value string) {
+	t := a.getArray(m)
+	t[a.index] = value
+	m[a.key] = t
+}
+
+func getCursor(key string) arg {
+	key = strings.ReplaceAll(key, "[", " ")
+	key = strings.ReplaceAll(key, "]", " ")
+	k := key
+	idx := 0
+
+	n, err := fmt.Sscanf(key, "%s %d", &k, &idx)
+
+	if n == 2 && err == nil {
+		return indexedKeyArg{
+			key:   k,
+			index: idx,
+		}
+	}
+
+	return keyArg{
+		key: key,
+	}
+}
+
+func ParseKey(key string) []string {
+	r := []string{}
+	part := ""
+	escaped := false
+	for _, rune := range key {
+		split := false
+		switch {
+		case escaped == false && rune == '\\':
+			escaped = true
+			continue
+		case rune == '.':
+			split = escaped == false
+		}
+		escaped = false
+		if split {
+			r = append(r, part)
+			part = ""
+		} else {
+			part = part + string(rune)
+		}
+	}
+	if len(part) > 0 {
+		r = append(r, part)
+	}
+	return r
+}
+
+func Set(m map[string]interface{}, key []string, value string) {
 	if len(key) == 0 {
 		panic(fmt.Errorf("bug: unexpected length of key: %d", len(key)))
 	}
 
-	k := key[0]
-
-	if len(key) == 1 {
-		m[k] = value
-		return m
+	for len(key) > 1 {
+		m, key = getCursor(key[0]).getMap(m), key[1:]
 	}
 
-	remain := key[1:]
-
-	nested, ok := m[k]
-	if !ok {
-		nested = map[string]interface{}{}
-	}
-	switch t := nested.(type) {
-	case map[string]interface{}:
-		nested = Set(t, remain, value)
-	default:
-		panic(fmt.Errorf("unexpected type: %v(%T)", t, t))
-	}
-
-	m[k] = nested
-
-	return m
+	getCursor(key[0]).set(m, value)
 }

pkg/maputil/maputil_test.go

@@ -59,3 +59,72 @@ func TestMapUtil_IFKeys(t *testing.T) {
 		t.Errorf("unexpected c: expected=C, got=%s", c)
 	}
 }
+
+func TestMapUtil_KeyArg(t *testing.T) {
+	m := map[string]interface{}{}
+
+	key := []string{"a", "b", "c"}
+
+	Set(m, key, "C")
+
+	c := (((m["a"].(map[string]interface{}))["b"]).(map[string]interface{}))["c"]
+
+	if c != "C" {
+		t.Errorf("unexpected c: expected=C, got=%s", c)
+	}
+}
+
+func TestMapUtil_IndexedKeyArg(t *testing.T) {
+	m := map[string]interface{}{}
+
+	key := []string{"a", "b[0]", "c"}
+
+	Set(m, key, "C")
+
+	c := (((m["a"].(map[string]interface{}))["b"].([]interface{}))[0].(map[string]interface{}))["c"]
+
+	if c != "C" {
+		t.Errorf("unexpected c: expected=C, got=%s", c)
+	}
+}
+
+type parseKeyTc struct {
+	key    string
+	result map[int]string
+}
+
+func TestMapUtil_ParseKey(t *testing.T) {
+	tcs := []parseKeyTc{
+		{
+			key: `a.b.c`,
+			result: map[int]string{
+				0: "a",
+				1: "b",
+				2: "c",
+			},
+		},
+		{
+			key: `a\.b.c`,
+			result: map[int]string{
+				0: "a.b",
+				1: "c",
+			},
+		},
+		{
+			key: `a\.b\.c`,
+			result: map[int]string{
+				0: "a.b.c",
+			},
+		},
+	}
+
+	for _, tc := range tcs {
+		parts := ParseKey(tc.key)
+
+		for index, value := range tc.result {
+			if parts[index] != value {
+				t.Errorf("unexpected key part[%d]: expected=%s, got=%s", index, value, parts[index])
+			}
+		}
+	}
+}

pkg/state/chart_dependency.go

@@ -3,7 +3,9 @@ package state
 import (
 	"fmt"
 	"github.com/Masterminds/semver"
+	goversion "github.com/hashicorp/go-version"
 	"github.com/r3labs/diff"
+	"github.com/roboll/helmfile/pkg/app/version"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
@@ -50,6 +52,7 @@ type ChartRequirements struct {
 }
 
 type ChartLockedRequirements struct {
+	Version              string                    `yaml:"version"`
 	ResolvedDependencies []ResolvedChartDependency `yaml:"dependencies"`
 	Digest               string                    `yaml:"digest"`
 	Generated            string                    `yaml:"generated"`
@@ -288,7 +291,7 @@ func (m *chartDependencyManager) Update(shell helmexec.DependencyUpdater, wd str
 
 func (m *chartDependencyManager) updateHelm3(shell helmexec.DependencyUpdater, wd string, unresolved *UnresolvedDependencies) (*ResolvedDependencies, error) {
 	// Generate `Chart.yaml` of the temporary local chart
-	chartMetaContent := fmt.Sprintf("name: %s\nversion: 1.0.0\n", m.Name)
+	chartMetaContent := fmt.Sprintf("name: %s\nversion: 1.0.0\napiVersion: v2\n", m.Name)
 
 	// Generate `requirements.yaml` of the temporary local chart from the helmfile state
 	reqsContent, err := yaml.Marshal(unresolved.ToChartRequirements())
@@ -374,6 +377,8 @@ func (m *chartDependencyManager) doUpdate(chartLockFile string, unresolved *Unre
 		}
 	}
 
+	lockedReqs.Version = version.Version
+
 	updatedLockFileContent, err = yaml.Marshal(lockedReqs)
 
 	if err != nil {
@@ -404,6 +409,25 @@ func (m *chartDependencyManager) Resolve(unresolved *UnresolvedDependencies) (*R
 		return nil, false, err
 	}
 
+	// Make sure go run main.go works and compatible with old lock files.
+	if version.Version != "" && lockedReqs.Version != "" {
+		lockedVersion, err := goversion.NewVersion(lockedReqs.Version)
+
+		if err != nil {
+			return nil, false, err
+		}
+
+		currentVersion, err := goversion.NewVersion(version.Version)
+
+		if err != nil {
+			return nil, false, err
+		}
+
+		if currentVersion.LessThan(lockedVersion) {
+			return nil, false, fmt.Errorf("the lockfile was created by Helmfile %s, which is newer than current %s; Please upgrade to Helmfile %s or greater", lockedVersion.Original(), currentVersion.Original(), lockedVersion.Original())
+		}
+	}
+
 	resolved := &ResolvedDependencies{deps: map[string][]ResolvedChartDependency{}}
 	for _, d := range lockedReqs.ResolvedDependencies {
 		if err := resolved.add(d); err != nil {

pkg/state/create.go

@@ -116,10 +116,10 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState,
 }
 
 // LoadEnvValues loads environment values files relative to the `baseDir`
-func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *environment.Environment) (*HelmState, error) {
+func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *environment.Environment, failOnMissingEnv bool) (*HelmState, error) {
 	state := *target
 
-	e, err := state.loadEnvValues(env, ctxEnv, c.readFile, c.glob)
+	e, err := state.loadEnvValues(env, failOnMissingEnv, ctxEnv, c.readFile, c.glob)
 	if err != nil {
 		return nil, &StateLoadError{fmt.Sprintf("failed to read %s", state.FilePath), err}
 	}
@@ -135,6 +135,7 @@ func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *envi
 }
 
 // Parses YAML into HelmState, while loading environment values files relative to the `baseDir`
+// evaluateBases=true means that this is NOT a base helmfile
 func (c *StateCreator) ParseAndLoad(content []byte, baseDir, file string, envName string, evaluateBases bool, envValues *environment.Environment) (*HelmState, error) {
 	state, err := c.Parse(content, baseDir, file)
 	if err != nil {
@@ -145,14 +146,14 @@ func (c *StateCreator) ParseAndLoad(content []byte, baseDir, file string, envNam
 		if len(state.Bases) > 0 {
 			return nil, errors.New("nested `base` helmfile is unsupported. please submit a feature request if you need this!")
 		}
+	} else {
+		state, err = c.loadBases(envValues, state, baseDir)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	state, err = c.loadBases(envValues, state, baseDir)
-	if err != nil {
-		return nil, err
-	}
-
-	state, err = c.LoadEnvValues(state, envName, envValues)
+	state, err = c.LoadEnvValues(state, envName, envValues, evaluateBases)
 	if err != nil {
 		return nil, err
 	}
@@ -182,7 +183,7 @@ func (c *StateCreator) loadBases(envValues *environment.Environment, st *HelmSta
 	return layers[0], nil
 }
 
-func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, readFile func(string) ([]byte, error), glob func(string) ([]string, error)) (*environment.Environment, error) {
+func (st *HelmState) loadEnvValues(name string, failOnMissingEnv bool, ctxEnv *environment.Environment, readFile func(string) ([]byte, error), glob func(string) ([]string, error)) (*environment.Environment, error) {
 	envVals := map[string]interface{}{}
 	envSpec, ok := st.Environments[name]
 	if ok {
@@ -210,7 +211,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 				return nil, err
 			}
 		}
-	} else if ctxEnv == nil && name != DefaultEnv {
+	} else if ctxEnv == nil && name != DefaultEnv && failOnMissingEnv {
 		return nil, &UndefinedEnvError{msg: fmt.Sprintf("environment \"%s\" is not defined", name)}
 	}
 

pkg/state/create_test.go

@@ -20,7 +20,7 @@ func createFromYaml(content []byte, file string, env string, logger *zap.Sugared
 		abs:      filepath.Abs,
 		Strict:   true,
 	}
-	return c.ParseAndLoad(content, filepath.Dir(file), file, env, false, nil)
+	return c.ParseAndLoad(content, filepath.Dir(file), file, env, true, nil)
 }
 
 func TestReadFromYaml(t *testing.T) {
@@ -108,7 +108,7 @@ bar: {{ readFile "bar.txt" }}
 	})
 	testFs.Cwd = "/example/path/to"
 
-	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil, nil).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", false, nil)
+	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil, nil).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", true, nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}

pkg/state/state.go

@@ -47,6 +47,7 @@ type HelmState struct {
 	Repositories       []RepositorySpec  `yaml:"repositories,omitempty"`
 	Releases           []ReleaseSpec     `yaml:"releases,omitempty"`
 	Selectors          []string          `yaml:"-"`
+	ApiVersions        []string          `yaml:"apiVersions,omitempty"`
 
 	Templates map[string]TemplateSpec `yaml:"templates"`
 
@@ -101,6 +102,10 @@ type HelmSpec struct {
 	Force bool `yaml:"force"`
 	// Atomic, when set to true, restore previous state in case of a failed install/upgrade attempt
 	Atomic bool `yaml:"atomic"`
+	// CleanupOnFail, when set to true, the --cleanup-on-fail helm flag is passed to the upgrade command
+	CleanupOnFail bool `yaml:"cleanupOnFail,omitempty"`
+	// HistoryMax, limit the maximum number of revisions saved per release. Use 0 for no limit (default 10)
+	HistoryMax *int `yaml:"historyMax,omitempty"`
 
 	TLS       bool   `yaml:"tls"`
 	TLSCACert string `yaml:"tlsCACert,omitempty"`
@@ -139,6 +144,10 @@ type ReleaseSpec struct {
 	Installed *bool `yaml:"installed,omitempty"`
 	// Atomic, when set to true, restore previous state in case of a failed install/upgrade attempt
 	Atomic *bool `yaml:"atomic,omitempty"`
+	// CleanupOnFail, when set to true, the --cleanup-on-fail helm flag is passed to the upgrade command
+	CleanupOnFail *bool `yaml:"cleanupOnFail,omitempty"`
+	// HistoryMax, limit the maximum number of revisions saved per release. Use 0 for no limit (default 10)
+	HistoryMax *int `yaml:"historyMax,omitempty"`
 
 	// MissingFileHandler is set to either "Error" or "Warn". "Error" instructs helmfile to fail when unable to find a values or secrets file. When "Warn", it prints the file and continues.
 	// The default value for MissingFileHandler is "Error".
@@ -649,6 +658,7 @@ func (st *HelmState) listReleases(context helmexec.HelmContext, helm helmexec.In
 	if helm.IsHelm3() && release.Namespace != "" {
 		flags = append(flags, "--namespace", release.Namespace)
 	}
+	flags = append(flags, "--deployed", "--failed", "--pending")
 	return helm.List(context, "^"+release.Name+"$", flags...)
 }
 
@@ -1054,11 +1064,19 @@ func (st *HelmState) createHelmContext(spec *ReleaseSpec, workerIndex int) helme
 	if spec.Tillerless != nil {
 		tillerless = *spec.Tillerless
 	}
+	historyMax := 10
+	if st.HelmDefaults.HistoryMax != nil {
+		historyMax = *st.HelmDefaults.HistoryMax
+	}
+	if spec.HistoryMax != nil {
+		historyMax = *spec.HistoryMax
+	}
 
 	return helmexec.HelmContext{
 		Tillerless:      tillerless,
 		TillerNamespace: namespace,
 		WorkerIndex:     workerIndex,
+		HistoryMax:      historyMax,
 	}
 }
 
@@ -1546,6 +1564,10 @@ func (st *HelmState) flagsForUpgrade(helm helmexec.Interface, release *ReleaseSp
 		flags = append(flags, "--atomic")
 	}
 
+	if release.CleanupOnFail != nil && *release.CleanupOnFail || release.CleanupOnFail == nil && st.HelmDefaults.CleanupOnFail {
+		flags = append(flags, "--cleanup-on-fail")
+	}
+
 	flags = st.appendConnectionFlags(flags, release)
 
 	var err error
@@ -1570,6 +1592,8 @@ func (st *HelmState) flagsForTemplate(helm helmexec.Interface, release *ReleaseS
 		return nil, err
 	}
 
+	flags = st.appendApiVersionsFlags(flags)
+
 	common, err := st.namespaceAndValuesFlags(helm, release, workerIndex)
 	if err != nil {
 		return nil, err
@@ -1602,6 +1626,13 @@ func (st *HelmState) flagsForDiff(helm helmexec.Interface, release *ReleaseSpec,
 	return append(flags, common...), nil
 }
 
+func (st *HelmState) appendApiVersionsFlags(flags []string) []string {
+	for _, a := range st.ApiVersions {
+		flags = append(flags, "--api-versions", a)
+	}
+	return flags
+}
+
 func (st *HelmState) isDevelopment(release *ReleaseSpec) bool {
 	result := st.HelmDefaults.Devel
 	if release.Devel != nil {

pkg/state/state_run.go

@@ -139,13 +139,15 @@ func SortedReleaseGroups(releases []Release, reverse bool) ([][]Release, error)
 
 func GroupReleasesByDependency(releases []Release) ([][]Release, error) {
 	idToReleases := map[string][]Release{}
+	idToIndex := map[string]int{}
 
 	d := dag.New()
-	for _, r := range releases {
+	for i, r := range releases {
 
 		id := ReleaseToID(&r.ReleaseSpec)
 
 		idToReleases[id] = append(idToReleases[id], r)
+		idToIndex[id] = i
 
 		// Only compute dependencies from non-filtered releases
 		if !r.Filtered {
@@ -182,7 +184,13 @@ func GroupReleasesByDependency(releases []Release) ([][]Release, error) {
 		}
 
 		// Make the helmfile behavior deterministic for reproducibility and ease of testing
-		sort.Strings(idsInGroup)
+		// We try to keep the order of definitions to keep backward-compatibility
+		// See https://github.com/roboll/helmfile/issues/988
+		sort.Slice(idsInGroup, func(i, j int) bool {
+			ii := idToIndex[idsInGroup[i]]
+			ij := idToIndex[idsInGroup[j]]
+			return ii < ij
+		})
 
 		for _, id := range idsInGroup {
 			releases, ok := idToReleases[id]

pkg/state/state_test.go

@@ -448,6 +448,58 @@ func TestHelmState_flagsForUpgrade(t *testing.T) {
 				"--namespace", "test-namespace",
 			},
 		},
+		{
+			name: "cleanup-on-fail",
+			defaults: HelmSpec{
+				CleanupOnFail: false,
+			},
+			release: &ReleaseSpec{
+				Chart:         "test/chart",
+				Version:       "0.1",
+				CleanupOnFail: &enable,
+				Name:          "test-charts",
+				Namespace:     "test-namespace",
+			},
+			want: []string{
+				"--version", "0.1",
+				"--cleanup-on-fail",
+				"--namespace", "test-namespace",
+			},
+		},
+		{
+			name: "cleanup-on-fail-override-default",
+			defaults: HelmSpec{
+				CleanupOnFail: true,
+			},
+			release: &ReleaseSpec{
+				Chart:         "test/chart",
+				Version:       "0.1",
+				CleanupOnFail: &disable,
+				Name:          "test-charts",
+				Namespace:     "test-namespace",
+			},
+			want: []string{
+				"--version", "0.1",
+				"--namespace", "test-namespace",
+			},
+		},
+		{
+			name: "cleanup-on-fail-from-default",
+			defaults: HelmSpec{
+				CleanupOnFail: true,
+			},
+			release: &ReleaseSpec{
+				Chart:     "test/chart",
+				Version:   "0.1",
+				Name:      "test-charts",
+				Namespace: "test-namespace",
+			},
+			want: []string{
+				"--version", "0.1",
+				"--cleanup-on-fail",
+				"--namespace", "test-namespace",
+			},
+		},
 		{
 			name:     "tiller",
 			defaults: HelmSpec{},
@@ -1116,7 +1168,7 @@ func TestHelmState_SyncReleasesAffectedRealeases(t *testing.T) {
 			//simulate the release is already installed
 			for i, release := range tt.releases {
 				if tt.installed != nil && tt.installed[i] {
-					helm.Lists[exectest.ListKey{Filter: "^" + release.Name + "$"}] = release.Name
+					helm.Lists[exectest.ListKey{Filter: "^" + release.Name + "$", Flags: "--deployed--failed--pending"}] = release.Name
 				}
 			}
 
@@ -1216,7 +1268,7 @@ func TestGetDeployedVersion(t *testing.T) {
 				Lists: map[exectest.ListKey]string{},
 			}
 			//simulate the helm.list call result
-			helm.Lists[exectest.ListKey{Filter: "^" + tt.release.Name + "$"}] = tt.listResult
+			helm.Lists[exectest.ListKey{Filter: "^" + tt.release.Name + "$", Flags: "--deployed--failed--pending"}] = tt.listResult
 
 			affectedReleases := AffectedReleases{}
 			state.SyncReleases(&affectedReleases, helm, []string{}, 1)