Feature/autobuild docker image for each release thru dockerhub auto builds (#886)

* feat: build hook for dockerhub autobuilds

* release new image

* Create pre_build

* Update push

.circleci/config.yml

@@ -4,7 +4,7 @@ jobs:
 
   build:
     docker:
-      - image: circleci/golang:1.12.4
+      - image: circleci/golang:1.13.4
     environment:
       GOPATH: /go
     working_directory: /home/circleci/workspace/helmfile
@@ -24,12 +24,45 @@ jobs:
           root: ./
           paths:
             - .
+  image:
+    docker:
+    - image: circleci/golang:1.13.4
+    working_directory: /home/circleci/workspace/helmfile
+    steps:
+    - checkout
+    - restore_cache:
+        keys:
+        - go-mod-cache-v1-{{ checksum "./go.sum" }}
+        - go-mod-cache-v1-
+    - setup_remote_docker
+    - run:
+        command: |
+          make tools
+          go mod vendor
+          ORG=roboll BUILD_URL="$CIRCLE_BUILD_URL" make image
+
+  image_helm3:
+    docker:
+    - image: circleci/golang:1.13.4
+    working_directory: /home/circleci/workspace/helmfile
+    steps:
+    - checkout
+    - restore_cache:
+        keys:
+        - go-mod-cache-v1-{{ checksum "./go.sum" }}
+        - go-mod-cache-v1-
+    - setup_remote_docker
+    - run:
+        command: |
+          make tools
+          go mod vendor
+          ORG=roboll BUILD_URL="$CIRCLE_BUILD_URL" make image/helm3
 
   test:
     environment:
       GOPATH: /go
     docker:
-      - image: circleci/golang:1.12.4
+      - image: circleci/golang:1.13.4
     working_directory: /home/circleci/workspace/helmfile
     steps:
       - checkout
@@ -58,7 +91,7 @@ jobs:
       - run:
           name: Install helm
           environment:
-            HELM_VERSION: v2.13.0
+            HELM_VERSION: v2.15.2
           command: |
             HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
             curl -Lo ${HELM_FILENAME} "https://kubernetes-helm.storage.googleapis.com/${HELM_FILENAME}"
@@ -87,11 +120,56 @@ jobs:
             export TERM=xterm
             make integration
 
+  integration_tests_helm3:
+    machine:
+      image: circleci/classic:201808-01
+    environment:
+      CHANGE_MINIKUBE_NONE_USER: true
+      MINIKUBE_WANTUPDATENOTIFICATION: false
+      MINIKUBE_WANTREPORTERRORPROMPT: false
+    steps:
+    - checkout
+    - run: mkdir ~/build
+    - attach_workspace:
+        at: ~/build
+    - run: cp ~/build/helmfile ~/project/helmfile
+    - run:
+        name: Install helm
+        environment:
+          HELM_VERSION: v3.0.0
+        command: |
+          HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
+          curl -Lo ${HELM_FILENAME} "https://get.helm.sh/${HELM_FILENAME}"
+          tar zxf ${HELM_FILENAME} linux-amd64/helm
+          chmod +x linux-amd64/helm
+          sudo mv linux-amd64/helm /usr/local/bin/
+    - run:
+        name: Deploy minikube
+        environment:
+          CHANGE_MINIKUBE_NONE_USER: true
+          K8S_VERSION: v1.12.3
+          MINIKUBE_VERSION: v0.30.0
+        command: |
+          curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl
+          chmod +x kubectl && sudo mv kubectl /usr/local/bin/
+          curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
+          chmod +x minikube && sudo mv minikube /usr/local/bin/
+          sudo -E minikube start --vm-driver=none --kubernetes-version=${K8S_VERSION}
+          sudo -E minikube update-context
+    - run:
+        name: Wait for nodes to become ready
+        command: JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done
+    - run:
+        name: Execute integration tests
+        command: |
+          export TERM=xterm
+          HELMFILE_HELM3=1 make integration
+
 # GITHUB_TOKEN env var must be setup in circleci console
 
   release:
     docker:
-      - image: circleci/golang:1.12.4
+      - image: circleci/golang:1.13.4
     working_directory: /home/circleci/workspace/helmfile
     steps:
     - checkout
@@ -105,22 +183,22 @@ jobs:
           docker login -u="$DOCKER_USER" -p="$DOCKER_PASS" quay.io
           make tools
           go mod vendor
-          ORG=roboll BUILD_URL="$CIRCLE_BUILD_URL" make push release
+          ORG=roboll BUILD_URL="$CIRCLE_BUILD_URL" make push push/helm3 release
 
 workflows:
   version: 2
   build_and_test:
     jobs:
       - build
+      - image
+      - image_helm3
       - test
       - integration_tests:
           requires:
             - build
-          filters:
-            branches:
-              only:
-                - master
-                - /pull.*/
+      - integration_tests_helm3:
+          requires:
+            - build
       - release:
           filters:
             branches:

.gitignore

@@ -1,3 +1,5 @@
 dist/
 .idea/
-helmfile
+helmfile
+helmfile.lock
+vendor/

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.12.4-alpine3.9 as builder
+FROM golang:1.13.6-alpine3.11 as builder
 
 RUN apk add --no-cache make git
 WORKDIR /workspace/helmfile
@@ -7,24 +7,37 @@ RUN make static-linux
 
 # -----------------------------------------------------------------------------
 
-FROM alpine:3.8
+FROM alpine:3.11
 
-RUN apk add --no-cache ca-certificates git bash curl
+RUN apk add --no-cache ca-certificates git bash curl jq
 
-ARG HELM_VERSION=v2.13.0
+ARG HELM_VERSION=v2.15.2
 ARG HELM_LOCATION="https://kubernetes-helm.storage.googleapis.com"
 ARG HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
-ARG HELM_SHA256="15eca6ad225a8279de80c7ced42305e24bc5ac60bb7d96f2d2fa4af86e02c794"
+ARG HELM_SHA256="a9d2db920bd4b3d824729bbe1ff3fa57ad27760487581af6e5d3156d1b3c2511"
 RUN wget ${HELM_LOCATION}/${HELM_FILENAME} && \
+    echo Verifying ${HELM_FILENAME}... && \
     sha256sum ${HELM_FILENAME} | grep -q "${HELM_SHA256}" && \
-    tar zxf ${HELM_FILENAME} && mv /linux-amd64/helm /usr/local/bin/ && \
+    echo Extracting ${HELM_FILENAME}... && \
+    tar zxvf ${HELM_FILENAME} && mv /linux-amd64/helm /usr/local/bin/ && \
     rm ${HELM_FILENAME} && rm -r /linux-amd64
 
-RUN mkdir -p "$(helm home)/plugins"
+# using the install documentation found at https://kubernetes.io/docs/tasks/tools/install-kubectl/
+# for now but in a future version of alpine (in the testing version at the time of writing)
+# we should be able to install using apk add.
+ENV KUBECTL_VERSION="v1.14.5"
+ENV KUBECTL_SHA256="26681319de56820a8467c9407e9203d5b15fb010ffc75ac5b99c9945ad0bd28c"
+RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && \
+    sha256sum kubectl | grep ${KUBECTL_SHA256} && \
+    chmod +x kubectl && \
+    mv kubectl /usr/local/bin/kubectl
+
+RUN ["helm", "init", "--client-only"]
 RUN helm plugin install https://github.com/databus23/helm-diff && \
     helm plugin install https://github.com/futuresimple/helm-secrets && \
     helm plugin install https://github.com/hypnoglow/helm-s3.git && \
-    helm plugin install https://github.com/aslafy-z/helm-git.git
+    helm plugin install https://github.com/aslafy-z/helm-git.git && \
+    helm plugin install https://github.com/rimusz/helm-tiller
 
 COPY --from=builder /workspace/helmfile/dist/helmfile_linux_amd64 /usr/local/bin/helmfile
 

Dockerfile.helm3

@@ -0,0 +1,44 @@
+FROM golang:1.13.6-alpine3.11 as builder
+
+RUN apk add --no-cache make git
+WORKDIR /workspace/helmfile
+COPY . /workspace/helmfile
+RUN make static-linux
+
+# -----------------------------------------------------------------------------
+
+FROM alpine:3.11
+
+RUN apk add --no-cache ca-certificates git bash curl jq
+
+ARG HELM_VERSION=v3.0.2
+ARG HELM_LOCATION="https://get.helm.sh"
+ARG HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
+ARG HELM_SHA256="c6b7aa7e4ffc66e8abb4be328f71d48c643cb8f398d95c74d075cfb348710e1d"
+RUN wget ${HELM_LOCATION}/${HELM_FILENAME} && \
+    echo Verifying ${HELM_FILENAME}... && \
+    sha256sum ${HELM_FILENAME} | grep -q "${HELM_SHA256}" && \
+    echo Extracting ${HELM_FILENAME}... && \
+    tar zxvf ${HELM_FILENAME} && mv /linux-amd64/helm /usr/local/bin/ && \
+    rm ${HELM_FILENAME} && rm -r /linux-amd64
+
+# using the install documentation found at https://kubernetes.io/docs/tasks/tools/install-kubectl/
+# for now but in a future version of alpine (in the testing version at the time of writing)
+# we should be able to install using apk add.
+ENV KUBECTL_VERSION="v1.14.5"
+ENV KUBECTL_SHA256="26681319de56820a8467c9407e9203d5b15fb010ffc75ac5b99c9945ad0bd28c"
+RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && \
+    sha256sum kubectl | grep ${KUBECTL_SHA256} && \
+    chmod +x kubectl && \
+    mv kubectl /usr/local/bin/kubectl
+
+RUN helm plugin install https://github.com/databus23/helm-diff --version v3.0.0-rc.7 && \
+    helm plugin install https://github.com/futuresimple/helm-secrets && \
+    helm plugin install https://github.com/hypnoglow/helm-s3.git && \
+    helm plugin install https://github.com/aslafy-z/helm-git.git
+
+COPY --from=builder /workspace/helmfile/dist/helmfile_linux_amd64 /usr/local/bin/helmfile
+
+ENV HELMFILE_HELM3="1"
+
+CMD ["/usr/local/bin/helmfile"]

Makefile

@@ -26,15 +26,15 @@ integration:
 .PHONY: integration
 
 cross:
-	env CGO_ENABLED=0 gox -os '!freebsd !netbsd' -arch '!arm' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS}
+	env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le !s390x' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X github.com/roboll/helmfile/pkg/app/version.Version=${TAG}' ${TARGETS}
 .PHONY: cross
 
 static-linux:
-	env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "dist/helmfile_linux_amd64" -ldflags '-X main.Version=${TAG}' ${TARGETS}
-.PHONY: linux
+	env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -o "dist/helmfile_linux_amd64" -ldflags '-X github.com/roboll/helmfile/pkg/app/version.Version=${TAG}' ${TARGETS}
+.PHONY: static-linux
 
 install:
-	env CGO_ENABLED=0 go install -ldflags '-X main.Version=${TAG}' ${TARGETS}
+	env CGO_ENABLED=0 go install -ldflags '-X github.com/roboll/helmfile/pkg/app/version.Version=${TAG}' ${TARGETS}
 .PHONY: install
 
 clean:
@@ -59,6 +59,13 @@ run: image
 push: image
 	docker push quay.io/${ORG}/helmfile:${TAG}
 
+
+image/helm3:
+	docker build -f Dockerfile.helm3 -t quay.io/${ORG}/helmfile:helm3-${TAG} .
+
+push/helm3: image/helm3
+	docker push quay.io/${ORG}/helmfile:helm3-${TAG}
+
 tools:
 	go get -u github.com/tcnksm/ghr github.com/mitchellh/gox
 .PHONY: tools

README.md

@@ -35,7 +35,7 @@ To avoid upgrades for each iteration of `helm`, the `helmfile` executable delega
 
 ## Configuration
 
-**CAUTION**: This documentation is for the development version of Helmfile. If you are looking for the documentation for any of releases, please switch to the corresponding release tag like [v0.79.1](https://github.com/roboll/helmfile/tree/v0.79.1).
+**CAUTION**: This documentation is for the development version of Helmfile. If you are looking for the documentation for any of releases, please switch to the corresponding release tag like [v0.92.1](https://github.com/roboll/helmfile/tree/v0.92.1).
 
 The default helmfile is `helmfile.yaml`:
 
@@ -61,6 +61,11 @@ repositories:
   keyFile: optional_client_key
   username: optional_username
   password: optional_password
+# Advanced configuration: You can use a ca bundle to use an https repo
+# with a self-signed certificate
+- name: insecure
+   url: https://charts.my-insecure-domain.com
+   caFile: optional_ca_crt
 
 # context: kube-context # this directive is deprecated, please consider using helmDefaults.kubeContext
 
@@ -69,6 +74,7 @@ helmDefaults:
   tillerNamespace: tiller-namespace  #dedicated default key for tiller-namespace
   tillerless: false                  #dedicated default key for tillerless
   kubeContext: kube-context          #dedicated default key for kube-context (--kube-context)
+  cleanupOnFail: false               #dedicated default key for helm flag --cleanup-on-fail
   # additional and global args passed to helm
   args:
     - "--set k=v"
@@ -86,6 +92,8 @@ helmDefaults:
   tlsCert: "path/to/cert.pem"
   # path to TLS key file (default "$HELM_HOME/key.pem")
   tlsKey: "path/to/key.pem"
+  # limit the maximum number of revisions saved per release. Use 0 for no limit (default 10)
+  historyMax: 10
 
 # The desired states of Helm releases.
 #
@@ -152,6 +160,8 @@ releases:
     installed: true
     # restores previous state in case of failed release
     atomic: true
+    # when true, cleans up any new resources created during a failed release
+    cleanupOnFail: false
     # name of the tiller namespace
     tillerNamespace: vault
     # if true, will use the helm-tiller plugin
@@ -168,6 +178,8 @@ releases:
     # CAUTION: this doesn't work as expected for `tilerless: true`.
     # See https://github.com/roboll/helmfile/issues/642
     kubeContext: kube-context
+    # limit the maximum number of revisions saved per release. Use 0 for no limit (default 10)
+    historyMax: 10
 
   # Local chart example
   - name: grafana                            # name of this release
@@ -192,7 +204,9 @@ helmfiles:
   # Override state values
   values:
   # Values files merged into the nested state's values
-  - additiona.values.yaml
+  - additional.values.yaml
+  # One important aspect of using values here is that they first need to be defined in the values section
+  # of the origin helmfile, so in this example key1 needs to be in the values or environments.NAME.values of path/to/subhelmfile.yaml
   # Inline state values merged into the nested state's values
   - key1: val1
 - # All the nested state files under `helmfiles:` is processed in the order of definition.
@@ -212,17 +226,17 @@ helmfiles:
 # The default is `environments: {"default": {}}` which implies:
 #
 # - `{{ .Environment.Name }}` evaluates to "default"
-# - `{{ .Environment.Values }}` being empty
+# - `{{ .Values }}` being empty
 environments:
   # The "default" environment is available and used when `helmfile` is run without `--environment NAME`.
   default:
-    # Everything from the values.yaml is available via `{{ .Environment.Values.KEY }}`.
+    # Everything from the values.yaml is available via `{{ .Values.KEY }}`.
     # Suppose `{"foo": {"bar": 1}}` contained in the values.yaml below,
-    # `{{ .Environment.Values.foo.bar }}` is evaluated to `1`.
+    # `{{ .Values.foo.bar }}` is evaluated to `1`.
     values:
     - environments/default/values.yaml
     # Each entry in values can be either a file path or inline values.
-    # The below is an example of inline values, which is merged to the `.Environment.Values`
+    # The below is an example of inline values, which is merged to the `.Values`
     - myChartVer: 1.0.0-dev
   # Any environment other than `default` is used only when `helmfile` is run with `--environment NAME`.
   # That is, the "production" env below is used when and only when it is run like `helmfile --environment production sync`.
@@ -252,6 +266,18 @@ bases:
 - environments.yaml
 - defaults.yaml
 - templates.yaml
+
+#
+# Advanced Configuration: API Capabilities
+#
+# 'helmfile template' renders releases locally without querying an actual cluster,
+# and in this case `.Capabilities.APIVersions` cannot be populated.
+# When a chart queries for a specific CRD, this can lead to unexpected results.
+# 
+# Configure a fixed list of api versions to pass to 'helm template' via the --api-versions flag:
+apiVersions:
+- example/v1
+
 ```
 
 ## Templating
@@ -324,7 +350,7 @@ Congratulations! You now have your first Prometheus deployment running inside yo
 
 Iterate on the `helmfile.yaml` by referencing:
 
-- [Configuration syntax](#configuration-syntax)
+- [Configuration](#configuration)
 - [CLI reference](#cli-reference).
 - [Helmfile Best Practices Guide](https://github.com/roboll/helmfile/blob/master/docs/writing-helmfile.md)
 
@@ -332,13 +358,13 @@ Iterate on the `helmfile.yaml` by referencing:
 
 ```
 NAME:
-   helmfile -
+   helmfile
 
 USAGE:
    helmfile [global options] command [command options] [arguments...]
 
 VERSION:
-   v0.70.0
+   v0.92.1
 
 COMMANDS:
      deps      update charts based on the contents of requirements.yaml
@@ -353,15 +379,19 @@ COMMANDS:
      delete    DEPRECATED: delete releases from state file (helm delete)
      destroy   deletes and then purges releases
      test      test releases from state file (helm test)
+     build     output compiled helmfile state(s) as YAML
+     list      list releases defined in state file
+     help, h   Shows a list of commands or help for one command
 
 GLOBAL OPTIONS:
-   --helm-binary value, -b value           path to helm binary
+   --helm-binary value, -b value           path to helm binary (default: "helm")
    --file helmfile.yaml, -f helmfile.yaml  load config from file or directory. defaults to helmfile.yaml or `helmfile.d`(means `helmfile.d/*.yaml`) in this preference
    --environment default, -e default       specify the environment name. defaults to default
    --state-values-set value                set state values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)
    --state-values-file value               specify state values in a YAML file
    --quiet, -q                             Silence output. Equivalent to log-level warn
    --kube-context value                    Set kubectl context. Uses current context by default
+   --no-color                              Output without color
    --log-level value                       Set log level, default info
    --namespace value, -n value             Set namespace. Uses the namespace set in the context by default, and is available in templates as {{ .Namespace }}
    --selector value, -l value              Only run using the releases that match labels. Labels can take the form of foo=bar or foo!=bar.
@@ -552,10 +582,10 @@ releases:
     namespace: {{ requiredEnv "NAME" }}
     chart: roboll/vault-secret-manager
     values:
-    - values.yaml.tmpl
+    - values.yaml.gotmpl
 ```
 
-`values.yaml.tmpl`:
+`values.yaml.gotmpl`:
 
 ```yaml
 db:
@@ -625,7 +655,7 @@ releaseName: prod
 `values.yaml.gotmpl`
 
 ```yaml
-domain: {{ .Environment.Values | getOrNil "my.domain" | default "dev.example.com" }}
+domain: {{ .Values | getOrNil "my.domain" | default "dev.example.com" }}
 ```
 
 `helmfile sync` installs `myapp` with the value `domain=dev.example.com`,
@@ -652,17 +682,26 @@ environments:
     - other.yaml.gotmpl  #  template directives with potential side-effects like `exec` and `readFile` will be honoured
 
 releases:
-- name: myapp-{{ .Environment.Values.releaseName }} # release name will be one of `dev` or `prod` depending on selected environment
+- name: myapp-{{ .Values.releaseName }} # release name will be one of `dev` or `prod` depending on selected environment
   values:
   - values.yaml.gotmpl
 
-{{ if eq (.Environment.Values.releaseName "prod" ) }}
+{{ if eq (.Values.releaseName "prod" ) }}
 # this release would be installed only if selected environment is `production`
 - name: production-specific-release
   ...
 {{ end }}
 ```
 
+### Note
+
+The `{{ .Values.foo }}` syntax is the recommended way of using environment values.
+
+Prior to this [pull request](https://github.com/roboll/helmfile/pull/647), environment values were made available through the `{{ .Environment.Values.foo }}` syntax.
+This is still working but is **deprecated** and the new `{{ .Values.foo }}` syntax should be used instead.
+
+You can read more infos about the feature proposal [here](https://github.com/roboll/helmfile/issues/640).
+
 ## Environment Secrets
 
 Environment Secrets (not to be confused with Kubernetes Secrets) are encrypted versions of `Environment Values`.
@@ -697,7 +736,7 @@ releases:
 Then the environment secret `foo.bar` can be referenced by the below template expression in your `values.yaml.gotmpl`:
 
 ```yaml
-{{ .Environment.Values.foo.bar }}
+{{ .Values.foo.bar }}
 ```
 
 ## Tillerless
@@ -707,6 +746,50 @@ With the [helm-tiller](https://github.com/rimusz/helm-tiller) plugin installed,
 To enable this mode, you need to define `tillerless: true` and set the `tillerNamespace` in the `helmDefaults` section
 or in the `releases` entries.
 
+## DAG-aware installation/deletion ordering
+
+`needs` controls the order of the installation/deletion of the release:
+
+```yaml
+releases:
+- name: somerelease
+  needs:
+  - [TILLER_NAMESPACE/][NAMESPACE/]anotherelease
+```
+
+All the releases listed under `needs` are installed before(or deleted after) the release itself.
+
+For the following example, `helmfile [sync|apply]` installs releases in this order:
+
+1. logging
+2. servicemesh
+3. myapp1 and myapp2
+
+```yaml
+  - name: myapp1
+    chart: charts/myapp
+    needs:
+    - servicemesh
+    - logging
+  - name: myapp2
+    chart: charts/myapp
+    needs:
+    - servicemesh
+    - logging
+  - name: servicemesh
+    chart: charts/istio
+    needs:
+    - logging
+  - name: logging
+    chart: charts/fluentd
+```
+
+Note that all the releases in a same group is installed concurrently. That is, myapp1 and myapp2 are installed concurrently.
+
+On `helmfile [delete|destroy]`, deletions happen in the reverse order.
+
+That is, `myapp1` and `myapp2` are deleted first, then `servicemesh`, and finally `logging`.
+
 ## Separating helmfile.yaml into multiple independent files
 
 Once your `helmfile.yaml` got to contain too many releases,
@@ -978,6 +1061,22 @@ Those features are set using the environment variable `HELMFILE_EXPERIMENTAL`. H
 
 If you want to enable all experimental features set the env var to `HELMFILE_EXPERIMENTAL=true`
 
+## Azure ACR integration
+
+Azure offers helm repository [support for Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-helm-repos) as a preview feature.
+
+To use this you must first `az login` and then `az acr helm repo add -n <MyRegistry>`. This will extract a token for the given ACR and configure `helm` to use it, e.g. `helm repo update` should work straight away.
+
+To use `helmfile` with ACR, on the other hand, you must either include a username/password in the repository definition for the ACR in your `helmfile.yaml` or use the `--skip-deps` switch, e.g. `helmfile template --skip-deps`.
+
+An ACR repository definition in `helmfile.yaml` looks like this:
+
+```
+repositories:
+  - name: <MyRegistry>
+    url: https://<MyRegistry>.azurecr.io/helm/v1/repo
+```
+
 ## Examples
 
 For more examples, see the [examples/README.md](https://github.com/roboll/helmfile/blob/master/examples/README.md) or the [`helmfile`](https://github.com/cloudposse/helmfiles/tree/master/releases) distribution by [Cloud Posse](https://github.com/cloudposse/).

USERS.md

@@ -14,6 +14,8 @@ information to this file.
 | [Sight Machine](https://sightmachine.com/) | production | We don't have anything publicly posted about it, but have been using it for quite a while in production. | San Francisco, CA and Ann Arbor, MI | December 2018 |
 | [VSHN – The DevOps Company](https://vshn.ch) | production	| | Zurich, Switzerland | March 2019 |
 | [Vlocity](https://vlocity.com/) | proof-of-concept | | Melbourne, Australia | March 2019 |
-| [transit](https://transit.app/) | production | Repos are private, but [here's a blog post](https://medium.com/@naseem_60378/helmfile-its-like-a-helm-for-your-helm-74a908581599). We're using `helmfile.d` with 1 helmfile per namespace. Multiple environments, but not all charts on all environments. Using installed: `{{ eq .Environment.Name "env1" "env2" "env5" }}`. Loving it so far! Thanks! | Montreal, Canada | March 2019 |
+| [transit](https://transit.app/) | production | [Blog post](https://medium.com/@naseem_60378/helmfile-its-like-a-helm-for-your-helm-74a908581599). | Montreal, Canada | March 2019 |
 | [uniqkey](https://uniqkey.eu/) | production | [Wiki Page](https://ocd-scm.github.io/ocd-meta/) | Copenhagen, Denmark | April 2019 |
+| [bitsofinfo](https://github.com/bitsofinfo/helmfile-deploy) | production | Used with [helmfile-deploy](https://github.com/bitsofinfo/helmfile-deploy) to manage releases for dozens of apps | USA | July 2019 |
+| [kloeckner-i](https://www.kloeckner-i.com/) | production | We are deploying our standard tools via helmfile to all our clusters.  | Berlin, Germany | September 2019 |
 <!-- TABLE_END -->

docs/writing-helmfile.md

@@ -11,13 +11,13 @@ One example of how helmfile achieves it is that, `helmfile` fails when you tried
 That is, the following example let `helmfile` fail when you have no `eventApi.replicas` defined in environment values.
 
 ```
-{{ .Environment.Values.eventApi.replicas | default 1 }}
+{{ .Values.eventApi.replicas | default 1 }}
 ```
 
 In case it isn't a mistake and you do want to allow missing keys, use the `getOrNil` template function:
 
 ```
-{{ .Environment.Values | getOrNil "eventApi.replicas" }}
+{{ .Values | getOrNil "eventApi.replicas" }}
 ```
 
 This result in printing `<no value` in your template, that may or may not result in a failure.
@@ -25,14 +25,14 @@ This result in printing `<no value` in your template, that may or may not result
 If you want a kind of default values that is used when a missing key was referenced, use `default` like:
 
 ```
-{{ .Environment.Values | getOrNil "eventApi.replicas" | default 1 }}
+{{ .Values | getOrNil "eventApi.replicas" | default 1 }}
 ```
 
 Now, you get `1` when there is no `eventApi.replicas` defined in environment values.
 
 ## Release Template / Conventional Directory Structure
 
-Introducing helmfile into a large-scale project that involes dozens of releases often results in a lot of repetitions in `helmfile.yaml` files.
+Introducing helmfile into a large-scale project that involves dozens of releases often results in a lot of repetitions in `helmfile.yaml` files.
 
 The example below shows repetitions in `namespace`, `chart`, `values`, and `secrets`:
 
@@ -57,7 +57,7 @@ releases:
   values:
   - "./config/kubernetes-dashboard/values.yaml"
   - "./config/kubernetes-dashboard/{{ .Environment.Name }}.yaml"
-  values:
+  secrets:
   - "./config/kubernetes-dashboard/secrets.yaml"
   - "./config/kubernetes-dashboard/{{ .Environment.Name }}-secrets.yaml"
 ```
@@ -89,8 +89,48 @@ releases:
   <<: *default
 ```
 
+Release Templating supports the following parts of release definition:
+- basic fields: `name`, `namespace`, `chart`, `version`
+- boolean fields: `installed`, `wait`, `tillerless`, `verify` by the means of additional text
+  fields designed for templating only: `installedTemplate`, `waitTemplate`, `tillerlessTemplate`, `verifyTemplate`
+  ```yaml
+  # ...
+    installedTemplate: '{{`{{ eq .Release.Namespace "kube-system" }}`}}'
+    waitTemplate: '{{`{{ eq .Release.Labels.tag "safe" | not }}`}}'
+  # ...
+  ```
+- `set` block values:
+  ```yaml
+  # ...
+    setTemplate:
+    - name: '{{`{{ .Release.Name }}`}}'
+      values: '{{`{{ .Release.Namespace }}`}}'
+  # ...
+  ```
+- `values` and `secrets` file paths:
+  ```yaml
+  # ...
+    valuesTemplate:
+    - config/{{`{{ .Release.Name }}`}}/values.yaml
+    secrets:
+    - config/{{`{{ .Release.Name }}`}}/secrets.yaml
+  # ...
+  ```
+- inline `values` map:
+  ```yaml
+  # ...
+    valuesTemplate:
+    - image:
+        tag: `{{ .Release.Labels.tag }}`
+  # ...
+  ```
+
 See the [issue 428](https://github.com/roboll/helmfile/issues/428) for more context on how this is supposed to work.
 
+## Layering Release Values
+
+Please note, that it is not possible to layer `values` sections. If `values` is defined in the release and in the release template, only the `values` defined in the release will be considered. The same applies to `secrets` and `set`.
+
 ## Layering State Files
 
 > See **Layering State Template Files** if you're layering templates.
@@ -188,7 +228,7 @@ A work-around is to treat the state file as a go template and use `readFile` tem
 ```yaml
 templates:
   metricbeat: &metricbeat
-  - name: metricbeat
+    name: metricbeat
     chart: stable/metricbeat
 ```
 
@@ -227,7 +267,7 @@ bases:
 # Part 3: Dynamic Releases
 releases:
   - name: test1
-    chart: mychart-{{ .Environment.Values.myname }}
+    chart: mychart-{{ .Values.myname }}
     values:
       replicaCount: 1
       image:
@@ -262,9 +302,9 @@ Where the gotmpl file loaded in the second part looks like:
 ```yaml
 helmDefaults:
   tillerNamespace: kube-system
-  kubeContext: {{ .Environment.Values.kubeContext }}
+  kubeContext: {{ .Values.kubeContext }}
   verify: false
-  {{ if .Environment.Values.wait }}
+  {{ if .Values.wait }}
   wait: true
   {{ else }}
   wait: false
@@ -274,9 +314,9 @@ helmDefaults:
   force: true
 ```
 
-Each go template is rendered in the context where `.Environment.Values` is inherited from the previous part. 
+Each go template is rendered in the context where `.Values` is inherited from the previous part. 
 
-So in `mydefaults.yaml.gotmpl`, both `.Environment.Values.kubeContext` and `.Environment.Values.wait` are valid as they do exist in the environment values inherited from the previous part(=the first part) of your `helmfile.yaml.gotmpl`, and therefore the template is rendered to:
+So in `mydefaults.yaml.gotmpl`, both `.Values.kubeContext` and `.Values.wait` are valid as they do exist in the environment values inherited from the previous part(=the first part) of your `helmfile.yaml.gotmpl`, and therefore the template is rendered to:
 
 ```yaml
 helmDefaults:
@@ -289,13 +329,13 @@ helmDefaults:
   force: true
 ```
 
-Similarly, the third part of the top-level `helmfile.yaml.gotmpl`, `.Environment.Values.myname` is valid as it is included in the environment values inherited from the previous parts:
+Similarly, the third part of the top-level `helmfile.yaml.gotmpl`, `.Values.myname` is valid as it is included in the environment values inherited from the previous parts:
 
 ```yaml
 # Part 3: Dynamic Releases
 releases:
   - name: test1
-    chart: mychart-{{ .Environment.Values.myname }}
+    chart: mychart-{{ .Values.myname }}
     values:
       replicaCount: 1
       image:

go.mod

@@ -1,22 +1,48 @@
 module github.com/roboll/helmfile
 
-go 1.12
+go 1.13
 
 require (
-	github.com/Masterminds/semver v1.4.1
-	github.com/Masterminds/sprig v2.16.0+incompatible
-	github.com/aokoli/goutils v1.0.1 // indirect
-	github.com/google/go-cmp v0.3.0
-	github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c // indirect
+	cloud.google.com/go v0.47.0 // indirect
+	github.com/Azure/azure-sdk-for-go v35.0.0+incompatible // indirect
+	github.com/Masterminds/goutils v1.1.0 // indirect
+	github.com/Masterminds/semver v1.4.2
+	github.com/Masterminds/sprig v2.22.0+incompatible
+	github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a
+	github.com/aws/aws-sdk-go v1.25.22 // indirect
+	github.com/go-test/deep v1.0.3
+	github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect
+	github.com/google/go-cmp v0.3.1
+	github.com/gosuri/uitable v0.0.3
 	github.com/hashicorp/go-getter v1.3.0
-	github.com/huandu/xstrings v1.0.0 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.3 // indirect
+	github.com/hashicorp/go-version v1.2.0
+	github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c // indirect
+	github.com/huandu/xstrings v1.2.0 // indirect
 	github.com/imdario/mergo v0.3.6
-	github.com/pkg/errors v0.8.1 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/mattn/go-colorable v0.1.4 // indirect
+	github.com/mattn/go-isatty v0.0.10 // indirect
+	github.com/pierrec/lz4 v2.3.0+incompatible // indirect
+	github.com/r3labs/diff v0.0.0-20190801153147-a71de73c46ad
 	github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939
-	github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28
-	go.uber.org/atomic v1.3.2 // indirect
-	go.uber.org/multierr v1.1.0 // indirect
-	go.uber.org/zap v1.8.0
-	gopkg.in/yaml.v2 v2.2.1
+	github.com/urfave/cli v1.20.0
+	github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363
+	github.com/variantdev/vals v0.0.0-20191123015158-21b16052816c
+	go.mozilla.org/sops v0.0.0-20190912205235-14a22d7a7060 // indirect
+	go.opencensus.io v0.22.1 // indirect
+	go.uber.org/multierr v1.1.0
+	go.uber.org/zap v1.9.1
+	golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf // indirect
+	golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271 // indirect
+	golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c // indirect
+	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
+	google.golang.org/api v0.13.0 // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6 // indirect
+	google.golang.org/grpc v1.24.0 // indirect
+	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v2 v2.2.4
+	gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d // indirect
 	gotest.tools v2.2.0+incompatible
 )

hooks/build

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+echo "=> Building container"
+docker build \
+  --tag "$IMAGE_NAME" \
+  --file Dockerfile .

hooks/pre_build

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+go()
+{
+  docker run --rm -v "$(pwd):/workspace" -w /workspace golang:latest go $@;
+}
+
+go get -u github.com/tcnksm/ghr github.com/mitchellh/gox
+go mod vendor

hooks/push

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "=> Fetch unshallow origin"
+git fetch --unshallow origin || true
+
+echo "=> Tag image:"
+docker tag "${IMAGE_NAME}" "${DOCKER_REPO}:$(git describe --tags --abbrev=0 HEAD^)"
+docker tag "${IMAGE_NAME}" "${DOCKER_REPO}:$(git describe --tags --abbrev=0 HEAD)"
+
+echo "=> Push images"
+docker push "${DOCKER_REPO}"

main.go

@@ -2,6 +2,11 @@ package main
 
 import (
 	"fmt"
+	"os"
+	"strings"
+
+	"github.com/roboll/helmfile/pkg/app/version"
+
 	"github.com/roboll/helmfile/pkg/app"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/maputil"
@@ -9,12 +14,8 @@ import (
 	"github.com/urfave/cli"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
-	"os"
-	"strings"
 )
 
-var Version string
-
 var logger *zap.SugaredLogger
 
 func configureLogging(c *cli.Context) error {
@@ -44,11 +45,12 @@ func main() {
 	cliApp := cli.NewApp()
 	cliApp.Name = "helmfile"
 	cliApp.Usage = ""
-	cliApp.Version = Version
+	cliApp.Version = version.Version
 	cliApp.Flags = []cli.Flag{
 		cli.StringFlag{
 			Name:  "helm-binary, b",
 			Usage: "path to helm binary",
+			Value: "helm",
 		},
 		cli.StringFlag{
 			Name:  "file, f",
@@ -74,6 +76,10 @@ func main() {
 			Name:  "kube-context",
 			Usage: "Set kubectl context. Uses current context by default",
 		},
+		cli.BoolFlag{
+			Name:  "no-color",
+			Usage: "Output without color",
+		},
 		cli.StringFlag{
 			Name:  "log-level",
 			Usage: "Set log level, default info",
@@ -110,6 +116,10 @@ func main() {
 					Value: "",
 					Usage: "pass args to helm exec",
 				},
+				cli.BoolFlag{
+					Name:  "skip-repos",
+					Usage: "skip running `helm repo update` before running `helm dependency build`",
+				},
 			},
 			Action: action(func(run *app.App, c configImpl) error {
 				return run.Deps(c)
@@ -138,6 +148,10 @@ func main() {
 					Value: "",
 					Usage: "pass args to helm exec",
 				},
+				cli.StringSliceFlag{
+					Name:  "set",
+					Usage: "additional values to be merged into the command",
+				},
 				cli.StringSliceFlag{
 					Name:  "values",
 					Usage: "additional value files to be merged into the command",
@@ -161,6 +175,10 @@ func main() {
 					Value: "",
 					Usage: "pass args to helm exec",
 				},
+				cli.StringSliceFlag{
+					Name:  "set",
+					Usage: "additional values to be merged into the command",
+				},
 				cli.StringSliceFlag{
 					Name:  "values",
 					Usage: "additional value files to be merged into the command",
@@ -182,6 +200,11 @@ func main() {
 					Value: 0,
 					Usage: "maximum number of concurrent helm processes to run, 0 is unlimited",
 				},
+				cli.IntFlag{
+					Name:  "context",
+					Value: 0,
+					Usage: "output NUM lines of context around changes",
+				},
 			},
 			Action: action(func(run *app.App, c configImpl) error {
 				return run.Diff(c)
@@ -196,10 +219,18 @@ func main() {
 					Value: "",
 					Usage: "pass args to helm template",
 				},
+				cli.StringSliceFlag{
+					Name:  "set",
+					Usage: "additional values to be merged into the command",
+				},
 				cli.StringSliceFlag{
 					Name:  "values",
 					Usage: "additional value files to be merged into the command",
 				},
+				cli.StringFlag{
+					Name:  "output-dir",
+					Usage: "output directory to pass to helm template (helm template --output-dir)",
+				},
 				cli.IntFlag{
 					Name:  "concurrency",
 					Value: 0,
@@ -223,6 +254,10 @@ func main() {
 					Value: "",
 					Usage: "pass args to helm exec",
 				},
+				cli.StringSliceFlag{
+					Name:  "set",
+					Usage: "additional values to be merged into the command",
+				},
 				cli.StringSliceFlag{
 					Name:  "values",
 					Usage: "additional value files to be merged into the command",
@@ -245,6 +280,10 @@ func main() {
 			Name:  "sync",
 			Usage: "sync all resources from state file (repos, releases and chart deps)",
 			Flags: []cli.Flag{
+				cli.StringSliceFlag{
+					Name:  "set",
+					Usage: "additional values to be merged into the command",
+				},
 				cli.StringSliceFlag{
 					Name:  "values",
 					Usage: "additional value files to be merged into the command",
@@ -272,6 +311,10 @@ func main() {
 			Name:  "apply",
 			Usage: "apply all resources from state file only when there are changes",
 			Flags: []cli.Flag{
+				cli.StringSliceFlag{
+					Name:  "set",
+					Usage: "additional values to be merged into the command",
+				},
 				cli.StringSliceFlag{
 					Name:  "values",
 					Usage: "additional value files to be merged into the command",
@@ -281,6 +324,11 @@ func main() {
 					Value: 0,
 					Usage: "maximum number of concurrent helm processes to run, 0 is unlimited",
 				},
+				cli.IntFlag{
+					Name:  "context",
+					Value: 0,
+					Usage: "output NUM lines of context around changes",
+				},
 				cli.StringFlag{
 					Name:  "args",
 					Value: "",
@@ -388,6 +436,22 @@ func main() {
 				return run.Test(c)
 			}),
 		},
+		{
+			Name:  "build",
+			Usage: "output compiled helmfile state(s) as YAML",
+			Flags: []cli.Flag{},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.PrintState(c)
+			}),
+		},
+		{
+			Name:  "list",
+			Usage: "list releases defined in state file",
+			Flags: []cli.Flag{},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.ListReleases(c)
+			}),
+		},
 	}
 
 	err := cliApp.Run(os.Args)
@@ -419,11 +483,11 @@ func NewUrfaveCliConfigImpl(c *cli.Context) (configImpl, error) {
 		for i := range optsSet {
 			ops := strings.Split(optsSet[i], ",")
 			for j := range ops {
-				op := strings.Split(ops[j], "=")
-				k := strings.Split(op[0], ".")
+				op := strings.SplitN(ops[j], "=", 2)
+				k := maputil.ParseKey(op[0])
 				v := op[1]
 
-				set = maputil.Set(set, k, v)
+				maputil.Set(set, k, v)
 			}
 		}
 		conf.set = set
@@ -432,6 +496,14 @@ func NewUrfaveCliConfigImpl(c *cli.Context) (configImpl, error) {
 	return conf, nil
 }
 
+func (c configImpl) Set() []string {
+	return c.c.StringSlice("set")
+}
+
+func (c configImpl) SkipRepos() bool {
+	return c.c.Bool("skip-repos")
+}
+
 func (c configImpl) Values() []string {
 	return c.c.StringSlice("values")
 }
@@ -440,6 +512,10 @@ func (c configImpl) Args() string {
 	return c.c.String("args")
 }
 
+func (c configImpl) OutputDir() string {
+	return c.c.String("output-dir")
+}
+
 func (c configImpl) Concurrency() int {
 	return c.c.Int("concurrency")
 }
@@ -500,11 +576,11 @@ func (c configImpl) Selectors() []string {
 	return c.c.GlobalStringSlice("selector")
 }
 
-func (c configImpl) Set() map[string]interface{} {
+func (c configImpl) StateValuesSet() map[string]interface{} {
 	return c.set
 }
 
-func (c configImpl) ValuesFiles() []string {
+func (c configImpl) StateValuesFiles() []string {
 	return c.c.GlobalStringSlice("state-values-file")
 }
 
@@ -512,6 +588,14 @@ func (c configImpl) Interactive() bool {
 	return c.c.GlobalBool("interactive")
 }
 
+func (c configImpl) NoColor() bool {
+	return c.c.GlobalBool("no-color")
+}
+
+func (c configImpl) Context() int {
+	return c.c.Int("context")
+}
+
 func (c configImpl) Logger() *zap.SugaredLogger {
 	return c.c.App.Metadata["logger"].(*zap.SugaredLogger)
 }

pkg/app/app.go

@@ -1,21 +1,30 @@
 package app
 
 import (
+	"bytes"
 	"fmt"
-	"github.com/roboll/helmfile/pkg/helmexec"
-	"github.com/roboll/helmfile/pkg/remote"
-	"github.com/roboll/helmfile/pkg/state"
 	"io/ioutil"
 	"log"
 	"os"
 	"os/signal"
-	"strings"
-	"syscall"
-
-	"go.uber.org/zap"
-
 	"path/filepath"
 	"sort"
+	"strings"
+	"syscall"
+	"text/tabwriter"
+
+	"github.com/gosuri/uitable"
+	"github.com/roboll/helmfile/pkg/argparser"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/remote"
+	"github.com/roboll/helmfile/pkg/state"
+	"github.com/variantdev/vals"
+	"go.uber.org/zap"
+)
+
+const (
+	// cache size for improving performance of ref+.* secrets rendering
+	valsCacheSize = 512
 )
 
 type App struct {
@@ -25,7 +34,6 @@ type App struct {
 	Env         string
 	Namespace   string
 	Selectors   []string
-	HelmBinary  string
 	Args        string
 	ValuesFiles []string
 	Set         map[string]interface{}
@@ -45,6 +53,10 @@ type App struct {
 	chdir func(string) error
 
 	remote *remote.Remote
+
+	helmExecer helmexec.Interface
+
+	valsRuntime vals.Evaluator
 }
 
 func New(conf ConfigProvider) *App {
@@ -54,11 +66,13 @@ func New(conf ConfigProvider) *App {
 		Env:         conf.Env(),
 		Namespace:   conf.Namespace(),
 		Selectors:   conf.Selectors(),
-		HelmBinary:  conf.HelmBinary(),
 		Args:        conf.Args(),
 		FileOrDir:   conf.FileOrDir(),
-		ValuesFiles: conf.ValuesFiles(),
-		Set:         conf.Set(),
+		ValuesFiles: conf.StateValuesFiles(),
+		Set:         conf.StateValuesSet(),
+		helmExecer: helmexec.New(conf.HelmBinary(), conf.Logger(), conf.KubeContext(), &helmexec.ShellRunner{
+			Logger: conf.Logger(),
+		}),
 	})
 }
 
@@ -71,17 +85,24 @@ func Init(app *App) *App {
 	app.fileExistsAt = fileExistsAt
 	app.fileExists = fileExists
 	app.directoryExistsAt = directoryExistsAt
+
+	var err error
+	app.valsRuntime, err = vals.New(vals.Options{CacheSize: valsCacheSize})
+	if err != nil {
+		panic(fmt.Sprintf("Failed to initialize vals runtime: %v", err))
+	}
+
 	return app
 }
 
 func (a *App) Deps(c DepsConfigProvider) error {
-	return a.ForEachState(func(run *Run) []error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
 		return run.Deps(c)
 	})
 }
 
 func (a *App) Repos(c ReposConfigProvider) error {
-	return a.ForEachState(func(run *Run) []error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
 		return run.Repos(c)
 	})
 }
@@ -93,65 +114,102 @@ func (a *App) reverse() *App {
 }
 
 func (a *App) DeprecatedSyncCharts(c DeprecatedChartsConfigProvider) error {
-	return a.ForEachState(func(run *Run) []error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
 		return run.DeprecatedSyncCharts(c)
 	})
 }
 
 func (a *App) Diff(c DiffConfigProvider) error {
-	return a.ForEachState(func(run *Run) []error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
 		return run.Diff(c)
 	})
 }
 
 func (a *App) Template(c TemplateConfigProvider) error {
-	return a.ForEachState(func(run *Run) []error {
-		return run.Template(c)
+	return a.ForEachState(func(run *Run) (bool, []error) {
+		return a.template(run, c)
 	})
 }
 
 func (a *App) Lint(c LintConfigProvider) error {
-	return a.ForEachState(func(run *Run) []error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
 		return run.Lint(c)
 	})
 }
 
 func (a *App) Sync(c SyncConfigProvider) error {
-	return a.ForEachState(func(run *Run) []error {
-		return run.Sync(c)
+	return a.ForEachState(func(run *Run) (bool, []error) {
+		return a.sync(run, c)
 	})
 }
 
 func (a *App) Apply(c ApplyConfigProvider) error {
-	return a.ForEachState(func(run *Run) []error {
-		return run.Apply(c)
+	return a.ForEachState(func(run *Run) (bool, []error) {
+		return a.apply(run, c)
 	})
 }
 
 func (a *App) Status(c StatusesConfigProvider) error {
-	return a.ForEachState(func(run *Run) []error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
 		return run.Status(c)
 	})
 }
 
 func (a *App) Delete(c DeleteConfigProvider) error {
-	return a.reverse().ForEachState(func(run *Run) []error {
-		return run.Delete(c)
+	return a.reverse().ForEachState(func(run *Run) (bool, []error) {
+		return a.delete(run, c.Purge(), c)
 	})
 }
 
 func (a *App) Destroy(c DestroyConfigProvider) error {
-	return a.reverse().ForEachState(func(run *Run) []error {
-		return run.Destroy(c)
+	return a.reverse().ForEachState(func(run *Run) (bool, []error) {
+		return a.delete(run, true, c)
 	})
 }
 
 func (a *App) Test(c TestConfigProvider) error {
-	return a.ForEachState(func(run *Run) []error {
+	if c.Cleanup() && a.helmExecer.IsHelm3() {
+		a.Logger.Warnf("warn: requested cleanup will not be applied. " +
+			"To clean up test resources with Helm 3, you have to remove them manually " +
+			"or set helm.sh/hook-delete-policy\n")
+	}
+	return a.ForEachStateFiltered(func(run *Run) []error {
 		return run.Test(c)
 	})
 }
 
+func (a *App) PrintState(c StateConfigProvider) error {
+
+	return a.ForEachStateFiltered(func(run *Run) []error {
+		state, err := run.state.ToYaml()
+		if err != nil {
+			return []error{err}
+		}
+		fmt.Printf("---\n#  Source: %s\n\n%+v", run.state.FilePath, state)
+		return []error{}
+	})
+}
+
+func (a *App) ListReleases(c StateConfigProvider) error {
+	table := uitable.New()
+	table.AddRow("NAME", "NAMESPACE", "INSTALLED", "LABELS")
+
+	err := a.ForEachStateFiltered(func(run *Run) []error {
+		//var releases m
+		for _, r := range run.state.Releases {
+			labels := ""
+			for k, v := range r.Labels {
+				labels = fmt.Sprintf("%s,%s:%s", labels, k, v)
+			}
+			installed := r.Installed == nil || *r.Installed
+			table.AddRow(r.Name, r.Namespace, fmt.Sprintf("%t", installed), strings.Trim(labels, ","))
+		}
+		return []error{}
+	})
+	fmt.Println(table.String())
+	return err
+}
+
 func (a *App) within(dir string, do func() error) error {
 	if dir == "." {
 		return do()
@@ -234,6 +292,8 @@ func (a *App) loadDesiredStateFromYaml(file string, opts ...LoadOpts) (*state.He
 		Reverse:     a.Reverse,
 		KubeContext: a.KubeContext,
 		glob:        a.glob,
+		helm:        a.helmExecer,
+		valsRuntime: a.valsRuntime,
 	}
 
 	var op LoadOpts
@@ -274,7 +334,7 @@ func (a *App) visitStates(fileOrDir string, defOpts LoadOpts, converge func(*sta
 
 		ctx := context{a, st}
 
-		helm := helmexec.New(a.Logger, a.KubeContext)
+		helm := a.helmExecer
 
 		if err != nil {
 			switch stateLoadErr := err.(type) {
@@ -324,8 +384,10 @@ func (a *App) visitStates(fileOrDir string, defOpts LoadOpts, converge func(*sta
 		if tmplErr != nil {
 			return appError(fmt.Sprintf("failed executing release templates in \"%s\"", f), tmplErr)
 		}
+
 		processed, errs := converge(templated, helm)
 		noMatchInHelmfiles = noMatchInHelmfiles && !processed
+
 		return context{a, templated}.clean(errs)
 	})
 
@@ -340,10 +402,9 @@ func (a *App) visitStates(fileOrDir string, defOpts LoadOpts, converge func(*sta
 	return nil
 }
 
-func (a *App) ForEachState(do func(*Run) []error) error {
+func (a *App) ForEachStateFiltered(do func(*Run) []error) error {
+	ctx := NewContext()
 	err := a.VisitDesiredStatesWithReleasesFiltered(a.FileOrDir, func(st *state.HelmState, helm helmexec.Interface) []error {
-		ctx := NewContext()
-
 		run := NewRun(st, helm, ctx)
 
 		return do(run)
@@ -356,7 +417,88 @@ func (a *App) ForEachState(do func(*Run) []error) error {
 	return err
 }
 
-func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) []error) error {
+func (a *App) ForEachState(do func(*Run) (bool, []error)) error {
+	ctx := NewContext()
+	err := a.visitStatesWithSelectorsAndRemoteSupport(a.FileOrDir, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
+		run := NewRun(st, helm, ctx)
+		return do(run)
+	})
+
+	if err != nil && a.ErrorHandler != nil {
+		return a.ErrorHandler(err)
+	}
+
+	return err
+}
+
+func printBatches(batches [][]state.Release) string {
+	buf := &bytes.Buffer{}
+
+	w := new(tabwriter.Writer)
+
+	w.Init(buf, 0, 1, 1, ' ', 0)
+
+	fmt.Fprintln(w, "GROUP\tRELEASES")
+
+	for i, batch := range batches {
+		ids := []string{}
+		for _, r := range batch {
+			ids = append(ids, state.ReleaseToID(&r.ReleaseSpec))
+		}
+		fmt.Fprintf(w, "%d\t%s\n", i+1, strings.Join(ids, ", "))
+	}
+
+	w.Flush()
+
+	return buf.String()
+}
+
+func withDAG(templated *state.HelmState, helm helmexec.Interface, logger *zap.SugaredLogger, reverse bool, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) (bool, []error) {
+	batches, err := templated.PlanReleases(reverse)
+	if err != nil {
+		return false, []error{err}
+	}
+
+	numBatches := len(batches)
+
+	logger.Debugf("processing %d groups of releases in this order:\n%s", numBatches, printBatches(batches))
+
+	any := false
+
+	for i, batch := range batches {
+		var targets []state.ReleaseSpec
+
+		for _, marked := range batch {
+			targets = append(targets, marked.ReleaseSpec)
+		}
+
+		var releaseIds []string
+		for _, r := range targets {
+			releaseIds = append(releaseIds, state.ReleaseToID(&r))
+		}
+
+		logger.Debugf("processing releases in group %d/%d: %s", i+1, numBatches, strings.Join(releaseIds, ", "))
+
+		batchSt := *templated
+		batchSt.Releases = targets
+
+		processed, errs := converge(&batchSt, helm)
+
+		if errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+
+		any = any || processed
+	}
+
+	return any, nil
+}
+
+type Opts struct {
+	DAGEnabled bool
+}
+
+func (a *App) visitStatesWithSelectorsAndRemoteSupport(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error {
 	opts := LoadOpts{
 		Selectors: a.Selectors,
 	}
@@ -395,7 +537,11 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge
 
 	a.remote = remote
 
-	return a.visitStates(fileOrDir, opts, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
+	return a.visitStates(fileOrDir, opts, converge)
+}
+
+func (a *App) Wrap(converge func(*state.HelmState, helmexec.Interface) []error) func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
+	return func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
 		if len(st.Selectors) > 0 {
 			err := st.FilterReleases()
 			if err != nil {
@@ -403,10 +549,6 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge
 			}
 		}
 
-		if a.HelmBinary != "" {
-			helm.SetHelmBinary(a.HelmBinary)
-		}
-
 		type Key struct {
 			TillerNamespace, Name string
 		}
@@ -430,6 +572,14 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge
 		processed := len(st.Releases) != 0 && len(errs) == 0
 
 		return processed, errs
+	}
+}
+
+func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) []error) error {
+	f := a.Wrap(converge)
+
+	return a.visitStatesWithSelectorsAndRemoteSupport(fileOrDir, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
+		return f(st, helm)
 	})
 }
 
@@ -505,6 +655,452 @@ func (a *App) findDesiredStateFiles(specifiedPath string) ([]string, error) {
 	return files, nil
 }
 
+func (a *App) apply(r *Run, c ApplyConfigProvider) (bool, []error) {
+	st := r.state
+	helm := r.helm
+	ctx := r.ctx
+
+	allReleases := st.GetReleasesWithOverrides()
+
+	toApply, err := st.GetSelectedReleasesWithOverrides()
+	if err != nil {
+		return false, []error{err}
+	}
+	if len(toApply) == 0 {
+		return false, nil
+	}
+
+	// Do build deps and prepare only on selected releases so that we won't waste time
+	// on running various helm commands on unnecessary releases
+	st.Releases = toApply
+
+	if !c.SkipDeps() {
+		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+	}
+	if errs := st.PrepareReleases(helm, "apply"); errs != nil && len(errs) > 0 {
+		return false, errs
+	}
+
+	// helm must be 2.11+ and helm-diff should be provided `--detailed-exitcode` in order for `helmfile apply` to work properly
+	detailedExitCode := true
+
+	diffOpts := &state.DiffOpts{
+		NoColor: c.NoColor(),
+		Context: c.Context(),
+		Set:     c.Set(),
+	}
+
+	var changedReleases []state.ReleaseSpec
+	var deletingReleases []state.ReleaseSpec
+	var planningErrs []error
+
+	// TODO Better way to detect diff on only filtered releases
+	{
+		changedReleases, planningErrs = st.DiffReleases(helm, c.Values(), c.Concurrency(), detailedExitCode, c.SuppressSecrets(), false, diffOpts)
+
+		var err error
+		deletingReleases, err = st.DetectReleasesToBeDeletedForSync(helm, st.Releases)
+		if err != nil {
+			planningErrs = append(planningErrs, err)
+		}
+	}
+
+	fatalErrs := []error{}
+
+	for _, e := range planningErrs {
+		switch err := e.(type) {
+		case *state.ReleaseError:
+			if err.Code != 2 {
+				fatalErrs = append(fatalErrs, e)
+			}
+		default:
+			fatalErrs = append(fatalErrs, e)
+		}
+	}
+
+	if len(fatalErrs) > 0 {
+		return false, fatalErrs
+	}
+
+	releasesToBeDeleted := map[string]state.ReleaseSpec{}
+	for _, r := range deletingReleases {
+		id := state.ReleaseToID(&r)
+		releasesToBeDeleted[id] = r
+	}
+
+	releasesToBeUpdated := map[string]state.ReleaseSpec{}
+	for _, r := range changedReleases {
+		id := state.ReleaseToID(&r)
+
+		// If `helm-diff` detected changes but it is not being `helm delete`ed, we should run `helm upgrade`
+		if _, ok := releasesToBeDeleted[id]; !ok {
+			releasesToBeUpdated[id] = r
+		}
+	}
+
+	// sync only when there are changes
+	if len(releasesToBeUpdated) == 0 && len(releasesToBeDeleted) == 0 {
+		// TODO better way to get the logger
+		logger := c.Logger()
+		logger.Infof("")
+		logger.Infof("No affected releases")
+		return true, nil
+	}
+
+	names := []string{}
+	for _, r := range releasesToBeUpdated {
+		names = append(names, fmt.Sprintf("  %s (%s) UPDATED", r.Name, r.Chart))
+	}
+	for _, r := range releasesToBeDeleted {
+		names = append(names, fmt.Sprintf("  %s (%s) DELETED", r.Name, r.Chart))
+	}
+	// Make the output deterministic for testing purpose
+	sort.Strings(names)
+
+	infoMsg := fmt.Sprintf(`Affected releases are:
+%s
+`, strings.Join(names, "\n"))
+	confMsg := fmt.Sprintf(`%s
+Do you really want to apply?
+  Helmfile will apply all your changes, as shown above.
+
+`, infoMsg)
+	interactive := c.Interactive()
+	if !interactive {
+		a.Logger.Debug(infoMsg)
+	}
+
+	syncErrs := []error{}
+
+	affectedReleases := state.AffectedReleases{}
+
+	// Traverse DAG of all the releases so that we don't suffer from false-positive missing dependencies
+	st.Releases = allReleases
+
+	if !interactive || interactive && r.askForConfirmation(confMsg) {
+		r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
+
+		// We deleted releases by traversing the DAG in reverse order
+		if len(releasesToBeDeleted) > 0 {
+			_, deletionErrs := withDAG(st, helm, a.Logger, true, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error {
+				var rs []state.ReleaseSpec
+
+				for _, r := range subst.Releases {
+					if r2, ok := releasesToBeDeleted[state.ReleaseToID(&r)]; ok {
+						rs = append(rs, r2)
+					}
+				}
+
+				subst.Releases = rs
+
+				return subst.DeleteReleasesForSync(&affectedReleases, helm, c.Concurrency())
+			}))
+
+			if deletionErrs != nil && len(deletionErrs) > 0 {
+				syncErrs = append(syncErrs, deletionErrs...)
+			}
+		}
+
+		// We upgrade releases by traversing the DAG
+		if len(releasesToBeUpdated) > 0 {
+			_, updateErrs := withDAG(st, helm, a.Logger, false, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error {
+				var rs []state.ReleaseSpec
+
+				for _, r := range subst.Releases {
+					if r2, ok := releasesToBeUpdated[state.ReleaseToID(&r)]; ok {
+						rs = append(rs, r2)
+					}
+				}
+
+				subst.Releases = rs
+
+				syncOpts := state.SyncOpts{
+					Set: c.Set(),
+				}
+				return subst.SyncReleases(&affectedReleases, helm, c.Values(), c.Concurrency(), &syncOpts)
+			}))
+
+			if updateErrs != nil && len(updateErrs) > 0 {
+				syncErrs = append(syncErrs, updateErrs...)
+			}
+		}
+	}
+
+	affectedReleases.DisplayAffectedReleases(c.Logger())
+	return true, syncErrs
+}
+
+func (a *App) delete(r *Run, purge bool, c DestroyConfigProvider) (bool, []error) {
+	st := r.state
+	helm := r.helm
+
+	affectedReleases := state.AffectedReleases{}
+
+	toSync, err := st.GetSelectedReleasesWithOverrides()
+	if err != nil {
+		return false, []error{err}
+	}
+	if len(toSync) == 0 {
+		return false, nil
+	}
+
+	toDelete, err := st.DetectReleasesToBeDeleted(helm, toSync)
+	if err != nil {
+		return false, []error{err}
+	}
+
+	releasesToDelete := map[string]state.ReleaseSpec{}
+	for _, r := range toDelete {
+		id := state.ReleaseToID(&r)
+		releasesToDelete[id] = r
+	}
+
+	names := make([]string, len(toSync))
+	for i, r := range toSync {
+		names[i] = fmt.Sprintf("  %s (%s)", r.Name, r.Chart)
+	}
+
+	var errs []error
+
+	msg := fmt.Sprintf(`Affected releases are:
+%s
+
+Do you really want to delete?
+  Helmfile will delete all your releases, as shown above.
+
+`, strings.Join(names, "\n"))
+	interactive := c.Interactive()
+	if !interactive || interactive && r.askForConfirmation(msg) {
+		r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
+
+		if len(releasesToDelete) > 0 {
+			_, deletionErrs := withDAG(st, helm, a.Logger, true, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error {
+				var rs []state.ReleaseSpec
+
+				for _, r := range subst.Releases {
+					if _, ok := releasesToDelete[state.ReleaseToID(&r)]; ok {
+						rs = append(rs, r)
+					}
+				}
+
+				subst.Releases = rs
+
+				return subst.DeleteReleases(&affectedReleases, helm, c.Concurrency(), purge)
+			}))
+
+			if deletionErrs != nil && len(deletionErrs) > 0 {
+				errs = append(errs, deletionErrs...)
+			}
+		}
+	}
+	affectedReleases.DisplayAffectedReleases(c.Logger())
+	return true, errs
+}
+
+func (a *App) sync(r *Run, c SyncConfigProvider) (bool, []error) {
+	st := r.state
+	helm := r.helm
+	ctx := r.ctx
+
+	allReleases := st.GetReleasesWithOverrides()
+
+	toSync, err := st.GetSelectedReleasesWithOverrides()
+	if err != nil {
+		return false, []error{err}
+	}
+	if len(toSync) == 0 {
+		return false, nil
+	}
+
+	// Do build deps and prepare only on selected releases so that we won't waste time
+	// on running various helm commands on unnecessary releases
+	st.Releases = toSync
+
+	if !c.SkipDeps() {
+		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+	}
+	if errs := st.PrepareReleases(helm, "sync"); errs != nil && len(errs) > 0 {
+		return false, errs
+	}
+
+	toDelete, err := st.DetectReleasesToBeDeletedForSync(helm, toSync)
+	if err != nil {
+		return false, []error{err}
+	}
+
+	releasesToDelete := map[string]state.ReleaseSpec{}
+	for _, r := range toDelete {
+		id := state.ReleaseToID(&r)
+		releasesToDelete[id] = r
+	}
+
+	var toUpdate []state.ReleaseSpec
+	for _, r := range toSync {
+		if _, deleted := releasesToDelete[state.ReleaseToID(&r)]; !deleted {
+			toUpdate = append(toUpdate, r)
+		}
+	}
+
+	releasesToUpdate := map[string]state.ReleaseSpec{}
+	for _, r := range toUpdate {
+		id := state.ReleaseToID(&r)
+		releasesToUpdate[id] = r
+	}
+
+	names := []string{}
+	for _, r := range releasesToUpdate {
+		names = append(names, fmt.Sprintf("  %s (%s) UPDATED", r.Name, r.Chart))
+	}
+	for _, r := range releasesToDelete {
+		names = append(names, fmt.Sprintf("  %s (%s) DELETED", r.Name, r.Chart))
+	}
+	// Make the output deterministic for testing purpose
+	sort.Strings(names)
+
+	infoMsg := fmt.Sprintf(`Affected releases are:
+%s
+`, strings.Join(names, "\n"))
+
+	a.Logger.Info(infoMsg)
+
+	var errs []error
+
+	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
+
+	// Traverse DAG of all the releases so that we don't suffer from false-positive missing dependencies
+	st.Releases = allReleases
+
+	affectedReleases := state.AffectedReleases{}
+
+	if len(releasesToDelete) > 0 {
+		_, deletionErrs := withDAG(st, helm, a.Logger, true, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error {
+			var rs []state.ReleaseSpec
+
+			for _, r := range subst.Releases {
+				if r2, ok := releasesToDelete[state.ReleaseToID(&r)]; ok {
+					rs = append(rs, r2)
+				}
+			}
+
+			subst.Releases = rs
+
+			return subst.DeleteReleasesForSync(&affectedReleases, helm, c.Concurrency())
+		}))
+
+		if deletionErrs != nil && len(deletionErrs) > 0 {
+			errs = append(errs, deletionErrs...)
+		}
+	}
+
+	if len(releasesToUpdate) > 0 {
+		_, syncErrs := withDAG(st, helm, a.Logger, false, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error {
+			var rs []state.ReleaseSpec
+
+			for _, r := range subst.Releases {
+				if r2, ok := releasesToUpdate[state.ReleaseToID(&r)]; ok {
+					rs = append(rs, r2)
+				}
+			}
+
+			subst.Releases = rs
+
+			opts := &state.SyncOpts{
+				Set: c.Set(),
+			}
+			return subst.SyncReleases(&affectedReleases, helm, c.Values(), c.Concurrency(), opts)
+		}))
+
+		if syncErrs != nil && len(syncErrs) > 0 {
+			errs = append(errs, syncErrs...)
+		}
+	}
+	affectedReleases.DisplayAffectedReleases(c.Logger())
+	return true, errs
+}
+
+func (a *App) template(r *Run, c TemplateConfigProvider) (bool, []error) {
+	st := r.state
+	helm := r.helm
+	ctx := r.ctx
+
+	allReleases := st.GetReleasesWithOverrides()
+
+	toRender, err := st.GetSelectedReleasesWithOverrides()
+	if err != nil {
+		return false, []error{err}
+	}
+	if len(toRender) == 0 {
+		return false, nil
+	}
+
+	// Do build deps and prepare only on selected releases so that we won't waste time
+	// on running various helm commands on unnecessary releases
+	st.Releases = toRender
+
+	if !c.SkipDeps() {
+		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+	}
+	if errs := st.PrepareReleases(helm, "template"); errs != nil && len(errs) > 0 {
+		return false, errs
+	}
+
+	releasesToRender := map[string]state.ReleaseSpec{}
+	for _, r := range toRender {
+		id := state.ReleaseToID(&r)
+		releasesToRender[id] = r
+	}
+
+	names := make([]string, len(toRender))
+	for i, r := range toRender {
+		names[i] = fmt.Sprintf("  %s (%s)", r.Name, r.Chart)
+	}
+
+	var errs []error
+
+	// Traverse DAG of all the releases so that we don't suffer from false-positive missing dependencies
+	st.Releases = allReleases
+
+	if len(releasesToRender) > 0 {
+		_, templateErrs := withDAG(st, helm, a.Logger, false, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error {
+			var rs []state.ReleaseSpec
+
+			for _, r := range subst.Releases {
+				if r2, ok := releasesToRender[state.ReleaseToID(&r)]; ok {
+					rs = append(rs, r2)
+				}
+			}
+
+			subst.Releases = rs
+
+			args := argparser.GetArgs(c.Args(), st)
+			opts := &state.TemplateOpts{
+				Set: c.Set(),
+			}
+			return subst.TemplateReleases(helm, c.OutputDir(), c.Values(), args, c.Concurrency(), opts)
+		}))
+
+		if templateErrs != nil && len(templateErrs) > 0 {
+			errs = append(errs, templateErrs...)
+		}
+	}
+	return true, errs
+}
+
 func fileExistsAt(path string) bool {
 	fileInfo, err := os.Stat(path)
 	return err == nil && fileInfo.Mode().IsRegular()

pkg/app/config.go

@@ -10,8 +10,8 @@ type ConfigProvider interface {
 	KubeContext() string
 	Namespace() string
 	Selectors() []string
-	Set() map[string]interface{}
-	ValuesFiles() []string
+	StateValuesSet() map[string]interface{}
+	StateValuesFiles() []string
 	Env() string
 
 	loggingConfig
@@ -26,6 +26,7 @@ type DeprecatedChartsConfigProvider interface {
 
 type DepsConfigProvider interface {
 	Args() string
+	SkipRepos() bool
 }
 
 type ReposConfigProvider interface {
@@ -36,10 +37,14 @@ type ApplyConfigProvider interface {
 	Args() string
 
 	Values() []string
+	Set() []string
 	SkipDeps() bool
 
 	SuppressSecrets() bool
 
+	NoColor() bool
+	Context() int
+
 	concurrencyConfig
 	interactive
 	loggingConfig
@@ -49,6 +54,7 @@ type SyncConfigProvider interface {
 	Args() string
 
 	Values() []string
+	Set() []string
 	SkipDeps() bool
 
 	concurrencyConfig
@@ -59,11 +65,14 @@ type DiffConfigProvider interface {
 	Args() string
 
 	Values() []string
+	Set() []string
 	SkipDeps() bool
 
 	SuppressSecrets() bool
 
 	DetailedExitcode() bool
+	NoColor() bool
+	Context() int
 
 	concurrencyConfig
 }
@@ -99,6 +108,7 @@ type LintConfigProvider interface {
 	Args() string
 
 	Values() []string
+	Set() []string
 	SkipDeps() bool
 
 	concurrencyConfig
@@ -108,7 +118,9 @@ type TemplateConfigProvider interface {
 	Args() string
 
 	Values() []string
+	Set() []string
 	SkipDeps() bool
+	OutputDir() string
 
 	concurrencyConfig
 }
@@ -119,6 +131,9 @@ type StatusesConfigProvider interface {
 	concurrencyConfig
 }
 
+type StateConfigProvider interface {
+}
+
 type concurrencyConfig interface {
 	Concurrency() int
 }

pkg/app/desired_state_file_loader.go

@@ -4,12 +4,15 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"github.com/imdario/mergo"
-	"github.com/roboll/helmfile/pkg/environment"
-	"github.com/roboll/helmfile/pkg/state"
-	"go.uber.org/zap"
 	"path/filepath"
 	"sort"
+
+	"github.com/imdario/mergo"
+	"github.com/roboll/helmfile/pkg/environment"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/state"
+	"github.com/variantdev/vals"
+	"go.uber.org/zap"
 )
 
 type desiredStateLoader struct {
@@ -24,7 +27,9 @@ type desiredStateLoader struct {
 	abs        func(string) (string, error)
 	glob       func(string) ([]string, error)
 
-	logger *zap.SugaredLogger
+	logger      *zap.SugaredLogger
+	helm        helmexec.Interface
+	valsRuntime vals.Evaluator
 }
 
 func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, error) {
@@ -71,10 +76,10 @@ func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, e
 	}
 
 	if ld.namespace != "" {
-		if st.Namespace != "" {
+		if st.OverrideNamespace != "" {
 			return nil, errors.New("err: Cannot use option --namespace and set attribute namespace.")
 		}
-		st.Namespace = ld.namespace
+		st.OverrideNamespace = ld.namespace
 	}
 
 	return st, nil
@@ -121,11 +126,24 @@ func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *e
 		)
 	}
 
-	return self, err
+	if err != nil {
+		return nil, err
+	}
+
+	for i, h := range self.Helmfiles {
+		if h.Path == f {
+			return nil, fmt.Errorf("%s contains a recursion into the same sub-helmfile at helmfiles[%d]", f, i)
+		}
+		if h.Path == "." {
+			return nil, fmt.Errorf("%s contains a recursion into the the directory containing this helmfile at helmfiles[%d]", f, i)
+		}
+	}
+
+	return self, nil
 }
 
 func (a *desiredStateLoader) underlying() *state.StateCreator {
-	c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob)
+	c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.helm, a.valsRuntime)
 	c.LoadFile = a.loadFile
 	return c
 }

pkg/app/run.go

@@ -1,11 +1,9 @@
 package app
 
 import (
-	"fmt"
 	"github.com/roboll/helmfile/pkg/argparser"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/state"
-	"strings"
 )
 
 type Run struct {
@@ -30,8 +28,10 @@ func (r *Run) askForConfirmation(msg string) bool {
 func (r *Run) Deps(c DepsConfigProvider) []error {
 	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
 
-	if errs := r.ctx.SyncReposOnce(r.state, r.helm); errs != nil && len(errs) > 0 {
-		return errs
+	if !c.SkipRepos() {
+		if errs := r.ctx.SyncReposOnce(r.state, r.helm); errs != nil && len(errs) > 0 {
+			return errs
+		}
 	}
 
 	return r.state.UpdateDeps(r.helm)
@@ -61,149 +61,6 @@ func (r *Run) Status(c StatusesConfigProvider) []error {
 	return r.state.ReleaseStatuses(r.helm, workers)
 }
 
-func (r *Run) Delete(c DeleteConfigProvider) []error {
-	affectedReleases := state.AffectedReleases{}
-	purge := c.Purge()
-
-	errs := []error{}
-
-	names := make([]string, len(r.state.Releases))
-	for i, r := range r.state.Releases {
-		names[i] = fmt.Sprintf("  %s (%s)", r.Name, r.Chart)
-	}
-
-	msg := fmt.Sprintf(`Affected releases are:
-%s
-
-Do you really want to delete?
-  Helmfile will delete all your releases, as shown above.
-
-`, strings.Join(names, "\n"))
-	interactive := c.Interactive()
-	if !interactive || interactive && r.askForConfirmation(msg) {
-		r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
-
-		errs = r.state.DeleteReleases(&affectedReleases, r.helm, c.Concurrency(), purge)
-	}
-	affectedReleases.DisplayAffectedReleases(c.Logger())
-	return errs
-}
-
-func (r *Run) Destroy(c DestroyConfigProvider) []error {
-	errs := []error{}
-	affectedReleases := state.AffectedReleases{}
-
-	names := make([]string, len(r.state.Releases))
-	for i, r := range r.state.Releases {
-		names[i] = fmt.Sprintf("  %s (%s)", r.Name, r.Chart)
-	}
-
-	msg := fmt.Sprintf(`Affected releases are:
-%s
-
-Do you really want to delete?
-  Helmfile will delete all your releases, as shown above.
-
-`, strings.Join(names, "\n"))
-	interactive := c.Interactive()
-	if !interactive || interactive && r.askForConfirmation(msg) {
-		r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
-
-		errs = r.state.DeleteReleases(&affectedReleases, r.helm, c.Concurrency(), true)
-	}
-	affectedReleases.DisplayAffectedReleases(c.Logger())
-	return errs
-}
-
-func (r *Run) Apply(c ApplyConfigProvider) []error {
-	st := r.state
-	helm := r.helm
-	ctx := r.ctx
-
-	affectedReleases := state.AffectedReleases{}
-	if !c.SkipDeps() {
-		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
-			return errs
-		}
-		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
-			return errs
-		}
-	}
-	if errs := st.PrepareReleases(helm, "apply"); errs != nil && len(errs) > 0 {
-		return errs
-	}
-
-	// helm must be 2.11+ and helm-diff should be provided `--detailed-exitcode` in order for `helmfile apply` to work properly
-	detailedExitCode := true
-
-	releases, errs := st.DiffReleases(helm, c.Values(), c.Concurrency(), detailedExitCode, c.SuppressSecrets(), false)
-
-	releasesToBeDeleted, err := st.DetectReleasesToBeDeleted(helm)
-	if err != nil {
-		errs = append(errs, err)
-	}
-
-	fatalErrs := []error{}
-
-	noError := true
-	for _, e := range errs {
-		switch err := e.(type) {
-		case *state.ReleaseError:
-			if err.Code != 2 {
-				noError = false
-				fatalErrs = append(fatalErrs, e)
-			}
-		default:
-			noError = false
-			fatalErrs = append(fatalErrs, e)
-		}
-	}
-
-	// sync only when there are changes
-	if noError {
-		if len(releases) == 0 && len(releasesToBeDeleted) == 0 {
-			// TODO better way to get the logger
-			logger := c.Logger()
-			logger.Infof("")
-			logger.Infof("No affected releases")
-		} else {
-			names := []string{}
-			for _, r := range releases {
-				names = append(names, fmt.Sprintf("  %s (%s) UPDATED", r.Name, r.Chart))
-			}
-			for _, r := range releasesToBeDeleted {
-				names = append(names, fmt.Sprintf("  %s (%s) DELETED", r.Name, r.Chart))
-			}
-
-			msg := fmt.Sprintf(`Affected releases are:
-%s
-
-Do you really want to apply?
-  Helmfile will apply all your changes, as shown above.
-
-`, strings.Join(names, "\n"))
-			interactive := c.Interactive()
-			if !interactive || interactive && r.askForConfirmation(msg) {
-				rs := []state.ReleaseSpec{}
-				for _, r := range releases {
-					rs = append(rs, *r)
-				}
-				for _, r := range releasesToBeDeleted {
-					rs = append(rs, *r)
-				}
-
-				r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
-
-				st.Releases = rs
-				return st.SyncReleases(&affectedReleases, helm, c.Values(), c.Concurrency())
-			}
-		}
-	}
-
-	affectedReleases.DisplayAffectedReleases(c.Logger())
-	return fatalErrs
-}
-
 func (r *Run) Diff(c DiffConfigProvider) []error {
 	st := r.state
 	helm := r.helm
@@ -223,56 +80,15 @@ func (r *Run) Diff(c DiffConfigProvider) []error {
 
 	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
 
-	_, errs := st.DiffReleases(helm, c.Values(), c.Concurrency(), c.DetailedExitcode(), c.SuppressSecrets(), true)
+	opts := &state.DiffOpts{
+		Context: c.Context(),
+		NoColor: c.NoColor(),
+		Set:     c.Set(),
+	}
+	_, errs := st.DiffReleases(helm, c.Values(), c.Concurrency(), c.DetailedExitcode(), c.SuppressSecrets(), true, opts)
 	return errs
 }
 
-func (r *Run) Sync(c SyncConfigProvider) []error {
-	st := r.state
-	helm := r.helm
-	ctx := r.ctx
-
-	affectedReleases := state.AffectedReleases{}
-	if !c.SkipDeps() {
-		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
-			return errs
-		}
-		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
-			return errs
-		}
-	}
-	if errs := st.PrepareReleases(helm, "sync"); errs != nil && len(errs) > 0 {
-		return errs
-	}
-
-	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
-
-	errs := st.SyncReleases(&affectedReleases, helm, c.Values(), c.Concurrency())
-	affectedReleases.DisplayAffectedReleases(c.Logger())
-	return errs
-}
-
-func (r *Run) Template(c TemplateConfigProvider) []error {
-	state := r.state
-	helm := r.helm
-	ctx := r.ctx
-
-	if !c.SkipDeps() {
-		if errs := ctx.SyncReposOnce(state, helm); errs != nil && len(errs) > 0 {
-			return errs
-		}
-		if errs := state.BuildDeps(helm); errs != nil && len(errs) > 0 {
-			return errs
-		}
-	}
-	if errs := state.PrepareReleases(helm, "template"); errs != nil && len(errs) > 0 {
-		return errs
-	}
-
-	args := argparser.GetArgs(c.Args(), state)
-	return state.TemplateReleases(helm, c.Values(), args, c.Concurrency())
-}
-
 func (r *Run) Test(c TestConfigProvider) []error {
 	cleanup := c.Cleanup()
 	timeout := c.Timeout()
@@ -284,23 +100,26 @@ func (r *Run) Test(c TestConfigProvider) []error {
 }
 
 func (r *Run) Lint(c LintConfigProvider) []error {
-	state := r.state
+	st := r.state
 	helm := r.helm
 	ctx := r.ctx
 
 	values := c.Values()
-	args := argparser.GetArgs(c.Args(), state)
+	args := argparser.GetArgs(c.Args(), st)
 	workers := c.Concurrency()
 	if !c.SkipDeps() {
-		if errs := ctx.SyncReposOnce(state, helm); errs != nil && len(errs) > 0 {
+		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
 			return errs
 		}
-		if errs := state.BuildDeps(helm); errs != nil && len(errs) > 0 {
+		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
 			return errs
 		}
 	}
-	if errs := state.PrepareReleases(helm, "lint"); errs != nil && len(errs) > 0 {
+	if errs := st.PrepareReleases(helm, "lint"); errs != nil && len(errs) > 0 {
 		return errs
 	}
-	return state.LintReleases(helm, values, args, workers)
+	opts := &state.LintOpts{
+		Set: c.Set(),
+	}
+	return st.LintReleases(helm, values, args, workers, opts)
 }

pkg/app/two_pass_renderer.go

@@ -31,12 +31,15 @@ func (r *desiredStateLoader) renderPrestate(firstPassEnv *environment.Environmen
 	yamlBuf, err := firstPassRenderer.RenderTemplateContentToBuffer(content)
 	if err != nil && r.logger != nil {
 		r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", filename, prependLineNumbers(string(content)))
+		r.logger.Debugf("template syntax error: %v", err)
 		if yamlBuf == nil { // we have a template syntax error, let the second parse report
-			r.logger.Debugf("template syntax error: %v", err)
 			return firstPassEnv, nil
 		}
 	}
 	yamlData := yamlBuf.String()
+	if r.logger != nil {
+		r.logger.Debugf("first-pass rendering output of \"%s\":\n%s", filename, prependLineNumbers(yamlData))
+	}
 
 	// Work-around for https://github.com/golang/go/issues/24963
 	sanitized := strings.ReplaceAll(yamlData, "<no value>", "")
@@ -53,7 +56,7 @@ func (r *desiredStateLoader) renderPrestate(firstPassEnv *environment.Environmen
 	if err != nil && r.logger != nil {
 		switch err.(type) {
 		case *state.StateLoadError:
-			r.logger.Infof("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err)
+			r.logger.Debugf("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err)
 		}
 		r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String()))
 	}

pkg/app/two_pass_renderer_test.go

@@ -1,13 +1,14 @@
 package app
 
 import (
-	"github.com/roboll/helmfile/pkg/helmexec"
-	"github.com/roboll/helmfile/pkg/state"
-	"github.com/roboll/helmfile/pkg/testhelper"
 	"os"
 	"strings"
 	"testing"
 
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/state"
+	"github.com/roboll/helmfile/pkg/testhelper"
+
 	"gopkg.in/yaml.v2"
 )
 

pkg/app/version/version.go

@@ -0,0 +1,3 @@
+package version
+
+var Version string

pkg/event/bus.go

@@ -20,7 +20,8 @@ type Hook struct {
 }
 
 type event struct {
-	Name string
+	Name  string
+	Error error
 }
 
 type Bus struct {
@@ -37,7 +38,7 @@ type Bus struct {
 	Logger   *zap.SugaredLogger
 }
 
-func (bus *Bus) Trigger(evt string, context map[string]interface{}) (bool, error) {
+func (bus *Bus) Trigger(evt string, evtErr error, context map[string]interface{}) (bool, error) {
 	if bus.Runner == nil {
 		bus.Runner = helmexec.ShellRunner{
 			Dir: bus.BasePath,
@@ -68,7 +69,8 @@ func (bus *Bus) Trigger(evt string, context map[string]interface{}) (bool, error
 			"Environment": bus.Env,
 			"Namespace":   bus.Namespace,
 			"Event": event{
-				Name: evt,
+				Name:  evt,
+				Error: evtErr,
 			},
 		}
 		for k, v := range context {

pkg/event/bus_test.go

@@ -104,7 +104,7 @@ func TestTrigger(t *testing.T) {
 			"Release":         "myrel",
 			"HelmfileCommand": "mycmd",
 		}
-		ok, err := bus.Trigger(c.triggeredEvt, data)
+		ok, err := bus.Trigger(c.triggeredEvt, nil, data)
 
 		if ok != c.expectedResult {
 			t.Errorf("unexpected result for case \"%s\": expected=%v, actual=%v", c.name, c.expectedResult, ok)

pkg/exectest/helm.go

@@ -0,0 +1,171 @@
+package exectest
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/roboll/helmfile/pkg/helmexec"
+)
+
+type ListKey struct {
+	Filter string
+	Flags  string
+}
+
+type DiffKey struct {
+	Name  string
+	Chart string
+	Flags string
+}
+
+type Helm struct {
+	Charts               []string
+	Repo                 []string
+	Releases             []Release
+	Deleted              []Release
+	Lists                map[ListKey]string
+	Diffs                map[DiffKey]error
+	Diffed               []Release
+	FailOnUnexpectedDiff bool
+	FailOnUnexpectedList bool
+
+	UpdateDepsCallbacks map[string]func(string) error
+
+	DiffMutex     *sync.Mutex
+	ChartsMutex   *sync.Mutex
+	ReleasesMutex *sync.Mutex
+}
+
+type Release struct {
+	Name  string
+	Flags []string
+}
+
+type Affected struct {
+	Upgraded []*Release
+	Deleted  []*Release
+	Failed   []*Release
+}
+
+func (helm *Helm) UpdateDeps(chart string) error {
+	if strings.Contains(chart, "error") {
+		return fmt.Errorf("simulated UpdateDeps failure for chart: %s", chart)
+	}
+	helm.Charts = append(helm.Charts, chart)
+
+	if helm.UpdateDepsCallbacks != nil {
+		callback, exists := helm.UpdateDepsCallbacks[chart]
+		if exists {
+			if err := callback(chart); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (helm *Helm) BuildDeps(name, chart string) error {
+	if strings.Contains(chart, "error") {
+		return errors.New("error")
+	}
+	helm.Charts = append(helm.Charts, chart)
+	return nil
+}
+
+func (helm *Helm) SetExtraArgs(args ...string) {
+	return
+}
+func (helm *Helm) SetHelmBinary(bin string) {
+	return
+}
+func (helm *Helm) AddRepo(name, repository, cafile, certfile, keyfile, username, password string) error {
+	helm.Repo = []string{name, repository, cafile, certfile, keyfile, username, password}
+	return nil
+}
+func (helm *Helm) UpdateRepo() error {
+	return nil
+}
+func (helm *Helm) SyncRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
+	if strings.Contains(name, "error") {
+		return errors.New("error")
+	}
+	helm.sync(helm.ReleasesMutex, func() {
+		helm.Releases = append(helm.Releases, Release{Name: name, Flags: flags})
+	})
+	helm.sync(helm.ChartsMutex, func() {
+		helm.Charts = append(helm.Charts, chart)
+	})
+
+	return nil
+}
+func (helm *Helm) DiffRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
+	if helm.DiffMutex != nil {
+		helm.DiffMutex.Lock()
+	}
+	helm.Diffed = append(helm.Diffed, Release{Name: name, Flags: flags})
+	if helm.DiffMutex != nil {
+		helm.DiffMutex.Unlock()
+	}
+	key := DiffKey{Name: name, Chart: chart, Flags: strings.Join(flags, "")}
+	err, ok := helm.Diffs[key]
+	if !ok && helm.FailOnUnexpectedDiff {
+		return fmt.Errorf("unexpected diff with key: %v", key)
+	}
+	return err
+}
+func (helm *Helm) ReleaseStatus(context helmexec.HelmContext, release string, flags ...string) error {
+	if strings.Contains(release, "error") {
+		return errors.New("error")
+	}
+	helm.Releases = append(helm.Releases, Release{Name: release, Flags: flags})
+	return nil
+}
+func (helm *Helm) DeleteRelease(context helmexec.HelmContext, name string, flags ...string) error {
+	if strings.Contains(name, "error") {
+		return errors.New("error")
+	}
+	helm.Deleted = append(helm.Deleted, Release{Name: name, Flags: flags})
+	return nil
+}
+func (helm *Helm) List(context helmexec.HelmContext, filter string, flags ...string) (string, error) {
+	key := ListKey{Filter: filter, Flags: strings.Join(flags, "")}
+	res, ok := helm.Lists[key]
+	if !ok && helm.FailOnUnexpectedList {
+		return "", fmt.Errorf("unexpected list key: %v", key)
+	}
+	return res, nil
+}
+func (helm *Helm) DecryptSecret(context helmexec.HelmContext, name string, flags ...string) (string, error) {
+	return "", nil
+}
+func (helm *Helm) TestRelease(context helmexec.HelmContext, name string, flags ...string) error {
+	if strings.Contains(name, "error") {
+		return errors.New("error")
+	}
+	helm.Releases = append(helm.Releases, Release{Name: name, Flags: flags})
+	return nil
+}
+func (helm *Helm) Fetch(chart string, flags ...string) error {
+	return nil
+}
+func (helm *Helm) Lint(name, chart string, flags ...string) error {
+	return nil
+}
+func (helm *Helm) TemplateRelease(name, chart string, flags ...string) error {
+	return nil
+}
+
+func (helm *Helm) IsHelm3() bool {
+	return false
+}
+
+func (helm *Helm) sync(m *sync.Mutex, f func()) {
+	if m != nil {
+		m.Lock()
+		defer m.Unlock()
+	}
+
+	f()
+}

pkg/helmexec/context.go

@@ -8,15 +8,16 @@ import (
 type HelmContext struct {
 	Tillerless      bool
 	TillerNamespace string
+	HistoryMax      int
 	WorkerIndex     int
 }
 
-func (context *HelmContext) GetTillerlessArgs(helmBinary string) []string {
-	if context.Tillerless {
+func (context *HelmContext) GetTillerlessArgs(helm *execer) []string {
+	if context.Tillerless && !helm.IsHelm3() {
 		if context.TillerNamespace != "" {
-			return []string{"tiller", "run", context.TillerNamespace, "--", helmBinary}
+			return []string{"tiller", "run", context.TillerNamespace, "--", helm.helmBinary}
 		} else {
-			return []string{"tiller", "run", "--", helmBinary}
+			return []string{"tiller", "run", "--", helm.helmBinary}
 		}
 	} else {
 		return []string{}

pkg/helmexec/exec.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -13,17 +14,20 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
-const (
-	command = "helm"
-)
+type decryptedSecret struct {
+	mutex sync.RWMutex
+	bytes []byte
+}
 
 type execer struct {
-	helmBinary      string
-	runner          Runner
-	logger          *zap.SugaredLogger
-	kubeContext     string
-	extra           []string
-	decryptionMutex sync.Mutex
+	helmBinary           string
+	isHelm3              bool
+	runner               Runner
+	logger               *zap.SugaredLogger
+	kubeContext          string
+	extra                []string
+	decryptedSecretMutex sync.Mutex
+	decryptedSecrets     map[string]*decryptedSecret
 }
 
 func NewLogger(writer io.Writer, logLevel string) *zap.SugaredLogger {
@@ -43,16 +47,31 @@ func NewLogger(writer io.Writer, logLevel string) *zap.SugaredLogger {
 	return zap.New(core).Sugar()
 }
 
-// New for running helm commands
-func New(logger *zap.SugaredLogger, kubeContext string) *execer {
-	return &execer{
-		helmBinary:  command,
-		logger:      logger,
-		kubeContext: kubeContext,
-		runner: &ShellRunner{
-			logger: logger,
-		},
+func detectHelm3(helmBinary string, logger *zap.SugaredLogger, runner Runner) bool {
+	// Support explicit opt-in via environment variable
+	if os.Getenv("HELMFILE_HELM3") != "" {
+		return true
 	}
+
+	// Autodetect from `helm verison`
+	bytes, err := runner.Execute(helmBinary, []string{"version", "--client", "--short"}, nil)
+	if err != nil {
+		panic(err)
+	}
+	return strings.HasPrefix(string(bytes), "v3.")
+}
+
+// New for running helm commands
+func New(helmBinary string, logger *zap.SugaredLogger, kubeContext string, runner Runner) *execer {
+	return &execer{
+		helmBinary:       helmBinary,
+		isHelm3:          detectHelm3(helmBinary, logger, runner),
+		logger:           logger,
+		kubeContext:      kubeContext,
+		runner:           runner,
+		decryptedSecrets: make(map[string]*decryptedSecret),
+	}
+
 }
 
 func (helm *execer) SetExtraArgs(args ...string) {
@@ -63,12 +82,15 @@ func (helm *execer) SetHelmBinary(bin string) {
 	helm.helmBinary = bin
 }
 
-func (helm *execer) AddRepo(name, repository, certfile, keyfile, username, password string) error {
+func (helm *execer) AddRepo(name, repository, cafile, certfile, keyfile, username, password string) error {
 	var args []string
 	args = append(args, "repo", "add", name, repository)
 	if certfile != "" && keyfile != "" {
 		args = append(args, "--cert-file", certfile, "--key-file", keyfile)
 	}
+	if cafile != "" {
+		args = append(args, "--ca-file", cafile)
+	}
 	if username != "" && password != "" {
 		args = append(args, "--username", username, "--password", password)
 	}
@@ -85,6 +107,13 @@ func (helm *execer) UpdateRepo() error {
 	return err
 }
 
+func (helm *execer) BuildDeps(name, chart string) error {
+	helm.logger.Infof("Building dependency release=%v, chart=%v", name, chart)
+	out, err := helm.exec([]string{"dependency", "build", chart}, map[string]string{})
+	helm.info(out)
+	return err
+}
+
 func (helm *execer) UpdateDeps(chart string) error {
 	helm.logger.Infof("Updating dependency %v", chart)
 	out, err := helm.exec([]string{"dependency", "update", chart}, map[string]string{})
@@ -92,17 +121,17 @@ func (helm *execer) UpdateDeps(chart string) error {
 	return err
 }
 
-func (helm *execer) BuildDeps(chart string) error {
-	helm.logger.Infof("Building dependency %v", chart)
-	out, err := helm.exec([]string{"dependency", "build", chart}, map[string]string{})
-	helm.info(out)
-	return err
-}
-
 func (helm *execer) SyncRelease(context HelmContext, name, chart string, flags ...string) error {
-	helm.logger.Infof("Upgrading %v", chart)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	helm.logger.Infof("Upgrading release=%v, chart=%v", name, chart)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
+
+	if helm.IsHelm3() {
+		flags = append(flags, "--history-max", strconv.Itoa(context.HistoryMax))
+	} else {
+		env["HELM_TILLER_HISTORY_MAX"] = strconv.Itoa(context.HistoryMax)
+	}
+
 	out, err := helm.exec(append(append(preArgs, "upgrade", "--install", "--reset-values", name, chart), flags...), env)
 	helm.write(out)
 	return err
@@ -110,7 +139,7 @@ func (helm *execer) SyncRelease(context HelmContext, name, chart string, flags .
 
 func (helm *execer) ReleaseStatus(context HelmContext, name string, flags ...string) error {
 	helm.logger.Infof("Getting status %v", name)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
 	out, err := helm.exec(append(append(preArgs, "status", name), flags...), env)
 	helm.write(out)
@@ -119,78 +148,116 @@ func (helm *execer) ReleaseStatus(context HelmContext, name string, flags ...str
 
 func (helm *execer) List(context HelmContext, filter string, flags ...string) (string, error) {
 	helm.logger.Infof("Listing releases matching %v", filter)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
-	out, err := helm.exec(append(append(preArgs, "list", filter), flags...), env)
+	var args []string
+	if helm.IsHelm3() {
+		args = []string{"list", "--filter", filter}
+	} else {
+		args = []string{"list", filter}
+	}
+
+	out, err := helm.exec(append(append(preArgs, args...), flags...), env)
+	// In v2 we have been expecting `helm list FILTER` prints nothing.
+	// In v3 helm still prints the header like `NAME	NAMESPACE	REVISION	UPDATED	STATUS	CHART	APP VERSION`,
+	// which confuses helmfile's existing logic that treats any non-empty output from `helm list` is considered as the indication
+	// of the release to exist.
+	//
+	// This fixes it by removing the header from the v3 output, so that the output is formatted the same as that of v2.
+	if helm.IsHelm3() {
+		lines := strings.Split(string(out), "\n")
+		lines = lines[1:]
+		out = []byte(strings.Join(lines, "\n"))
+	}
 	helm.write(out)
 	return string(out), err
 }
 
 func (helm *execer) DecryptSecret(context HelmContext, name string, flags ...string) (string, error) {
-	// Prevents https://github.com/roboll/helmfile/issues/258
-	helm.decryptionMutex.Lock()
-	defer helm.decryptionMutex.Unlock()
-
 	absPath, err := filepath.Abs(name)
 	if err != nil {
 		return "", err
 	}
-	helm.logger.Infof("Decrypting secret %v", absPath)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
-	env := context.getTillerlessEnv()
-	out, err := helm.exec(append(append(preArgs, "secrets", "dec", absPath), flags...), env)
-	helm.info(out)
-	if err != nil {
-		return "", err
+
+	helm.logger.Debugf("Preparing to decrypt secret %v", absPath)
+	helm.decryptedSecretMutex.Lock()
+
+	secret, ok := helm.decryptedSecrets[absPath]
+
+	// Cache miss
+	if !ok {
+
+		secret = &decryptedSecret{}
+		helm.decryptedSecrets[absPath] = secret
+
+		secret.mutex.Lock()
+		defer secret.mutex.Unlock()
+		helm.decryptedSecretMutex.Unlock()
+
+		helm.logger.Infof("Decrypting secret %v", absPath)
+		preArgs := context.GetTillerlessArgs(helm)
+		env := context.getTillerlessEnv()
+		out, err := helm.exec(append(append(preArgs, "secrets", "dec", absPath), flags...), env)
+		helm.info(out)
+		if err != nil {
+			return "", err
+		}
+
+		// HELM_SECRETS_DEC_SUFFIX is used by the helm-secrets plugin to define the output file
+		decSuffix := os.Getenv("HELM_SECRETS_DEC_SUFFIX")
+		if len(decSuffix) == 0 {
+			decSuffix = ".yaml.dec"
+		}
+		decFilename := strings.Replace(absPath, ".yaml", decSuffix, 1)
+
+		secretBytes, err := ioutil.ReadFile(decFilename)
+		if err != nil {
+			return "", err
+		}
+		secret.bytes = secretBytes
+
+		if err := os.Remove(decFilename); err != nil {
+			return "", err
+		}
+
+	} else {
+		// Cache hit
+		helm.logger.Debugf("Found secret in cache %v", absPath)
+
+		secret.mutex.RLock()
+		helm.decryptedSecretMutex.Unlock()
+		defer secret.mutex.RUnlock()
 	}
 
 	tmpFile, err := ioutil.TempFile("", "secret")
 	if err != nil {
 		return "", err
 	}
-	defer tmpFile.Close()
-
-	// HELM_SECRETS_DEC_SUFFIX is used by the helm-secrets plugin to define the output file
-	decSuffix := os.Getenv("HELM_SECRETS_DEC_SUFFIX")
-	if len(decSuffix) == 0 {
-		decSuffix = ".yaml.dec"
-	}
-	decFilename := strings.Replace(absPath, ".yaml", decSuffix, 1)
-
-	// os.Rename seems to results in "cross-device link` errors in some cases
-	// Instead of moving, copy it to the destination temp file as a work-around
-	// See https://github.com/roboll/helmfile/issues/251#issuecomment-417166296f
-	decFile, err := os.Open(decFilename)
+	_, err = tmpFile.Write(secret.bytes)
 	if err != nil {
 		return "", err
 	}
-	defer decFile.Close()
-
-	_, err = io.Copy(tmpFile, decFile)
-	if err != nil {
-		return "", err
-	}
-
-	if err := decFile.Close(); err != nil {
-		return "", err
-	}
-
-	if err := os.Remove(decFilename); err != nil {
-		return "", err
-	}
 
 	return tmpFile.Name(), err
 }
 
-func (helm *execer) TemplateRelease(chart string, flags ...string) error {
-	out, err := helm.exec(append([]string{"template", chart}, flags...), map[string]string{})
+func (helm *execer) TemplateRelease(name string, chart string, flags ...string) error {
+	helm.logger.Infof("Templating release=%v, chart=%v", name, chart)
+	var args []string
+	if helm.IsHelm3() {
+		args = []string{"template", name, chart}
+	} else {
+		args = []string{"template", chart, "--name", name}
+	}
+
+	out, err := helm.exec(append(args, flags...), map[string]string{})
 	helm.write(out)
 	return err
 }
 
 func (helm *execer) DiffRelease(context HelmContext, name, chart string, flags ...string) error {
-	helm.logger.Infof("Comparing %v %v", name, chart)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	helm.logger.Infof("Comparing release=%v, chart=%v", name, chart)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
 	out, err := helm.exec(append(append(preArgs, "diff", "upgrade", "--reset-values", "--allow-unreleased", name, chart), flags...), env)
 	// Do our best to write STDOUT only when diff existed
@@ -216,8 +283,8 @@ func (helm *execer) DiffRelease(context HelmContext, name, chart string, flags .
 	return err
 }
 
-func (helm *execer) Lint(chart string, flags ...string) error {
-	helm.logger.Infof("Linting %v", chart)
+func (helm *execer) Lint(name, chart string, flags ...string) error {
+	helm.logger.Infof("Linting release=%v, chart=%v", name, chart)
 	out, err := helm.exec(append([]string{"lint", chart}, flags...), map[string]string{})
 	helm.write(out)
 	return err
@@ -232,7 +299,7 @@ func (helm *execer) Fetch(chart string, flags ...string) error {
 
 func (helm *execer) DeleteRelease(context HelmContext, name string, flags ...string) error {
 	helm.logger.Infof("Deleting %v", name)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
 	out, err := helm.exec(append(append(preArgs, "delete", name), flags...), env)
 	helm.write(out)
@@ -241,9 +308,10 @@ func (helm *execer) DeleteRelease(context HelmContext, name string, flags ...str
 
 func (helm *execer) TestRelease(context HelmContext, name string, flags ...string) error {
 	helm.logger.Infof("Testing %v", name)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
-	out, err := helm.exec(append(append(preArgs, "test", name), flags...), env)
+	args := []string{"test", name}
+	out, err := helm.exec(append(append(preArgs, args...), flags...), env)
 	helm.write(out)
 	return err
 }
@@ -274,3 +342,7 @@ func (helm *execer) write(out []byte) {
 		fmt.Printf("%s\n", out)
 	}
 }
+
+func (helm *execer) IsHelm3() bool {
+	return helm.isHelm3
+}

pkg/helmexec/exec_test.go

@@ -20,12 +20,11 @@ type mockRunner struct {
 }
 
 func (mock *mockRunner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) {
-	return []byte{}, nil
+	return mock.output, mock.err
 }
 
 func MockExecer(logger *zap.SugaredLogger, kubeContext string) *execer {
-	execer := New(logger, kubeContext)
-	execer.runner = &mockRunner{}
+	execer := New("helm", logger, kubeContext, &mockRunner{})
 	return execer
 }
 
@@ -33,8 +32,7 @@ func MockExecer(logger *zap.SugaredLogger, kubeContext string) *execer {
 
 func TestNewHelmExec(t *testing.T) {
 	buffer := bytes.NewBufferString("something")
-	logger := NewLogger(buffer, "debug")
-	helm := New(logger, "dev")
+	helm := MockExecer(NewLogger(buffer, "debug"), "dev")
 	if helm.kubeContext != "dev" {
 		t.Error("helmexec.New() - kubeContext")
 	}
@@ -47,7 +45,7 @@ func TestNewHelmExec(t *testing.T) {
 }
 
 func Test_SetExtraArgs(t *testing.T) {
-	helm := New(NewLogger(os.Stdout, "info"), "dev")
+	helm := MockExecer(NewLogger(os.Stdout, "info"), "dev")
 	helm.SetExtraArgs()
 	if len(helm.extra) != 0 {
 		t.Error("helmexec.SetExtraArgs() - passing no arguments should not change extra field")
@@ -63,7 +61,7 @@ func Test_SetExtraArgs(t *testing.T) {
 }
 
 func Test_SetHelmBinary(t *testing.T) {
-	helm := New(NewLogger(os.Stdout, "info"), "dev")
+	helm := MockExecer(NewLogger(os.Stdout, "info"), "dev")
 	if helm.helmBinary != "helm" {
 		t.Error("helmexec.command - default command is not helm")
 	}
@@ -77,7 +75,7 @@ func Test_AddRepo(t *testing.T) {
 	var buffer bytes.Buffer
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
-	helm.AddRepo("myRepo", "https://repo.example.com/", "cert.pem", "key.pem", "", "")
+	helm.AddRepo("myRepo", "https://repo.example.com/", "", "cert.pem", "key.pem", "", "")
 	expected := `Adding repo myRepo https://repo.example.com/
 exec: helm repo add myRepo https://repo.example.com/ --cert-file cert.pem --key-file key.pem --kube-context dev
 exec: helm repo add myRepo https://repo.example.com/ --cert-file cert.pem --key-file key.pem --kube-context dev: 
@@ -87,7 +85,17 @@ exec: helm repo add myRepo https://repo.example.com/ --cert-file cert.pem --key-
 	}
 
 	buffer.Reset()
-	helm.AddRepo("myRepo", "https://repo.example.com/", "", "", "", "")
+	helm.AddRepo("myRepo", "https://repo.example.com/", "ca.crt", "", "", "", "")
+	expected = `Adding repo myRepo https://repo.example.com/
+exec: helm repo add myRepo https://repo.example.com/ --ca-file ca.crt --kube-context dev
+exec: helm repo add myRepo https://repo.example.com/ --ca-file ca.crt --kube-context dev: 
+`
+	if buffer.String() != expected {
+		t.Errorf("helmexec.AddRepo()\nactual = %v\nexpect = %v", buffer.String(), expected)
+	}
+
+	buffer.Reset()
+	helm.AddRepo("myRepo", "https://repo.example.com/", "", "", "", "", "")
 	expected = `Adding repo myRepo https://repo.example.com/
 exec: helm repo add myRepo https://repo.example.com/ --kube-context dev
 exec: helm repo add myRepo https://repo.example.com/ --kube-context dev: 
@@ -97,7 +105,7 @@ exec: helm repo add myRepo https://repo.example.com/ --kube-context dev:
 	}
 
 	buffer.Reset()
-	helm.AddRepo("myRepo", "https://repo.example.com/", "", "", "example_user", "example_password")
+	helm.AddRepo("myRepo", "https://repo.example.com/", "", "", "", "example_user", "example_password")
 	expected = `Adding repo myRepo https://repo.example.com/
 exec: helm repo add myRepo https://repo.example.com/ --username example_user --password example_password --kube-context dev
 exec: helm repo add myRepo https://repo.example.com/ --username example_user --password example_password --kube-context dev: 
@@ -126,7 +134,7 @@ func Test_SyncRelease(t *testing.T) {
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
 	helm.SyncRelease(HelmContext{}, "release", "chart", "--timeout 10", "--wait")
-	expected := `Upgrading chart
+	expected := `Upgrading release=release, chart=chart
 exec: helm upgrade --install --reset-values release chart --timeout 10 --wait --kube-context dev
 exec: helm upgrade --install --reset-values release chart --timeout 10 --wait --kube-context dev: 
 `
@@ -136,7 +144,7 @@ exec: helm upgrade --install --reset-values release chart --timeout 10 --wait --
 
 	buffer.Reset()
 	helm.SyncRelease(HelmContext{}, "release", "chart")
-	expected = `Upgrading chart
+	expected = `Upgrading release=release, chart=chart
 exec: helm upgrade --install --reset-values release chart --kube-context dev
 exec: helm upgrade --install --reset-values release chart --kube-context dev: 
 `
@@ -151,7 +159,7 @@ func Test_SyncReleaseTillerless(t *testing.T) {
 	helm := MockExecer(logger, "dev")
 	helm.SyncRelease(HelmContext{Tillerless: true, TillerNamespace: "foo"}, "release", "chart",
 		"--timeout 10", "--wait")
-	expected := `Upgrading chart
+	expected := `Upgrading release=release, chart=chart
 exec: helm tiller run foo -- helm upgrade --install --reset-values release chart --timeout 10 --wait --kube-context dev
 exec: helm tiller run foo -- helm upgrade --install --reset-values release chart --timeout 10 --wait --kube-context dev: 
 `
@@ -189,8 +197,8 @@ func Test_BuildDeps(t *testing.T) {
 	var buffer bytes.Buffer
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
-	helm.BuildDeps("./chart/foo")
-	expected := `Building dependency ./chart/foo
+	helm.BuildDeps("foo", "./chart/foo")
+	expected := `Building dependency release=foo, chart=./chart/foo
 exec: helm dependency build ./chart/foo --kube-context dev
 exec: helm dependency build ./chart/foo --kube-context dev: 
 `
@@ -200,8 +208,8 @@ exec: helm dependency build ./chart/foo --kube-context dev:
 
 	buffer.Reset()
 	helm.SetExtraArgs("--verify")
-	helm.BuildDeps("./chart/foo")
-	expected = `Building dependency ./chart/foo
+	helm.BuildDeps("foo", "./chart/foo")
+	expected = `Building dependency release=foo, chart=./chart/foo
 exec: helm dependency build ./chart/foo --verify --kube-context dev
 exec: helm dependency build ./chart/foo --verify --kube-context dev: 
 `
@@ -219,10 +227,16 @@ func Test_DecryptSecret(t *testing.T) {
 	if err != nil {
 		t.Errorf("Error: %v", err)
 	}
-	expected := fmt.Sprintf(`Decrypting secret %s/secretName
+	// Run again for caching
+	helm.DecryptSecret(HelmContext{}, "secretName")
+
+	expected := fmt.Sprintf(`Preparing to decrypt secret %v/secretName
+Decrypting secret %s/secretName
 exec: helm secrets dec %s/secretName --kube-context dev
 exec: helm secrets dec %s/secretName --kube-context dev: 
-`, cwd, cwd, cwd)
+Preparing to decrypt secret %s/secretName
+Found secret in cache %s/secretName
+`, cwd, cwd, cwd, cwd, cwd, cwd)
 	if buffer.String() != expected {
 		t.Errorf("helmexec.DecryptSecret()\nactual = %v\nexpect = %v", buffer.String(), expected)
 	}
@@ -233,7 +247,7 @@ func Test_DiffRelease(t *testing.T) {
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
 	helm.DiffRelease(HelmContext{}, "release", "chart", "--timeout 10", "--wait")
-	expected := `Comparing release chart
+	expected := `Comparing release=release, chart=chart
 exec: helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev
 exec: helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev: 
 `
@@ -243,7 +257,7 @@ exec: helm diff upgrade --reset-values --allow-unreleased release chart --timeou
 
 	buffer.Reset()
 	helm.DiffRelease(HelmContext{}, "release", "chart")
-	expected = `Comparing release chart
+	expected = `Comparing release=release, chart=chart
 exec: helm diff upgrade --reset-values --allow-unreleased release chart --kube-context dev
 exec: helm diff upgrade --reset-values --allow-unreleased release chart --kube-context dev: 
 `
@@ -257,7 +271,7 @@ func Test_DiffReleaseTillerless(t *testing.T) {
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
 	helm.DiffRelease(HelmContext{Tillerless: true}, "release", "chart", "--timeout 10", "--wait")
-	expected := `Comparing release chart
+	expected := `Comparing release=release, chart=chart
 exec: helm tiller run -- helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev
 exec: helm tiller run -- helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev: 
 `
@@ -398,8 +412,8 @@ func Test_Lint(t *testing.T) {
 	var buffer bytes.Buffer
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
-	helm.Lint("path/to/chart", "--values", "file.yml")
-	expected := `Linting path/to/chart
+	helm.Lint("release", "path/to/chart", "--values", "file.yml")
+	expected := `Linting release=release, chart=path/to/chart
 exec: helm lint path/to/chart --values file.yml --kube-context dev
 exec: helm lint path/to/chart --values file.yml --kube-context dev: 
 `
@@ -438,7 +452,7 @@ func Test_LogLevels(t *testing.T) {
 		buffer.Reset()
 		logger := NewLogger(&buffer, logLevel)
 		helm := MockExecer(logger, "")
-		helm.AddRepo("myRepo", "https://repo.example.com/", "", "", "example_user", "example_password")
+		helm.AddRepo("myRepo", "https://repo.example.com/", "", "", "", "example_user", "example_password")
 		if buffer.String() != expected {
 			t.Errorf("helmexec.AddRepo()\nactual = %v\nexpect = %v", buffer.String(), expected)
 		}
@@ -478,3 +492,31 @@ func Test_mergeEnv(t *testing.T) {
 		t.Errorf("mergeEnv()\nactual = %v\nexpect = %v", actual, expected)
 	}
 }
+
+func Test_Template(t *testing.T) {
+	var buffer bytes.Buffer
+	logger := NewLogger(&buffer, "debug")
+	helm := MockExecer(logger, "dev")
+	helm.TemplateRelease("release", "path/to/chart", "--values", "file.yml")
+	expected := `Templating release=release, chart=path/to/chart
+exec: helm template path/to/chart --name release --values file.yml --kube-context dev
+exec: helm template path/to/chart --name release --values file.yml --kube-context dev: 
+`
+	if buffer.String() != expected {
+		t.Errorf("helmexec.Template()\nactual = %v\nexpect = %v", buffer.String(), expected)
+	}
+}
+
+func Test_IsHelm3(t *testing.T) {
+	helm2Runner := mockRunner{output: []byte("Client: v2.16.0+ge13bc94\n")}
+	helm := New("helm", NewLogger(os.Stdout, "info"), "dev", &helm2Runner)
+	if helm.IsHelm3() {
+		t.Error("helmexec.IsHelm3() - Detected Helm 3 with Helm 2 version")
+	}
+
+	helm3Runner := mockRunner{output: []byte("v3.0.0+ge29ce2a\n")}
+	helm = New("helm", NewLogger(os.Stdout, "info"), "dev", &helm3Runner)
+	if !helm.IsHelm3() {
+		t.Error("helmexec.IsHelm3() - Failed to detect Helm 3")
+	}
+}

pkg/helmexec/exit_error.go

@@ -8,8 +8,8 @@ import (
 
 func newExitError(helmCmdPath string, exitStatus int, errorMessage string) ExitError {
 	return ExitError{
-		msg:        fmt.Sprintf("%s exited with status %d:\n%s", filepath.Base(helmCmdPath), exitStatus, indent(strings.TrimSpace(errorMessage))),
-		exitStatus: exitStatus,
+		Message: fmt.Sprintf("%s exited with status %d:\n%s", filepath.Base(helmCmdPath), exitStatus, indent(strings.TrimSpace(errorMessage))),
+		Code:    exitStatus,
 	}
 }
 
@@ -23,14 +23,14 @@ func indent(text string) string {
 
 // ExitError is created whenever your shell command exits with a non-zero exit status
 type ExitError struct {
-	msg        string
-	exitStatus int
+	Message string
+	Code    int
 }
 
 func (e ExitError) Error() string {
-	return e.msg
+	return e.Message
 }
 
 func (e ExitError) ExitStatus() int {
-	return e.exitStatus
+	return e.Code
 }

pkg/helmexec/helmexec.go

@@ -5,22 +5,24 @@ type Interface interface {
 	SetExtraArgs(args ...string)
 	SetHelmBinary(bin string)
 
-	AddRepo(name, repository, certfile, keyfile, username, password string) error
+	AddRepo(name, repository, cafile, certfile, keyfile, username, password string) error
 	UpdateRepo() error
-	BuildDeps(chart string) error
+	BuildDeps(name, chart string) error
 	UpdateDeps(chart string) error
 	SyncRelease(context HelmContext, name, chart string, flags ...string) error
 	DiffRelease(context HelmContext, name, chart string, flags ...string) error
-	TemplateRelease(chart string, flags ...string) error
+	TemplateRelease(name, chart string, flags ...string) error
 	Fetch(chart string, flags ...string) error
-	Lint(chart string, flags ...string) error
+	Lint(name, chart string, flags ...string) error
 	ReleaseStatus(context HelmContext, name string, flags ...string) error
 	DeleteRelease(context HelmContext, name string, flags ...string) error
 	TestRelease(context HelmContext, name string, flags ...string) error
 	List(context HelmContext, filter string, flags ...string) (string, error)
 	DecryptSecret(context HelmContext, name string, flags ...string) (string, error)
+	IsHelm3() bool
 }
 
 type DependencyUpdater interface {
 	UpdateDeps(chart string) error
+	IsHelm3() bool
 }

pkg/helmexec/runner.go

@@ -25,7 +25,7 @@ type Runner interface {
 type ShellRunner struct {
 	Dir string
 
-	logger *zap.SugaredLogger
+	Logger *zap.SugaredLogger
 }
 
 // Execute a shell command
@@ -33,7 +33,7 @@ func (shell ShellRunner) Execute(cmd string, args []string, env map[string]strin
 	preparedCmd := exec.Command(cmd, args...)
 	preparedCmd.Dir = shell.Dir
 	preparedCmd.Env = mergeEnv(os.Environ(), env)
-	return combinedOutput(preparedCmd, shell.logger)
+	return combinedOutput(preparedCmd, shell.Logger)
 }
 
 func combinedOutput(c *exec.Cmd, logger *zap.SugaredLogger) ([]byte, error) {

pkg/maputil/maputil.go

@@ -1,6 +1,9 @@
 package maputil
 
-import "fmt"
+import (
+	"fmt"
+	"strings"
+)
 
 func CastKeysToStrings(s interface{}) (map[string]interface{}, error) {
 	new := map[string]interface{}{}
@@ -60,27 +63,129 @@ func recursivelyStringifyMapKey(v interface{}) (interface{}, error) {
 	return casted_v, nil
 }
 
-func Set(m map[string]interface{}, key []string, value string) map[string]interface{} {
+type arg interface {
+	getMap(map[string]interface{}) map[string]interface{}
+	set(map[string]interface{}, string)
+}
+
+type keyArg struct {
+	key string
+}
+
+func (a keyArg) getMap(m map[string]interface{}) map[string]interface{} {
+	_, ok := m[a.key]
+	if !ok {
+		m[a.key] = map[string]interface{}{}
+	}
+	switch t := m[a.key].(type) {
+	case map[string]interface{}:
+		return t
+	default:
+		panic(fmt.Errorf("unexpected type: %v(%T)", t, t))
+	}
+}
+
+func (a keyArg) set(m map[string]interface{}, value string) {
+	m[a.key] = value
+}
+
+type indexedKeyArg struct {
+	key   string
+	index int
+}
+
+func (a indexedKeyArg) getArray(m map[string]interface{}) []interface{} {
+	_, ok := m[a.key]
+	if !ok {
+		m[a.key] = make([]interface{}, a.index+1)
+	}
+	switch t := m[a.key].(type) {
+	case []interface{}:
+		if len(t) <= a.index {
+			t2 := make([]interface{}, a.index+1)
+			copy(t, t2)
+			t = t2
+		}
+		return t
+	default:
+		panic(fmt.Errorf("unexpected type: %v(%T)", t, t))
+	}
+}
+
+func (a indexedKeyArg) getMap(m map[string]interface{}) map[string]interface{} {
+	t := a.getArray(m)
+	if t[a.index] == nil {
+		t[a.index] = map[string]interface{}{}
+	}
+	switch t := t[a.index].(type) {
+	case map[string]interface{}:
+		return t
+	default:
+		panic(fmt.Errorf("unexpected type: %v(%T)", t, t))
+	}
+}
+
+func (a indexedKeyArg) set(m map[string]interface{}, value string) {
+	t := a.getArray(m)
+	t[a.index] = value
+	m[a.key] = t
+}
+
+func getCursor(key string) arg {
+	key = strings.ReplaceAll(key, "[", " ")
+	key = strings.ReplaceAll(key, "]", " ")
+	k := key
+	idx := 0
+
+	n, err := fmt.Sscanf(key, "%s %d", &k, &idx)
+
+	if n == 2 && err == nil {
+		return indexedKeyArg{
+			key:   k,
+			index: idx,
+		}
+	}
+
+	return keyArg{
+		key: key,
+	}
+}
+
+func ParseKey(key string) []string {
+	r := []string{}
+	part := ""
+	escaped := false
+	for _, rune := range key {
+		split := false
+		switch {
+		case escaped == false && rune == '\\':
+			escaped = true
+			continue
+		case rune == '.':
+			split = escaped == false
+		}
+		escaped = false
+		if split {
+			r = append(r, part)
+			part = ""
+		} else {
+			part = part + string(rune)
+		}
+	}
+	if len(part) > 0 {
+		r = append(r, part)
+	}
+	return r
+}
+
+func Set(m map[string]interface{}, key []string, value string) {
 	if len(key) == 0 {
 		panic(fmt.Errorf("bug: unexpected length of key: %d", len(key)))
 	}
 
-	k := key[0]
-
-	if len(key) == 1 {
-		m[k] = value
-		return m
+	for len(key) > 1 {
+		m, key = getCursor(key[0]).getMap(m), key[1:]
 	}
 
-	remain := key[1:]
-
-	nested, ok := m[k]
-	if !ok {
-		new_m := map[string]interface{}{}
-		nested = Set(new_m, remain, value)
-	}
-
-	m[k] = nested
-
-	return m
+	getCursor(key[0]).set(m, value)
 }

pkg/maputil/maputil_test.go

@@ -59,3 +59,72 @@ func TestMapUtil_IFKeys(t *testing.T) {
 		t.Errorf("unexpected c: expected=C, got=%s", c)
 	}
 }
+
+func TestMapUtil_KeyArg(t *testing.T) {
+	m := map[string]interface{}{}
+
+	key := []string{"a", "b", "c"}
+
+	Set(m, key, "C")
+
+	c := (((m["a"].(map[string]interface{}))["b"]).(map[string]interface{}))["c"]
+
+	if c != "C" {
+		t.Errorf("unexpected c: expected=C, got=%s", c)
+	}
+}
+
+func TestMapUtil_IndexedKeyArg(t *testing.T) {
+	m := map[string]interface{}{}
+
+	key := []string{"a", "b[0]", "c"}
+
+	Set(m, key, "C")
+
+	c := (((m["a"].(map[string]interface{}))["b"].([]interface{}))[0].(map[string]interface{}))["c"]
+
+	if c != "C" {
+		t.Errorf("unexpected c: expected=C, got=%s", c)
+	}
+}
+
+type parseKeyTc struct {
+	key    string
+	result map[int]string
+}
+
+func TestMapUtil_ParseKey(t *testing.T) {
+	tcs := []parseKeyTc{
+		{
+			key: `a.b.c`,
+			result: map[int]string{
+				0: "a",
+				1: "b",
+				2: "c",
+			},
+		},
+		{
+			key: `a\.b.c`,
+			result: map[int]string{
+				0: "a.b",
+				1: "c",
+			},
+		},
+		{
+			key: `a\.b\.c`,
+			result: map[int]string{
+				0: "a.b.c",
+			},
+		},
+	}
+
+	for _, tc := range tcs {
+		parts := ParseKey(tc.key)
+
+		for index, value := range tc.result {
+			if parts[index] != value {
+				t.Errorf("unexpected key part[%d]: expected=%s, got=%s", index, value, parts[index])
+			}
+		}
+	}
+}

pkg/remote/remote.go

@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"github.com/hashicorp/go-getter"
 	"github.com/hashicorp/go-getter/helper/url"
+	"go.uber.org/multierr"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
+	"os"
 	"path/filepath"
 	"strings"
 )
@@ -80,6 +82,9 @@ func (r *Remote) GetBytes(goGetterSrc string) ([]byte, error) {
 // If the argument was an URL, it fetches the remote directory contained within the URL,
 // and returns the path to the file in the fetched directory
 func (r *Remote) Locate(urlOrPath string) (string, error) {
+	if r.FileExists(urlOrPath) || r.DirExists(urlOrPath) {
+		return urlOrPath, nil
+	}
 	fetched, err := r.Fetch(urlOrPath)
 	if err != nil {
 		switch err.(type) {
@@ -177,8 +182,10 @@ func (r *Remote) Fetch(goGetterSrc string) (string, error) {
 
 	cached := false
 
+	// e.g. .helmfile/cache/https_github_com_cloudposse_helmfiles_git.ref=0.xx.0
 	getterDst := filepath.Join(cacheBaseDir, cacheKey)
 
+	// e.g. $PWD/.helmfile/cache/https_github_com_cloudposse_helmfiles_git.ref=0.xx.0
 	cacheDirPath := filepath.Join(r.Home, getterDst)
 
 	r.Logger.Debugf("home: %s", r.Home)
@@ -213,7 +220,11 @@ func (r *Remote) Fetch(goGetterSrc string) (string, error) {
 
 		r.Logger.Debugf("downloading %s to %s", getterSrc, getterDst)
 
-		if err := r.Getter.Get(r.Home, getterSrc, getterDst); err != nil {
+		if err := r.Getter.Get(r.Home, getterSrc, cacheDirPath); err != nil {
+			rmerr := os.RemoveAll(cacheDirPath)
+			if rmerr != nil {
+				return "", multierr.Append(err, rmerr)
+			}
 			return "", err
 		}
 	}

pkg/state/chart_dependency.go

@@ -3,6 +3,9 @@ package state
 import (
 	"fmt"
 	"github.com/Masterminds/semver"
+	goversion "github.com/hashicorp/go-version"
+	"github.com/r3labs/diff"
+	"github.com/roboll/helmfile/pkg/app/version"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
@@ -49,6 +52,7 @@ type ChartRequirements struct {
 }
 
 type ChartLockedRequirements struct {
+	Version              string                    `yaml:"version"`
 	ResolvedDependencies []ResolvedChartDependency `yaml:"dependencies"`
 	Digest               string                    `yaml:"digest"`
 	Generated            string                    `yaml:"generated"`
@@ -194,6 +198,7 @@ func (st *HelmState) updateDependenciesInTempDir(shell helmexec.DependencyUpdate
 	}
 
 	if len(unresolved.deps) == 0 {
+		st.logger.Warnf("There are no repositories defined in your helmfile.yaml.\nThis means helmfile cannot update your dependencies or create a lock file.\nSee https://github.com/roboll/helmfile/issues/878 for more information.")
 		return st, nil
 	}
 
@@ -278,8 +283,31 @@ func (m *chartDependencyManager) lockFileName() string {
 }
 
 func (m *chartDependencyManager) Update(shell helmexec.DependencyUpdater, wd string, unresolved *UnresolvedDependencies) (*ResolvedDependencies, error) {
+	if shell.IsHelm3() {
+		return m.updateHelm3(shell, wd, unresolved)
+	}
+	return m.updateHelm2(shell, wd, unresolved)
+}
+
+func (m *chartDependencyManager) updateHelm3(shell helmexec.DependencyUpdater, wd string, unresolved *UnresolvedDependencies) (*ResolvedDependencies, error) {
 	// Generate `Chart.yaml` of the temporary local chart
-	if err := m.writeBytes(filepath.Join(wd, "Chart.yaml"), []byte(fmt.Sprintf("name: %s\n", m.Name))); err != nil {
+	chartMetaContent := fmt.Sprintf("name: %s\nversion: 1.0.0\napiVersion: v2\n", m.Name)
+
+	// Generate `requirements.yaml` of the temporary local chart from the helmfile state
+	reqsContent, err := yaml.Marshal(unresolved.ToChartRequirements())
+	if err != nil {
+		return nil, err
+	}
+	if err := m.writeBytes(filepath.Join(wd, "Chart.yaml"), []byte(chartMetaContent+string(reqsContent))); err != nil {
+		return nil, err
+	}
+
+	return m.doUpdate("Chart.lock", unresolved, shell, wd)
+}
+
+func (m *chartDependencyManager) updateHelm2(shell helmexec.DependencyUpdater, wd string, unresolved *UnresolvedDependencies) (*ResolvedDependencies, error) {
+	// Generate `Chart.yaml` of the temporary local chart
+	if err := m.writeBytes(filepath.Join(wd, "Chart.yaml"), []byte(fmt.Sprintf("name: %s\nversion: 1.0.0\n", m.Name))); err != nil {
 		return nil, err
 	}
 
@@ -292,16 +320,21 @@ func (m *chartDependencyManager) Update(shell helmexec.DependencyUpdater, wd str
 		return nil, err
 	}
 
+	return m.doUpdate("requirements.lock", unresolved, shell, wd)
+}
+
+func (m *chartDependencyManager) doUpdate(chartLockFile string, unresolved *UnresolvedDependencies, shell helmexec.DependencyUpdater, wd string) (*ResolvedDependencies, error) {
+
 	// Generate `requirements.lock` of the temporary local chart by coping `<basename>.lock`
 	lockFile := m.lockFileName()
 
-	lockFileContent, err := m.readBytes(lockFile)
+	originalLockFileContent, err := m.readBytes(lockFile)
 	if err != nil && !os.IsNotExist(err) {
 		return nil, err
 	}
 
-	if lockFileContent != nil {
-		if err := m.writeBytes(filepath.Join(wd, "requirements.lock"), lockFileContent); err != nil {
+	if shell.IsHelm3() && originalLockFileContent != nil {
+		if err := m.writeBytes(filepath.Join(wd, chartLockFile), originalLockFileContent); err != nil {
 			return nil, err
 		}
 	}
@@ -311,7 +344,7 @@ func (m *chartDependencyManager) Update(shell helmexec.DependencyUpdater, wd str
 		return nil, err
 	}
 
-	updatedLockFileContent, err := m.readBytes(filepath.Join(wd, "requirements.lock"))
+	updatedLockFileContent, err := m.readBytes(filepath.Join(wd, chartLockFile))
 	if err != nil {
 		return nil, err
 	}
@@ -326,6 +359,26 @@ func (m *chartDependencyManager) Update(shell helmexec.DependencyUpdater, wd str
 		return lockedReqs.ResolvedDependencies[i].ChartName < lockedReqs.ResolvedDependencies[j].ChartName
 	})
 
+	// Don't update lock file if no dependency updated.
+	if !shell.IsHelm3() && originalLockFileContent != nil {
+		originalLockedReqs := &ChartLockedRequirements{}
+		if err := yaml.Unmarshal(originalLockFileContent, originalLockedReqs); err != nil {
+			return nil, err
+		}
+
+		changes, err := diff.Diff(originalLockedReqs.ResolvedDependencies, lockedReqs.ResolvedDependencies)
+
+		if err != nil {
+			return nil, err
+		}
+
+		if len(changes) == 0 {
+			lockedReqs.Generated = originalLockedReqs.Generated
+		}
+	}
+
+	lockedReqs.Version = version.Version
+
 	updatedLockFileContent, err = yaml.Marshal(lockedReqs)
 
 	if err != nil {
@@ -356,6 +409,25 @@ func (m *chartDependencyManager) Resolve(unresolved *UnresolvedDependencies) (*R
 		return nil, false, err
 	}
 
+	// Make sure go run main.go works and compatible with old lock files.
+	if version.Version != "" && lockedReqs.Version != "" {
+		lockedVersion, err := goversion.NewVersion(lockedReqs.Version)
+
+		if err != nil {
+			return nil, false, err
+		}
+
+		currentVersion, err := goversion.NewVersion(version.Version)
+
+		if err != nil {
+			return nil, false, err
+		}
+
+		if currentVersion.LessThan(lockedVersion) {
+			return nil, false, fmt.Errorf("the lockfile was created by Helmfile %s, which is newer than current %s; Please upgrade to Helmfile %s or greater", lockedVersion.Original(), currentVersion.Original(), lockedVersion.Original())
+		}
+	}
+
 	resolved := &ResolvedDependencies{deps: map[string][]ResolvedChartDependency{}}
 	for _, d := range lockedReqs.ResolvedDependencies {
 		if err := resolved.add(d); err != nil {

pkg/state/create.go

@@ -4,14 +4,16 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
+	"os"
+
 	"github.com/imdario/mergo"
 	"github.com/roboll/helmfile/pkg/environment"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/maputil"
+	"github.com/variantdev/vals"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
-	"io"
-	"os"
 )
 
 type StateLoadError struct {
@@ -32,25 +34,29 @@ func (e *UndefinedEnvError) Error() string {
 }
 
 type StateCreator struct {
-	logger     *zap.SugaredLogger
-	readFile   func(string) ([]byte, error)
-	fileExists func(string) (bool, error)
-	abs        func(string) (string, error)
-	glob       func(string) ([]string, error)
+	logger      *zap.SugaredLogger
+	readFile    func(string) ([]byte, error)
+	fileExists  func(string) (bool, error)
+	abs         func(string) (string, error)
+	glob        func(string) ([]string, error)
+	helm        helmexec.Interface
+	valsRuntime vals.Evaluator
 
 	Strict bool
 
 	LoadFile func(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*HelmState, error)
 }
 
-func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error)) *StateCreator {
+func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), helm helmexec.Interface, valsRuntime vals.Evaluator) *StateCreator {
 	return &StateCreator{
-		logger:     logger,
-		readFile:   readFile,
-		fileExists: fileExists,
-		abs:        abs,
-		glob:       glob,
-		Strict:     true,
+		logger:      logger,
+		readFile:    readFile,
+		fileExists:  fileExists,
+		abs:         abs,
+		glob:        glob,
+		Strict:      true,
+		helm:        helm,
+		valsRuntime: valsRuntime,
 	}
 }
 
@@ -60,6 +66,7 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState,
 
 	state.FilePath = file
 	state.basePath = baseDir
+	state.helm = c.helm
 
 	decoder := yaml.NewDecoder(bytes.NewReader(content))
 	if !c.Strict {
@@ -103,15 +110,16 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState,
 	state.removeFile = os.Remove
 	state.fileExists = c.fileExists
 	state.glob = c.glob
+	state.valsRuntime = c.valsRuntime
 
 	return &state, nil
 }
 
 // LoadEnvValues loads environment values files relative to the `baseDir`
-func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *environment.Environment) (*HelmState, error) {
+func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *environment.Environment, failOnMissingEnv bool) (*HelmState, error) {
 	state := *target
 
-	e, err := state.loadEnvValues(env, ctxEnv, c.readFile, c.glob)
+	e, err := state.loadEnvValues(env, failOnMissingEnv, ctxEnv, c.readFile, c.glob)
 	if err != nil {
 		return nil, &StateLoadError{fmt.Sprintf("failed to read %s", state.FilePath), err}
 	}
@@ -127,6 +135,7 @@ func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *envi
 }
 
 // Parses YAML into HelmState, while loading environment values files relative to the `baseDir`
+// evaluateBases=true means that this is NOT a base helmfile
 func (c *StateCreator) ParseAndLoad(content []byte, baseDir, file string, envName string, evaluateBases bool, envValues *environment.Environment) (*HelmState, error) {
 	state, err := c.Parse(content, baseDir, file)
 	if err != nil {
@@ -137,14 +146,14 @@ func (c *StateCreator) ParseAndLoad(content []byte, baseDir, file string, envNam
 		if len(state.Bases) > 0 {
 			return nil, errors.New("nested `base` helmfile is unsupported. please submit a feature request if you need this!")
 		}
+	} else {
+		state, err = c.loadBases(envValues, state, baseDir)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	state, err = c.loadBases(envValues, state, baseDir)
-	if err != nil {
-		return nil, err
-	}
-
-	state, err = c.LoadEnvValues(state, envName, envValues)
+	state, err = c.LoadEnvValues(state, envName, envValues, evaluateBases)
 	if err != nil {
 		return nil, err
 	}
@@ -174,7 +183,7 @@ func (c *StateCreator) loadBases(envValues *environment.Environment, st *HelmSta
 	return layers[0], nil
 }
 
-func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, readFile func(string) ([]byte, error), glob func(string) ([]string, error)) (*environment.Environment, error) {
+func (st *HelmState) loadEnvValues(name string, failOnMissingEnv bool, ctxEnv *environment.Environment, readFile func(string) ([]byte, error), glob func(string) ([]string, error)) (*environment.Environment, error) {
 	envVals := map[string]interface{}{}
 	envSpec, ok := st.Environments[name]
 	if ok {
@@ -185,7 +194,6 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 		}
 
 		if len(envSpec.Secrets) > 0 {
-			helm := helmexec.New(st.logger, "")
 
 			var envSecretFiles []string
 			for _, urlOrPath := range envSpec.Secrets {
@@ -199,43 +207,11 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 
 				envSecretFiles = append(envSecretFiles, resolved...)
 			}
-
-			for _, path := range envSecretFiles {
-				// Work-around to allow decrypting environment secrets
-				//
-				// We don't have releases loaded yet and therefore unable to decide whether
-				// helmfile should use helm-tiller to call helm-secrets or not.
-				//
-				// This means that, when you use environment secrets + tillerless setup, you still need a tiller
-				// installed on the cluster, just for decrypting secrets!
-				// Related: https://github.com/futuresimple/helm-secrets/issues/83
-				release := &ReleaseSpec{}
-				flags := st.appendConnectionFlags([]string{}, release)
-				decFile, err := helm.DecryptSecret(st.createHelmContext(release, 0), path, flags...)
-				if err != nil {
-					return nil, err
-				}
-				bytes, err := readFile(decFile)
-				if err != nil {
-					return nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err)
-				}
-				m := map[string]interface{}{}
-				if err := yaml.Unmarshal(bytes, &m); err != nil {
-					return nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err)
-				}
-				// All the nested map key should be string. Otherwise we get strange errors due to that
-				// mergo or reflect is unable to merge map[interface{}]interface{} with map[string]interface{} or vice versa.
-				// See https://github.com/roboll/helmfile/issues/677
-				vals, err := maputil.CastKeysToStrings(m)
-				if err != nil {
-					return nil, err
-				}
-				if err := mergo.Merge(&envVals, &vals, mergo.WithOverride); err != nil {
-					return nil, fmt.Errorf("failed to load \"%s\": %v", path, err)
-				}
+			if err = st.scatterGatherEnvSecretFiles(envSecretFiles, envVals, readFile); err != nil {
+				return nil, err
 			}
 		}
-	} else if ctxEnv == nil && name != DefaultEnv {
+	} else if ctxEnv == nil && name != DefaultEnv && failOnMissingEnv {
 		return nil, &UndefinedEnvError{msg: fmt.Sprintf("environment \"%s\" is not defined", name)}
 	}
 
@@ -254,6 +230,83 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 	return newEnv, nil
 }
 
+func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, envVals map[string]interface{}, readFile func(string) ([]byte, error)) error {
+	var errs []error
+
+	inputs := envSecretFiles
+	inputsSize := len(inputs)
+
+	type secretResult struct {
+		result map[string]interface{}
+		err    error
+		path   string
+	}
+
+	secrets := make(chan string, inputsSize)
+	results := make(chan secretResult, inputsSize)
+	helm := st.helm
+
+	st.scatterGather(0, inputsSize,
+		func() {
+			for _, secretFile := range envSecretFiles {
+				secrets <- secretFile
+			}
+			close(secrets)
+		},
+		func(id int) {
+			for path := range secrets {
+				release := &ReleaseSpec{}
+				flags := st.appendConnectionFlags([]string{}, release)
+				decFile, err := helm.DecryptSecret(st.createHelmContext(release, 0), path, flags...)
+				if err != nil {
+					results <- secretResult{nil, err, path}
+					continue
+				}
+				bytes, err := readFile(decFile)
+				if err != nil {
+					results <- secretResult{nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err), path}
+					continue
+				}
+				m := map[string]interface{}{}
+				if err := yaml.Unmarshal(bytes, &m); err != nil {
+					results <- secretResult{nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err), path}
+					continue
+				}
+				// All the nested map key should be string. Otherwise we get strange errors due to that
+				// mergo or reflect is unable to merge map[interface{}]interface{} with map[string]interface{} or vice versa.
+				// See https://github.com/roboll/helmfile/issues/677
+				vals, err := maputil.CastKeysToStrings(m)
+				if err != nil {
+					results <- secretResult{nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err), path}
+					continue
+				}
+				results <- secretResult{vals, nil, path}
+			}
+		},
+		func() {
+			for i := 0; i < inputsSize; i++ {
+				result := <-results
+				if result.err != nil {
+					errs = append(errs, result.err)
+				} else {
+					if err := mergo.Merge(&envVals, &result.result, mergo.WithOverride); err != nil {
+						errs = append(errs, fmt.Errorf("failed to load environment secrets file \"%s\": %v", result.path, err))
+					}
+				}
+			}
+			close(results)
+		},
+	)
+
+	if len(errs) > 1 {
+		for _, err := range errs {
+			st.logger.Error(err)
+		}
+		return fmt.Errorf("Failed loading environment secrets with %d errors", len(errs))
+	}
+	return nil
+}
+
 func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []interface{}) (map[string]interface{}, error) {
 	envVals := map[string]interface{}{}
 

pkg/state/create_test.go

@@ -1,13 +1,14 @@
 package state
 
 import (
-	"github.com/roboll/helmfile/pkg/testhelper"
-	"go.uber.org/zap"
 	"io/ioutil"
 	"path/filepath"
 	"reflect"
 	"testing"
 
+	"github.com/roboll/helmfile/pkg/testhelper"
+	"go.uber.org/zap"
+
 	. "gotest.tools/assert"
 	"gotest.tools/assert/cmp"
 )
@@ -19,7 +20,7 @@ func createFromYaml(content []byte, file string, env string, logger *zap.Sugared
 		abs:      filepath.Abs,
 		Strict:   true,
 	}
-	return c.ParseAndLoad(content, filepath.Dir(file), file, env, false, nil)
+	return c.ParseAndLoad(content, filepath.Dir(file), file, env, true, nil)
 }
 
 func TestReadFromYaml(t *testing.T) {
@@ -107,7 +108,7 @@ bar: {{ readFile "bar.txt" }}
 	})
 	testFs.Cwd = "/example/path/to"
 
-	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", false, nil)
+	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil, nil).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", true, nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}

pkg/state/environment.go

@@ -1,8 +1,8 @@
 package state
 
 type EnvironmentSpec struct {
-	Values  []interface{} `yaml:"values"`
-	Secrets []string      `yaml:"secrets"`
+	Values  []interface{} `yaml:"values,omitempty"`
+	Secrets []string      `yaml:"secrets,omitempty"`
 
 	// MissingFileHandler instructs helmfile to fail when unable to find a environment values file listed
 	// under `environments.NAME.values`.
@@ -11,5 +11,5 @@ type EnvironmentSpec struct {
 	//
 	// Use "Warn", "Info", or "Debug" if you want helmfile to not fail when a values file is missing, while just leaving
 	// a message about the missing file at the log-level.
-	MissingFileHandler *string `yaml:"missingFileHandler"`
+	MissingFileHandler *string `yaml:"missingFileHandler,omitempty"`
 }

pkg/state/helmx.go

@@ -19,6 +19,10 @@ func (st *HelmState) appendHelmXFlags(flags []string, release *ReleaseSpec) ([]s
 		flags = append(flags, "--dependency", dep)
 	}
 
+	for _, adopt := range release.Adopt {
+		flags = append(flags, "--adopt", adopt)
+	}
+
 	jsonPatches := release.JSONPatches
 	if len(jsonPatches) > 0 {
 		generatedFiles, err := st.generateTemporaryValuesFiles(jsonPatches, release.MissingFileHandler)

pkg/state/release.go

@@ -48,6 +48,80 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
 		}
 	}
 
+	if result.WaitTemplate != nil {
+		ts := *result.WaitTemplate
+		resultTmpl, err := renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err)
+		}
+		result.WaitTemplate = &resultTmpl
+	}
+
+	if result.InstalledTemplate != nil {
+		ts := *result.InstalledTemplate
+		resultTmpl, err := renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err)
+		}
+		result.InstalledTemplate = &resultTmpl
+	}
+
+	if result.TillerlessTemplate != nil {
+		ts := *result.TillerlessTemplate
+		resultTmpl, err := renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err)
+		}
+		result.TillerlessTemplate = &resultTmpl
+	}
+
+	if result.VerifyTemplate != nil {
+		ts := *result.VerifyTemplate
+		resultTmpl, err := renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err)
+		}
+		result.VerifyTemplate = &resultTmpl
+	}
+
+	for key, val := range result.Labels {
+		ts := val
+		s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".labels[%s] = \"%s\": %v", r.Name, key, ts, err)
+		}
+		result.Labels[key] = s.String()
+	}
+
+	if len(result.ValuesTemplate) > 0 {
+		for i, t := range result.ValuesTemplate {
+			switch ts := t.(type) {
+			case map[interface{}]interface{}:
+				serialized, err := yaml.Marshal(ts)
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err)
+				}
+
+				s, err := renderer.RenderTemplateContentToBuffer([]byte(serialized))
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, serialized, err)
+				}
+
+				var deserialized map[interface{}]interface{}
+
+				if err := yaml.Unmarshal(s.Bytes(), &deserialized); err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err)
+				}
+
+				result.ValuesTemplate[i] = deserialized
+			}
+		}
+
+		var newvals []interface{}
+		newvals = append(newvals, result.ValuesTemplate...)
+		result.Values = append(newvals, result.Values...)
+	}
+
 	for i, t := range result.Values {
 		switch ts := t.(type) {
 		case string:
@@ -67,6 +141,50 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
 		result.Secrets[i] = s.String()
 	}
 
+	if len(result.SetValuesTemplate) > 0 {
+		for i, val := range result.SetValuesTemplate {
+			{
+				// name
+				ts := val.Name
+				s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].name = \"%s\": %v", r.Name, i, ts, err)
+				}
+				result.SetValuesTemplate[i].Name = s.String()
+			}
+			{
+				// value
+				ts := val.Value
+				s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].value = \"%s\": %v", r.Name, i, ts, err)
+				}
+				result.SetValuesTemplate[i].Value = s.String()
+			}
+			{
+				// file
+				ts := val.File
+				s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].file = \"%s\": %v", r.Name, i, ts, err)
+				}
+				result.SetValuesTemplate[i].File = s.String()
+			}
+			for j, ts := range val.Values {
+				// values
+				s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].values[%d] = \"%s\": %v", r.Name, i, j, ts, err)
+				}
+				result.SetValuesTemplate[i].Values[j] = s.String()
+			}
+		}
+
+		var newvals []SetValue
+		newvals = append(newvals, result.SetValuesTemplate...)
+		result.SetValues = append(newvals, result.SetValues...)
+	}
+
 	return result, nil
 }
 

pkg/state/state_exec_tmpl.go

@@ -2,9 +2,12 @@ package state
 
 import (
 	"fmt"
+	"reflect"
+
 	"github.com/imdario/mergo"
 	"github.com/roboll/helmfile/pkg/maputil"
 	"github.com/roboll/helmfile/pkg/tmpl"
+	"gopkg.in/yaml.v2"
 )
 
 func (st *HelmState) Values() (map[string]interface{}, error) {
@@ -36,11 +39,60 @@ func (st *HelmState) mustLoadVals() map[string]interface{} {
 func (st *HelmState) valuesFileTemplateData() EnvironmentTemplateData {
 	return EnvironmentTemplateData{
 		Environment: st.Env,
-		Namespace:   st.Namespace,
+		Namespace:   st.OverrideNamespace,
 		Values:      st.mustLoadVals(),
 	}
 }
 
+func getBoolRefFromStringTemplate(templateRef string) (*bool, error) {
+	var result bool
+	if err := yaml.Unmarshal([]byte(templateRef), &result); err != nil {
+		return nil, fmt.Errorf("failed deserialising string %s: %v", templateRef, err)
+	}
+	return &result, nil
+}
+
+func updateBoolTemplatedValues(r *ReleaseSpec) error {
+
+	if r.InstalledTemplate != nil {
+		if installed, err := getBoolRefFromStringTemplate(*r.InstalledTemplate); err != nil {
+			return fmt.Errorf("installedTemplate: %v", err)
+		} else {
+			r.InstalledTemplate = nil
+			r.Installed = installed
+		}
+	}
+
+	if r.WaitTemplate != nil {
+		if wait, err := getBoolRefFromStringTemplate(*r.WaitTemplate); err != nil {
+			return fmt.Errorf("waitTemplate: %v", err)
+		} else {
+			r.WaitTemplate = nil
+			r.Wait = wait
+		}
+	}
+
+	if r.TillerlessTemplate != nil {
+		if tillerless, err := getBoolRefFromStringTemplate(*r.TillerlessTemplate); err != nil {
+			return fmt.Errorf("tillerlessTemplate: %v", err)
+		} else {
+			r.TillerlessTemplate = nil
+			r.Tillerless = tillerless
+		}
+	}
+
+	if r.VerifyTemplate != nil {
+		if verify, err := getBoolRefFromStringTemplate(*r.VerifyTemplate); err != nil {
+			return fmt.Errorf("verifyTemplate: %v", err)
+		} else {
+			r.VerifyTemplate = nil
+			r.Verify = verify
+		}
+	}
+
+	return nil
+}
+
 func (st *HelmState) ExecuteTemplates() (*HelmState, error) {
 	r := *st
 
@@ -50,17 +102,32 @@ func (st *HelmState) ExecuteTemplates() (*HelmState, error) {
 	}
 
 	for i, rt := range st.Releases {
-		tmplData := releaseTemplateData{
-			Environment: st.Env,
-			Release:     rt,
-			Values:      vals,
+		successFlag := false
+		for it, prev := 0, &rt; it < 6; it++ {
+			tmplData := releaseTemplateData{
+				Environment: st.Env,
+				Release:     *prev,
+				Values:      vals,
+			}
+			renderer := tmpl.NewFileRenderer(st.readFile, st.basePath, tmplData)
+			r, err := rt.ExecuteTemplateExpressions(renderer)
+			if err != nil {
+				return nil, fmt.Errorf("failed executing templates in release \"%s\".\"%s\": %v", st.FilePath, rt.Name, err)
+			}
+			if reflect.DeepEqual(prev, r) {
+				successFlag = true
+				if err := updateBoolTemplatedValues(r); err != nil {
+					return nil, fmt.Errorf("failed executing templates in release \"%s\".\"%s\": %v", st.FilePath, rt.Name, err)
+				}
+				st.Releases[i] = *r
+				break
+			}
+			prev = r
 		}
-		renderer := tmpl.NewFileRenderer(st.readFile, st.basePath, tmplData)
-		r, err := rt.ExecuteTemplateExpressions(renderer)
-		if err != nil {
-			return nil, fmt.Errorf("failed executing templates in release \"%s\".\"%s\": %v", st.FilePath, rt.Name, err)
+		if !successFlag {
+			return nil, fmt.Errorf("failed executing templates in release \"%s\".\"%s\": %s", st.FilePath, rt.Name,
+				"recursive references can't be resolved")
 		}
-		st.Releases[i] = *r
 	}
 
 	return &r, nil

pkg/state/state_exec_tmpl_test.go

@@ -1,11 +1,27 @@
 package state
 
 import (
-	"github.com/roboll/helmfile/pkg/environment"
+	"fmt"
 	"reflect"
+	"strings"
 	"testing"
+
+	"github.com/go-test/deep"
+	"github.com/roboll/helmfile/pkg/environment"
 )
 
+func boolPtrToString(ptr *bool) string {
+	if ptr == nil {
+		return "<nil>"
+	}
+	return fmt.Sprintf("&%t", *ptr)
+}
+
+func ptr(v interface{}) interface{} {
+	r := v
+	return reflect.ValueOf(r).Addr().Interface()
+}
+
 func TestHelmState_executeTemplates(t *testing.T) {
 	tests := []struct {
 		name  string
@@ -13,24 +29,103 @@ func TestHelmState_executeTemplates(t *testing.T) {
 		want  ReleaseSpec
 	}{
 		{
-			name: "Has template expressions in chart, values, and secrets",
+			name: "Has template expressions in chart, values, secrets, version, labels",
 			input: ReleaseSpec{
-				Chart:     "test-charts/{{ .Release.Name }}",
-				Version:   "{{ .Release.Name }}-0.1",
-				Verify:    nil,
-				Name:      "test-app",
-				Namespace: "test-namespace-{{ .Release.Name }}",
-				Values:    []interface{}{"config/{{ .Environment.Name }}/{{ .Release.Name }}/values.yaml"},
-				Secrets:   []string{"config/{{ .Environment.Name }}/{{ .Release.Name }}/secrets.yaml"},
+				Chart:          "test-charts/{{ .Release.Name }}",
+				Version:        "{{ .Release.Name }}-0.1",
+				Name:           "test-app",
+				Namespace:      "test-namespace-{{ .Release.Name }}",
+				ValuesTemplate: []interface{}{"config/{{ .Environment.Name }}/{{ .Release.Name }}/values.yaml"},
+				Secrets:        []string{"config/{{ .Environment.Name }}/{{ .Release.Name }}/secrets.yaml"},
+				Labels:         map[string]string{"id": "{{ .Release.Name }}"},
 			},
 			want: ReleaseSpec{
 				Chart:     "test-charts/test-app",
 				Version:   "test-app-0.1",
-				Verify:    nil,
 				Name:      "test-app",
 				Namespace: "test-namespace-test-app",
 				Values:    []interface{}{"config/test_env/test-app/values.yaml"},
 				Secrets:   []string{"config/test_env/test-app/secrets.yaml"},
+				Labels:    map[string]string{"id": "test-app"},
+			},
+		},
+		{
+			name: "Has template expressions in name with recursive refs",
+			input: ReleaseSpec{
+				Chart:     "test-chart",
+				Name:      "{{ .Release.Labels.id }}-{{ .Release.Namespace }}",
+				Namespace: "dev",
+				Labels:    map[string]string{"id": "{{ .Release.Chart }}"},
+			},
+			want: ReleaseSpec{
+				Chart:     "test-chart",
+				Name:      "test-chart-dev",
+				Namespace: "dev",
+				Labels:    map[string]string{"id": "test-chart"},
+			},
+		},
+		{
+			name: "Has template expressions in boolean values",
+			input: ReleaseSpec{
+				Chart:              "test-chart",
+				Name:               "app-dev",
+				Namespace:          "dev",
+				Labels:             map[string]string{"id": "app"},
+				InstalledTemplate:  func(i string) *string { return &i }(`{{ eq .Release.Labels.id "app" | ternary "yes" "no" }}`),
+				VerifyTemplate:     func(i string) *string { return &i }(`{{ true }}`),
+				Verify:             func(i bool) *bool { return &i }(false),
+				WaitTemplate:       func(i string) *string { return &i }(`{{ false }}`),
+				TillerlessTemplate: func(i string) *string { return &i }(`yes`),
+			},
+			want: ReleaseSpec{
+				Chart:      "test-chart",
+				Name:       "app-dev",
+				Namespace:  "dev",
+				Labels:     map[string]string{"id": "app"},
+				Installed:  func(i bool) *bool { return &i }(true),
+				Verify:     func(i bool) *bool { return &i }(true),
+				Wait:       func(i bool) *bool { return &i }(false),
+				Tillerless: func(i bool) *bool { return &i }(true),
+			},
+		},
+		{
+			name: "Has template in set-values",
+			input: ReleaseSpec{
+				Chart:     "test-charts/chart",
+				Name:      "test-app",
+				Namespace: "dev",
+				SetValuesTemplate: []SetValue{
+					SetValue{Name: "val1", Value: "{{ .Release.Name }}-val1"},
+					SetValue{Name: "val2", File: "{{ .Release.Name }}.yml"},
+					SetValue{Name: "val3", Values: []string{"{{ .Release.Name }}-val2", "{{ .Release.Name }}-val3"}},
+				},
+			},
+			want: ReleaseSpec{
+				Chart:     "test-charts/chart",
+				Name:      "test-app",
+				Namespace: "dev",
+				SetValues: []SetValue{
+					SetValue{Name: "val1", Value: "test-app-val1"},
+					SetValue{Name: "val2", File: "test-app.yml"},
+					SetValue{Name: "val3", Values: []string{"test-app-val2", "test-app-val3"}},
+				},
+			},
+		},
+		{
+			name: "Has template in values (map)",
+			input: ReleaseSpec{
+				Chart:          "test-charts/chart",
+				Verify:         nil,
+				Name:           "app",
+				Namespace:      "dev",
+				ValuesTemplate: []interface{}{map[string]string{"key": "{{ .Release.Name }}-val0"}},
+			},
+			want: ReleaseSpec{
+				Chart:     "test-charts/chart",
+				Verify:    nil,
+				Name:      "app",
+				Namespace: "dev",
+				Values:    []interface{}{map[interface{}]interface{}{"key": "app-val0"}},
 			},
 		},
 	}
@@ -43,9 +138,9 @@ func TestHelmState_executeTemplates(t *testing.T) {
 				HelmDefaults: HelmSpec{
 					KubeContext: "test_context",
 				},
-				Env:          environment.Environment{Name: "test_env"},
-				Namespace:    "test-namespace_",
-				Repositories: nil,
+				Env:               environment.Environment{Name: "test_env"},
+				OverrideNamespace: "test-namespace_",
+				Repositories:      nil,
 				Releases: []ReleaseSpec{
 					tt.input,
 				},
@@ -59,20 +154,102 @@ func TestHelmState_executeTemplates(t *testing.T) {
 
 			actual := r.Releases[0]
 
+			if !reflect.DeepEqual(actual.Name, tt.want.Name) {
+				t.Errorf("expected Name %+v, got %+v", tt.want.Name, actual.Name)
+			}
 			if !reflect.DeepEqual(actual.Chart, tt.want.Chart) {
-				t.Errorf("expected %+v, got %+v", tt.want.Chart, actual.Chart)
+				t.Errorf("expected Chart %+v, got %+v", tt.want.Chart, actual.Chart)
 			}
 			if !reflect.DeepEqual(actual.Namespace, tt.want.Namespace) {
-				t.Errorf("expected %+v, got %+v", tt.want.Namespace, actual.Namespace)
+				t.Errorf("expected Namespace %+v, got %+v", tt.want.Namespace, actual.Namespace)
 			}
-			if !reflect.DeepEqual(actual.Values, tt.want.Values) {
-				t.Errorf("expected %+v, got %+v", tt.want.Values, actual.Values)
+			if diff := deep.Equal(actual.Values, tt.want.Values); diff != nil && len(actual.Values) > 0 {
+				t.Errorf("Values differs \n%+v", strings.Join(diff, "\n"))
 			}
-			if !reflect.DeepEqual(actual.Secrets, tt.want.Secrets) {
-				t.Errorf("expected %+v, got %+v", tt.want.Secrets, actual.Secrets)
+			if diff := deep.Equal(actual.Secrets, tt.want.Secrets); diff != nil && len(actual.Secrets) > 0 {
+				t.Errorf("Secrets differs \n%+v", strings.Join(diff, "\n"))
+			}
+			if diff := deep.Equal(actual.SetValues, tt.want.SetValues); diff != nil && len(actual.SetValues) > 0 {
+				t.Errorf("SetValues differs \n%+v", strings.Join(diff, "\n"))
+			}
+			if diff := deep.Equal(actual.Labels, tt.want.Labels); diff != nil && len(actual.Labels) > 0 {
+				t.Errorf("Labels differs \n%+v", strings.Join(diff, "\n"))
 			}
 			if !reflect.DeepEqual(actual.Version, tt.want.Version) {
-				t.Errorf("expected %+v, got %+v", tt.want.Version, actual.Version)
+				t.Errorf("expected Version %+v, got %+v", tt.want.Version, actual.Version)
+			}
+			if !reflect.DeepEqual(actual.Installed, tt.want.Installed) {
+				t.Errorf("expected actual.Installed %+v, got %+v",
+					boolPtrToString(tt.want.Installed), boolPtrToString(actual.Installed),
+				)
+			}
+			if !reflect.DeepEqual(actual.Tillerless, tt.want.Tillerless) {
+				t.Errorf("expected actual.Tillerless %+v, got %+v",
+					boolPtrToString(tt.want.Tillerless), boolPtrToString(actual.Tillerless),
+				)
+			}
+			if !reflect.DeepEqual(actual.Verify, tt.want.Verify) {
+				t.Errorf("expected actual.Verify %+v, got %+v",
+					boolPtrToString(tt.want.Verify), boolPtrToString(actual.Verify),
+				)
+			}
+			if !reflect.DeepEqual(actual.Wait, tt.want.Wait) {
+				t.Errorf("expected actual.Wait %+v, got %+v",
+					boolPtrToString(tt.want.Wait), boolPtrToString(actual.Wait),
+				)
+			}
+		})
+	}
+}
+
+func TestHelmState_recursiveRefsTemplates(t *testing.T) {
+
+	tests := []struct {
+		name  string
+		input ReleaseSpec
+	}{
+		{
+			name: "Has reqursive references",
+			input: ReleaseSpec{
+				Chart:     "test-charts/{{ .Release.Name }}",
+				Verify:    nil,
+				Name:      "{{ .Release.Labels.id }}",
+				Namespace: "dev",
+				Labels:    map[string]string{"id": "app-{{ .Release.Name }}"},
+			},
+		},
+		{
+			name: "Has unresolvable boolean templates",
+			input: ReleaseSpec{
+				Name:         "app-dev",
+				Chart:        "test-charts/app",
+				Verify:       nil,
+				Namespace:    "dev",
+				WaitTemplate: func(i string) *string { return &i }("hi"),
+			},
+		},
+	}
+
+	for i := range tests {
+		tt := tests[i]
+		t.Run(tt.name, func(t *testing.T) {
+			state := &HelmState{
+				basePath: ".",
+				HelmDefaults: HelmSpec{
+					KubeContext: "test_context",
+				},
+				Env:               environment.Environment{Name: "test_env"},
+				OverrideNamespace: "test-namespace_",
+				Repositories:      nil,
+				Releases: []ReleaseSpec{
+					tt.input,
+				},
+			}
+
+			r, err := state.ExecuteTemplates()
+			if err == nil {
+				t.Errorf("Expected error, got valid response: %v", r)
+				t.FailNow()
 			}
 		})
 	}

pkg/state/state_run.go

@@ -2,8 +2,11 @@ package state
 
 import (
 	"fmt"
-	"github.com/roboll/helmfile/pkg/helmexec"
+	"sort"
 	"sync"
+
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/variantdev/dag/pkg/dag"
 )
 
 type result struct {
@@ -12,11 +15,9 @@ type result struct {
 }
 
 func (st *HelmState) scatterGather(concurrency int, items int, produceInputs func(), receiveInputsAndProduceIntermediates func(int), aggregateIntermediates func()) {
-	numReleases := len(st.Releases)
-	if concurrency < 1 {
-		concurrency = numReleases
-	} else if concurrency > numReleases {
-		concurrency = numReleases
+
+	if concurrency < 1 || concurrency > items {
+		concurrency = items
 	}
 
 	for _, r := range st.Releases {
@@ -52,9 +53,14 @@ func (st *HelmState) scatterGather(concurrency int, items int, produceInputs fun
 
 func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency int,
 	do func(ReleaseSpec, int) error) []error {
+
+	return st.iterateOnReleases(helm, concurrency, st.Releases, do)
+}
+
+func (st *HelmState) iterateOnReleases(helm helmexec.Interface, concurrency int, inputs []ReleaseSpec,
+	do func(ReleaseSpec, int) error) []error {
 	var errs []error
 
-	inputs := st.Releases
 	inputsSize := len(inputs)
 
 	releases := make(chan ReleaseSpec)
@@ -97,3 +103,105 @@ func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency
 
 	return nil
 }
+
+type PlanOpts struct {
+	Reverse bool
+}
+
+func (st *HelmState) PlanReleases(reverse bool) ([][]Release, error) {
+	marked, err := st.SelectReleasesWithOverrides()
+	if err != nil {
+		return nil, err
+	}
+
+	groups, err := SortedReleaseGroups(marked, reverse)
+	if err != nil {
+		return nil, err
+	}
+
+	return groups, nil
+}
+
+func SortedReleaseGroups(releases []Release, reverse bool) ([][]Release, error) {
+	groups, err := GroupReleasesByDependency(releases)
+	if err != nil {
+		return nil, err
+	}
+
+	if reverse {
+		sort.Slice(groups, func(i, j int) bool {
+			return j < i
+		})
+	}
+
+	return groups, nil
+}
+
+func GroupReleasesByDependency(releases []Release) ([][]Release, error) {
+	idToReleases := map[string][]Release{}
+	idToIndex := map[string]int{}
+
+	d := dag.New()
+	for i, r := range releases {
+
+		id := ReleaseToID(&r.ReleaseSpec)
+
+		idToReleases[id] = append(idToReleases[id], r)
+		idToIndex[id] = i
+
+		// Only compute dependencies from non-filtered releases
+		if !r.Filtered {
+			d.Add(id, dag.Dependencies(r.Needs))
+		}
+	}
+
+	for _, r := range releases {
+		if !r.Filtered {
+			for _, n := range r.Needs {
+				if _, ok := idToReleases[n]; !ok {
+					id := ReleaseToID(&r.ReleaseSpec)
+					return nil, fmt.Errorf("%q has dependency to inexistent release %q", id, n)
+				}
+			}
+		}
+	}
+
+	plan, err := d.Plan()
+	if err != nil {
+		return nil, err
+	}
+
+	var result [][]Release
+
+	for groupIndex := 0; groupIndex < len(plan); groupIndex++ {
+		dagNodesInGroup := plan[groupIndex]
+
+		var idsInGroup []string
+		var releasesInGroup []Release
+
+		for _, node := range dagNodesInGroup {
+			idsInGroup = append(idsInGroup, node.Id)
+		}
+
+		// Make the helmfile behavior deterministic for reproducibility and ease of testing
+		// We try to keep the order of definitions to keep backward-compatibility
+		// See https://github.com/roboll/helmfile/issues/988
+		sort.Slice(idsInGroup, func(i, j int) bool {
+			ii := idToIndex[idsInGroup[i]]
+			ij := idToIndex[idsInGroup[j]]
+			return ii < ij
+		})
+
+		for _, id := range idsInGroup {
+			releases, ok := idToReleases[id]
+			if !ok {
+				panic(fmt.Errorf("bug: unexpectedly failed to get releases for id %q", id))
+			}
+			releasesInGroup = append(releasesInGroup, releases...)
+		}
+
+		result = append(result, releasesInGroup)
+	}
+
+	return result, nil
+}

pkg/state/util.go

@@ -1,6 +1,7 @@
 package state
 
 import (
+	"path/filepath"
 	"regexp"
 	"strings"
 )
@@ -38,3 +39,14 @@ func resolveRemoteChart(repoAndChart string) (string, string, bool) {
 
 	return repo, chart, true
 }
+
+// normalizeChart allows for the distinction between a file path reference and repository references.
+// - Any single (or double character) followed by a `/` will be considered a local file reference and
+// 	 be constructed relative to the `base path`.
+// - Everything else is assumed to be an absolute path or an actual <repository>/<chart> reference.
+func normalizeChart(basePath, chart string) string {
+	if !isLocalChart(chart) || chart[0] == '/' {
+		return chart
+	}
+	return filepath.Join(basePath, chart)
+}

pkg/state/util_test.go

@@ -27,6 +27,10 @@ func TestIsLocalChart(t *testing.T) {
 			input:    "./charts/mysubsystem/myapp",
 			expected: true,
 		},
+		{
+			input:    "/charts/mysubsystem/myapp",
+			expected: true,
+		},
 		{
 			// Regression test case for:
 			// * https://github.com/roboll/helmfile/issues/675
@@ -80,6 +84,10 @@ func TestResolveRemortChart(t *testing.T) {
 			input:  "./charts/mysubsystem/myapp",
 			remote: false,
 		},
+		{
+			input:  "/charts/mysubsystem/myapp",
+			remote: false,
+		},
 		{
 			// Regression test case for:
 			// * https://github.com/roboll/helmfile/issues/675
@@ -111,3 +119,29 @@ func TestResolveRemortChart(t *testing.T) {
 		}
 	}
 }
+
+func TestNormalizeChart(t *testing.T) {
+	testcases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "mychart",
+			expected: "/path/to/mychart",
+		},
+		{
+			input:    "/charts/mychart",
+			expected: "/charts/mychart",
+		},
+	}
+
+	for i := range testcases {
+		testcase := testcases[i]
+
+		actual := normalizeChart("/path/to", testcase.input)
+
+		if testcase.expected != actual {
+			t.Fatalf("unexpected result: normalizeChart(\"/path/to\", \"%s\"): expected=%v, got=%v", testcase.input, testcase.expected, actual)
+		}
+	}
+}

pkg/testhelper/diff.go

@@ -0,0 +1,103 @@
+package testhelper
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math"
+	"strings"
+
+	"github.com/aryann/difflib"
+)
+
+func Diff(want, got string, context int) (string, bool) {
+	records := difflib.Diff(
+		strings.Split(want, "\n"),
+		strings.Split(got, "\n"),
+	)
+
+	w := &bytes.Buffer{}
+
+	changed := checkAndPrintRecords(w, records, context)
+
+	return w.String(), changed
+}
+
+func checkAndPrintRecords(w io.Writer, records []difflib.DiffRecord, context int) bool {
+	var changed bool
+	if context >= 0 {
+		distances := calculateDistances(records)
+		omitting := false
+		for i, diff := range records {
+			if diff.Delta != difflib.Common {
+				changed = true
+			}
+			if distances[i] > context {
+				if !omitting {
+					fmt.Fprintln(w, "...")
+					omitting = true
+				}
+			} else {
+				omitting = false
+				fmt.Fprintln(w, formatRecord(diff))
+			}
+		}
+	} else {
+		for _, diff := range records {
+			if diff.Delta != difflib.Common {
+				changed = true
+			}
+			fmt.Fprintln(w, formatRecord(diff))
+		}
+	}
+	return changed
+}
+
+func formatRecord(diff difflib.DiffRecord) string {
+	var prefix string
+	switch diff.Delta {
+	case difflib.RightOnly:
+		prefix = "+ "
+	case difflib.LeftOnly:
+		prefix = "- "
+	case difflib.Common:
+		prefix = "  "
+	}
+
+	return prefix + diff.Payload
+}
+
+// Shamelessly and thankfully copied from https://github.com/databus23/helm-diff/blob/99b8474af7726ca6f57b37b0b8b8f3cd36c991e8/diff/diff.go#L116
+func calculateDistances(diffs []difflib.DiffRecord) map[int]int {
+	distances := map[int]int{}
+
+	// Iterate forwards through diffs, set 'distance' based on closest 'change' before this line
+	change := -1
+	for i, diff := range diffs {
+		if diff.Delta != difflib.Common {
+			change = i
+		}
+		distance := math.MaxInt32
+		if change != -1 {
+			distance = i - change
+		}
+		distances[i] = distance
+	}
+
+	// Iterate backwards through diffs, reduce 'distance' based on closest 'change' after this line
+	change = -1
+	for i := len(diffs) - 1; i >= 0; i-- {
+		diff := diffs[i]
+		if diff.Delta != difflib.Common {
+			change = i
+		}
+		if change != -1 {
+			distance := change - i
+			if distance < distances[i] {
+				distances[i] = distance
+			}
+		}
+	}
+
+	return distances
+}

pkg/tmpl/context_funcs.go

@@ -149,7 +149,7 @@ func FromYaml(str string) (Values, error) {
 	m := Values{}
 
 	if err := yaml.Unmarshal([]byte(str), &m); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%s, offending yaml: %s", err, str)
 	}
 	return m, nil
 }

test/integration/run.sh

@@ -45,9 +45,15 @@ function retry() {
 
 set -e
 info "Using namespace: ${test_ns}"
-info "Using Helm version: $(helm version --short --client | grep -o v.*$)"
-${helm} init --wait --override spec.template.spec.automountServiceAccountToken=true
-${helm} plugin install https://github.com/databus23/helm-diff --version master
+# helm v2
+if helm version --client 2>/dev/null | grep '"v2\.'; then
+  info "Using Helm version: $(helm version --short --client | grep -o v.*$)"
+  ${helm} init --wait --override spec.template.spec.automountServiceAccountToken=true
+# helm v3
+else
+  info "Using Helm version: $(helm version --short | grep -o v.*$)"
+fi
+${helm} plugin install https://github.com/databus23/helm-diff --version v3.0.0-rc.7
 ${kubectl} get namespace ${test_ns} &> /dev/null && warn "Namespace ${test_ns} exists, from a previous test run?"
 $kubectl create namespace ${test_ns} || fail "Could not create namespace ${test_ns}"
 trap "{ $kubectl delete namespace ${test_ns}; }" EXIT # remove namespace whenever we exit this script
@@ -60,6 +66,12 @@ test_start "happypath - simple rollout of httpbin chart"
 info "Diffing ${dir}/happypath.yaml"
 bash -c "${helmfile} -f ${dir}/happypath.yaml diff --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
 
+info "Diffing ${dir}/happypath.yaml without color"
+bash -c "${helmfile} -f ${dir}/happypath.yaml --no-color diff --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
+
+info "Diffing ${dir}/happypath.yaml with limited context"
+bash -c "${helmfile} -f ${dir}/happypath.yaml diff --context 3 --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
+
 info "Templating ${dir}/happypath.yaml"
 ${helmfile} -f ${dir}/happypath.yaml template
 code=$?
@@ -85,6 +97,7 @@ info "Applying ${dir}/happypath.yaml with locked dependencies"
 ${helmfile} -f ${dir}/happypath.yaml apply
 code=$?
 [ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile apply: ${code}"
+${helm} list --namespace=${test_ns} || fail "unable to list releases"
 
 info "Deleting release"
 ${helmfile} -f ${dir}/happypath.yaml delete