feat: GA of Kustomize and K8s manifests support (#1172)

This is the GA version of the helm-x integration #673 developed last year.

You get all the following benefits without an extra helm plugin:

- Ability to add ad-hoc chart dependencies/aliases, without forking the chart (Fixes #876 )
- Ability to patch resulting K8s resources before installing the helm chart
- Ability to install a kustomization as a chart (Requires `kustomize` binary to be available in `$PATH`
- Ability to install a directory of K8s manifests as a chart
- etc.

.circleci/config.yml

@@ -4,7 +4,7 @@ jobs:
 
   build:
     docker:
-      - image: circleci/golang:1.12.4
+      - image: circleci/golang:1.14.2
     environment:
       GOPATH: /go
     working_directory: /home/circleci/workspace/helmfile
@@ -24,12 +24,45 @@ jobs:
           root: ./
           paths:
             - .
+  image:
+    docker:
+    - image: circleci/golang:1.14.2
+    working_directory: /home/circleci/workspace/helmfile
+    steps:
+    - checkout
+    - restore_cache:
+        keys:
+        - go-mod-cache-v1-{{ checksum "./go.sum" }}
+        - go-mod-cache-v1-
+    - setup_remote_docker
+    - run:
+        command: |
+          make tools
+          go mod vendor
+          ORG=roboll BUILD_URL="$CIRCLE_BUILD_URL" make image
+
+  image_helm3:
+    docker:
+    - image: circleci/golang:1.14.2
+    working_directory: /home/circleci/workspace/helmfile
+    steps:
+    - checkout
+    - restore_cache:
+        keys:
+        - go-mod-cache-v1-{{ checksum "./go.sum" }}
+        - go-mod-cache-v1-
+    - setup_remote_docker
+    - run:
+        command: |
+          make tools
+          go mod vendor
+          ORG=roboll BUILD_URL="$CIRCLE_BUILD_URL" make image/helm3
 
   test:
     environment:
       GOPATH: /go
     docker:
-      - image: circleci/golang:1.12.4
+      - image: circleci/golang:1.14.2
     working_directory: /home/circleci/workspace/helmfile
     steps:
       - checkout
@@ -58,7 +91,7 @@ jobs:
       - run:
           name: Install helm
           environment:
-            HELM_VERSION: v2.13.0
+            HELM_VERSION: v2.16.1
           command: |
             HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
             curl -Lo ${HELM_FILENAME} "https://kubernetes-helm.storage.googleapis.com/${HELM_FILENAME}"
@@ -87,11 +120,56 @@ jobs:
             export TERM=xterm
             make integration
 
+  integration_tests_helm3:
+    machine:
+      image: circleci/classic:201808-01
+    environment:
+      CHANGE_MINIKUBE_NONE_USER: true
+      MINIKUBE_WANTUPDATENOTIFICATION: false
+      MINIKUBE_WANTREPORTERRORPROMPT: false
+    steps:
+    - checkout
+    - run: mkdir ~/build
+    - attach_workspace:
+        at: ~/build
+    - run: cp ~/build/helmfile ~/project/helmfile
+    - run:
+        name: Install helm
+        environment:
+          HELM_VERSION: v3.0.0
+        command: |
+          HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
+          curl -Lo ${HELM_FILENAME} "https://get.helm.sh/${HELM_FILENAME}"
+          tar zxf ${HELM_FILENAME} linux-amd64/helm
+          chmod +x linux-amd64/helm
+          sudo mv linux-amd64/helm /usr/local/bin/
+    - run:
+        name: Deploy minikube
+        environment:
+          CHANGE_MINIKUBE_NONE_USER: true
+          K8S_VERSION: v1.12.3
+          MINIKUBE_VERSION: v0.30.0
+        command: |
+          curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl
+          chmod +x kubectl && sudo mv kubectl /usr/local/bin/
+          curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
+          chmod +x minikube && sudo mv minikube /usr/local/bin/
+          sudo -E minikube start --vm-driver=none --kubernetes-version=${K8S_VERSION}
+          sudo -E minikube update-context
+    - run:
+        name: Wait for nodes to become ready
+        command: JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done
+    - run:
+        name: Execute integration tests
+        command: |
+          export TERM=xterm
+          HELMFILE_HELM3=1 make integration
+
 # GITHUB_TOKEN env var must be setup in circleci console
 
   release:
     docker:
-      - image: circleci/golang:1.12.4
+      - image: circleci/golang:1.14.2
     working_directory: /home/circleci/workspace/helmfile
     steps:
     - checkout
@@ -105,22 +183,22 @@ jobs:
           docker login -u="$DOCKER_USER" -p="$DOCKER_PASS" quay.io
           make tools
           go mod vendor
-          ORG=roboll BUILD_URL="$CIRCLE_BUILD_URL" make push release
+          ORG=roboll BUILD_URL="$CIRCLE_BUILD_URL" make push push/helm3 release
 
 workflows:
   version: 2
   build_and_test:
     jobs:
       - build
+      - image
+      - image_helm3
       - test
       - integration_tests:
           requires:
             - build
-          filters:
-            branches:
-              only:
-                - master
-                - /pull.*/
+      - integration_tests_helm3:
+          requires:
+            - build
       - release:
           filters:
             branches:

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.12.4-alpine3.9 as builder
+FROM golang:1.14.2-alpine3.11 as builder
 
 RUN apk add --no-cache make git
 WORKDIR /workspace/helmfile
@@ -7,17 +7,20 @@ RUN make static-linux
 
 # -----------------------------------------------------------------------------
 
-FROM alpine:3.8
+FROM alpine:3.11
 
 RUN apk add --no-cache ca-certificates git bash curl jq
 
-ARG HELM_VERSION=v2.13.0
+ARG HELM_VERSION=v2.16.1
 ARG HELM_LOCATION="https://kubernetes-helm.storage.googleapis.com"
 ARG HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
-ARG HELM_SHA256="15eca6ad225a8279de80c7ced42305e24bc5ac60bb7d96f2d2fa4af86e02c794"
+ARG HELM_SHA256="7eebaaa2da4734242bbcdced62cc32ba8c7164a18792c8acdf16c77abffce202"
 RUN wget ${HELM_LOCATION}/${HELM_FILENAME} && \
+    echo Verifying ${HELM_FILENAME}... && \
     sha256sum ${HELM_FILENAME} | grep -q "${HELM_SHA256}" && \
-    tar zxf ${HELM_FILENAME} && mv /linux-amd64/helm /usr/local/bin/ && \
+    echo Extracting ${HELM_FILENAME}... && \
+    tar zxvf ${HELM_FILENAME} && mv /linux-amd64/helm /usr/local/bin/ && \
+    mv /linux-amd64/tiller /usr/local/bin/ && \
     rm ${HELM_FILENAME} && rm -r /linux-amd64
 
 # using the install documentation found at https://kubernetes.io/docs/tasks/tools/install-kubectl/
@@ -25,12 +28,12 @@ RUN wget ${HELM_LOCATION}/${HELM_FILENAME} && \
 # we should be able to install using apk add.
 ENV KUBECTL_VERSION="v1.14.5"
 ENV KUBECTL_SHA256="26681319de56820a8467c9407e9203d5b15fb010ffc75ac5b99c9945ad0bd28c"
-RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && \
+RUN curl --retry 3 -LO "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && \
     sha256sum kubectl | grep ${KUBECTL_SHA256} && \
     chmod +x kubectl && \
     mv kubectl /usr/local/bin/kubectl
 
-RUN mkdir -p "$(helm home)/plugins"
+RUN ["helm", "init", "--client-only"]
 RUN helm plugin install https://github.com/databus23/helm-diff && \
     helm plugin install https://github.com/futuresimple/helm-secrets && \
     helm plugin install https://github.com/hypnoglow/helm-s3.git && \

Dockerfile.helm3

@@ -0,0 +1,42 @@
+FROM golang:1.14.2-alpine3.11 as builder
+
+RUN apk add --no-cache make git
+WORKDIR /workspace/helmfile
+COPY . /workspace/helmfile
+RUN make static-linux
+
+# -----------------------------------------------------------------------------
+
+FROM alpine:3.11
+
+RUN apk add --no-cache ca-certificates git bash curl jq
+
+ARG HELM_VERSION=v3.2.1
+ARG HELM_LOCATION="https://get.helm.sh"
+ARG HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
+ARG HELM_SHA256="018f9908cb950701a5d59e757653a790c66d8eda288625dbb185354ca6f41f6b"
+RUN wget ${HELM_LOCATION}/${HELM_FILENAME} && \
+    echo Verifying ${HELM_FILENAME}... && \
+    sha256sum ${HELM_FILENAME} | grep -q "${HELM_SHA256}" && \
+    echo Extracting ${HELM_FILENAME}... && \
+    tar zxvf ${HELM_FILENAME} && mv /linux-amd64/helm /usr/local/bin/ && \
+    rm ${HELM_FILENAME} && rm -r /linux-amd64
+
+# using the install documentation found at https://kubernetes.io/docs/tasks/tools/install-kubectl/
+# for now but in a future version of alpine (in the testing version at the time of writing)
+# we should be able to install using apk add.
+ENV KUBECTL_VERSION="v1.14.5"
+ENV KUBECTL_SHA256="26681319de56820a8467c9407e9203d5b15fb010ffc75ac5b99c9945ad0bd28c"
+RUN curl --retry 3 -LO "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && \
+    sha256sum kubectl | grep ${KUBECTL_SHA256} && \
+    chmod +x kubectl && \
+    mv kubectl /usr/local/bin/kubectl
+
+RUN helm plugin install https://github.com/databus23/helm-diff --version v3.0.0-rc.7 && \
+    helm plugin install https://github.com/futuresimple/helm-secrets && \
+    helm plugin install https://github.com/hypnoglow/helm-s3.git && \
+    helm plugin install https://github.com/aslafy-z/helm-git.git
+
+COPY --from=builder /workspace/helmfile/dist/helmfile_linux_amd64 /usr/local/bin/helmfile
+
+CMD ["/usr/local/bin/helmfile"]

Makefile

@@ -26,15 +26,15 @@ integration:
 .PHONY: integration
 
 cross:
-	env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le !s390x' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS}
+	env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le !s390x' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X github.com/roboll/helmfile/pkg/app/version.Version=${TAG}' ${TARGETS}
 .PHONY: cross
 
 static-linux:
-	env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -o "dist/helmfile_linux_amd64" -ldflags '-X main.Version=${TAG}' ${TARGETS}
+	env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -o "dist/helmfile_linux_amd64" -ldflags '-X github.com/roboll/helmfile/pkg/app/version.Version=${TAG}' ${TARGETS}
 .PHONY: static-linux
 
 install:
-	env CGO_ENABLED=0 go install -ldflags '-X main.Version=${TAG}' ${TARGETS}
+	env CGO_ENABLED=0 go install -ldflags '-X github.com/roboll/helmfile/pkg/app/version.Version=${TAG}' ${TARGETS}
 .PHONY: install
 
 clean:
@@ -59,6 +59,13 @@ run: image
 push: image
 	docker push quay.io/${ORG}/helmfile:${TAG}
 
+
+image/helm3:
+	docker build -f Dockerfile.helm3 -t quay.io/${ORG}/helmfile:helm3-${TAG} .
+
+push/helm3: image/helm3
+	docker push quay.io/${ORG}/helmfile:helm3-${TAG}
+
 tools:
 	go get -u github.com/tcnksm/ghr github.com/mitchellh/gox
 .PHONY: tools

README.md

@@ -35,9 +35,9 @@ To avoid upgrades for each iteration of `helm`, the `helmfile` executable delega
 
 ## Configuration
 
-**CAUTION**: This documentation is for the development version of Helmfile. If you are looking for the documentation for any of releases, please switch to the corresponding release tag like [v0.79.1](https://github.com/roboll/helmfile/tree/v0.79.1).
+**CAUTION**: This documentation is for the development version of Helmfile. If you are looking for the documentation for any of releases, please switch to the corresponding release tag like [v0.92.1](https://github.com/roboll/helmfile/tree/v0.92.1).
 
-The default helmfile is `helmfile.yaml`:
+The default name for a helmfile is `helmfile.yaml`:
 
 ```yaml
 # Chart repositories used from within this state file
@@ -69,21 +69,28 @@ repositories:
 
 # context: kube-context # this directive is deprecated, please consider using helmDefaults.kubeContext
 
-#default values to set for args along with dedicated keys that can be set by contributers, cli args take precedence over these
+# Default values to set for args along with dedicated keys that can be set by contributors, cli args take precedence over these.
+# In other words, unset values results in no flags passed to helm.
+# See the helm usage (helm SUBCOMMAND -h) for more info on default values when those flags aren't provided.
 helmDefaults:
   tillerNamespace: tiller-namespace  #dedicated default key for tiller-namespace
   tillerless: false                  #dedicated default key for tillerless
   kubeContext: kube-context          #dedicated default key for kube-context (--kube-context)
-  # additional and global args passed to helm
+  cleanupOnFail: false               #dedicated default key for helm flag --cleanup-on-fail
+  # additional and global args passed to helm (default "")
   args:
     - "--set k=v"
-  # defaults for verify, wait, force, timeout and recreatePods under releases[]
+  # verify the chart before upgrading (only works with packaged charts not directories) (default false)
   verify: true
+  # wait for k8s resources via --wait. (default false)
   wait: true
+  # time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks, and waits on pod/pvc/svc/deployment readiness) (default 300)
   timeout: 600
+  # performs pods restart for the resource if applicable (default false)
   recreatePods: true
-  force: true
-  # enable TLS for request to Tiller
+  # forces resource update through delete/recreate if needed (default false)
+  force: false
+  # enable TLS for request to Tiller (default false)
   tls: true
   # path to TLS CA certificate file (default "$HELM_HOME/ca.pem")
   tlsCACert: "path/to/ca.pem"
@@ -91,6 +98,11 @@ helmDefaults:
   tlsCert: "path/to/cert.pem"
   # path to TLS key file (default "$HELM_HOME/key.pem")
   tlsKey: "path/to/key.pem"
+  # limit the maximum number of revisions saved per release. Use 0 for no limit. (default 10)
+  historyMax: 10
+  # when using helm 3.2+, automatically create release namespaces if they do not exist (default true)
+  createNamespace: true
+
 
 # The desired states of Helm releases.
 #
@@ -99,10 +111,12 @@ releases:
   # Published chart example
   - name: vault                            # name of this release
     namespace: vault                       # target namespace
-    labels:                                  # Arbitrary key value pairs for filtering releases
+    createNamespace: true                  # helm 3.2+ automatically create release namespace (default true)
+    labels:                                # Arbitrary key value pairs for filtering releases
       foo: bar
     chart: roboll/vault-secret-manager     # the chart being installed to create this release, referenced by `repository/chart` syntax
     version: ~1.24.1                       # the semver of the chart. range constraint is supported
+    condition: vault.enabled               # The values lookup key for filtering releases. Corresponds to the boolean value of `vault.enabled`, where `vault` is an arbitrary value
     missingFileHandler: Warn # set to either "Error" or "Warn". "Error" instructs helmfile to fail when unable to find a values or secrets file. When "Warn", it prints the file and continues.
     # Values files used for rendering the chart
     values:
@@ -143,25 +157,23 @@ releases:
     # will attempt to decrypt it using helm-secrets plugin
     secrets:
       - vault_secret.yaml
-    # verify the chart before upgrading (only works with packaged charts not directories)
+    # Override helmDefaults options for verify, wait, timeout, recreatePods and force.
     verify: true
-    # wait for k8s resources via --wait. Defaults to `false`
     wait: true
-    # time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks, and waits on pod/pvc/svc/deployment readiness) (default 300)
     timeout: 60
-    # performs pods restart for the resource if applicable
     recreatePods: true
-    # forces resource update through delete/recreate if needed
-    force: true
-    # set `false` to uninstall on sync
+    force: false
+    # set `false` to uninstall this release on sync.  (default true)
     installed: true
-    # restores previous state in case of failed release
+    # restores previous state in case of failed release (default false)
     atomic: true
-    # name of the tiller namespace
+    # when true, cleans up any new resources created during a failed release (default false)
+    cleanupOnFail: false
+    # name of the tiller namespace (default "")
     tillerNamespace: vault
-    # if true, will use the helm-tiller plugin
+    # if true, will use the helm-tiller plugin (default false)
     tillerless: false
-    # enable TLS for request to Tiller
+    # enable TLS for request to Tiller (default false)
     tls: true
     # path to TLS CA certificate file (default "$HELM_HOME/ca.pem")
     tlsCACert: "path/to/ca.pem"
@@ -172,7 +184,10 @@ releases:
     # --kube-context to be passed to helm commands
     # CAUTION: this doesn't work as expected for `tilerless: true`.
     # See https://github.com/roboll/helmfile/issues/642
+    # (default "", which means the standard kubeconfig, either ~/kubeconfig or the file pointed by $KUBECONFIG environment variable)
     kubeContext: kube-context
+    # limit the maximum number of revisions saved per release. Use 0 for no limit (default 10)
+    historyMax: 10
 
   # Local chart example
   - name: grafana                            # name of this release
@@ -197,7 +212,9 @@ helmfiles:
   # Override state values
   values:
   # Values files merged into the nested state's values
-  - additiona.values.yaml
+  - additional.values.yaml
+  # One important aspect of using values here is that they first need to be defined in the values section
+  # of the origin helmfile, so in this example key1 needs to be in the values or environments.NAME.values of path/to/subhelmfile.yaml
   # Inline state values merged into the nested state's values
   - key1: val1
 - # All the nested state files under `helmfiles:` is processed in the order of definition.
@@ -207,6 +224,9 @@ helmfiles:
   # The nested-state file is locally checked-out along with the remote directory containing it.
   # Therefore all the local paths in the file are resolved relative to the file
   path: git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0
+# If set to "Error", return an error when a subhelmfile points to a
+# non-existent path. The default behavior is to print a warning and continue.
+missingFileHandler: Error
 
 #
 # Advanced Configuration: Environments
@@ -235,7 +255,10 @@ environments:
     values:
     - environment/production/values.yaml
     - myChartVer: 1.0.0
-    ## `secrets.yaml` is decrypted by `helm-secrets` and available via `{{ .Environment.Secrets.KEY }}`
+    # disable vault release processing
+    - vault:
+        enabled: false
+    ## `secrets.yaml` is decrypted by `helm-secrets` and available via `{{ .Environment.Values.KEY }}`
     secrets:
     - environment/production/secrets.yaml
     # Instructs helmfile to fail when unable to find a environment values file listed under `environments.NAME.values`.
@@ -257,6 +280,18 @@ bases:
 - environments.yaml
 - defaults.yaml
 - templates.yaml
+
+#
+# Advanced Configuration: API Capabilities
+#
+# 'helmfile template' renders releases locally without querying an actual cluster,
+# and in this case `.Capabilities.APIVersions` cannot be populated.
+# When a chart queries for a specific CRD, this can lead to unexpected results.
+#
+# Configure a fixed list of api versions to pass to 'helm template' via the --api-versions flag:
+apiVersions:
+- example/v1
+
 ```
 
 ## Templating
@@ -295,6 +330,10 @@ releases:
         value: {{ env "SCHEME" | default "https" }}
 ```
 
+### Note
+
+If you wish to treat your enviroment variables as strings always, even if they are boolean or numeric values you can use `{{ env "ENV_NAME" | quote }}` or `"{{ env "ENV_NAME" }}"`. These approaches also work with `requiredEnv`.
+
 ## installation
 
 - download one of [releases](https://github.com/roboll/helmfile/releases) or
@@ -329,7 +368,7 @@ Congratulations! You now have your first Prometheus deployment running inside yo
 
 Iterate on the `helmfile.yaml` by referencing:
 
-- [Configuration syntax](#configuration-syntax)
+- [Configuration](#configuration)
 - [CLI reference](#cli-reference).
 - [Helmfile Best Practices Guide](https://github.com/roboll/helmfile/blob/master/docs/writing-helmfile.md)
 
@@ -337,13 +376,13 @@ Iterate on the `helmfile.yaml` by referencing:
 
 ```
 NAME:
-   helmfile -
+   helmfile
 
 USAGE:
    helmfile [global options] command [command options] [arguments...]
 
 VERSION:
-   v0.70.0
+   v0.92.1
 
 COMMANDS:
      deps      update charts based on the contents of requirements.yaml
@@ -358,15 +397,19 @@ COMMANDS:
      delete    DEPRECATED: delete releases from state file (helm delete)
      destroy   deletes and then purges releases
      test      test releases from state file (helm test)
+     build     output compiled helmfile state(s) as YAML
+     list      list releases defined in state file
+     help, h   Shows a list of commands or help for one command
 
 GLOBAL OPTIONS:
-   --helm-binary value, -b value           path to helm binary
+   --helm-binary value, -b value           path to helm binary (default: "helm")
    --file helmfile.yaml, -f helmfile.yaml  load config from file or directory. defaults to helmfile.yaml or `helmfile.d`(means `helmfile.d/*.yaml`) in this preference
    --environment default, -e default       specify the environment name. defaults to default
    --state-values-set value                set state values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)
    --state-values-file value               specify state values in a YAML file
    --quiet, -q                             Silence output. Equivalent to log-level warn
    --kube-context value                    Set kubectl context. Uses current context by default
+   --no-color                              Output without color
    --log-level value                       Set log level, default info
    --namespace value, -n value             Set namespace. Uses the namespace set in the context by default, and is available in templates as {{ .Namespace }}
    --selector value, -l value              Only run using the releases that match labels. Labels can take the form of foo=bar or foo!=bar.
@@ -495,7 +538,7 @@ In addition to built-in ones, the following custom template functions are availa
 You can reference a template of values file in your `helmfile.yaml` like below:
 
 ```yaml
-releases
+releases:
 - name: myapp
   chart: mychart
   values:
@@ -557,10 +600,10 @@ releases:
     namespace: {{ requiredEnv "NAME" }}
     chart: roboll/vault-secret-manager
     values:
-    - values.yaml.tmpl
+    - values.yaml.gotmpl
 ```
 
-`values.yaml.tmpl`:
+`values.yaml.gotmpl`:
 
 ```yaml
 db:
@@ -590,11 +633,9 @@ environments:
   production:
 
 releases:
-
-{{ if eq .Environment.Name "production" }}
 - name: newrelic-agent
+  installed: {{ eq .Environment.Name "production" | toYaml }}
   # snip
-{{ end }}
 - name: myapp
   # snip
 ```
@@ -660,12 +701,10 @@ releases:
 - name: myapp-{{ .Values.releaseName }} # release name will be one of `dev` or `prod` depending on selected environment
   values:
   - values.yaml.gotmpl
-
-{{ if eq (.Values.releaseName "prod" ) }}
-# this release would be installed only if selected environment is `production`
 - name: production-specific-release
+  # this release would be installed only if selected environment is `production`
+  installed: {{ eq .Values.releaseName "prod" | toYaml }}
   ...
-{{ end }}
 ```
 
 ### Note
@@ -721,6 +760,50 @@ With the [helm-tiller](https://github.com/rimusz/helm-tiller) plugin installed,
 To enable this mode, you need to define `tillerless: true` and set the `tillerNamespace` in the `helmDefaults` section
 or in the `releases` entries.
 
+## DAG-aware installation/deletion ordering
+
+`needs` controls the order of the installation/deletion of the release:
+
+```yaml
+releases:
+- name: somerelease
+  needs:
+  - [TILLER_NAMESPACE/][NAMESPACE/]anotherelease
+```
+
+All the releases listed under `needs` are installed before(or deleted after) the release itself.
+
+For the following example, `helmfile [sync|apply]` installs releases in this order:
+
+1. logging
+2. servicemesh
+3. myapp1 and myapp2
+
+```yaml
+  - name: myapp1
+    chart: charts/myapp
+    needs:
+    - servicemesh
+    - logging
+  - name: myapp2
+    chart: charts/myapp
+    needs:
+    - servicemesh
+    - logging
+  - name: servicemesh
+    chart: charts/istio
+    needs:
+    - logging
+  - name: logging
+    chart: charts/fluentd
+```
+
+Note that all the releases in a same group is installed concurrently. That is, myapp1 and myapp2 are installed concurrently.
+
+On `helmfile [delete|destroy]`, deletions happen in the reverse order.
+
+That is, `myapp1` and `myapp2` are deleted first, then `servicemesh`, and finally `logging`.
+
 ## Separating helmfile.yaml into multiple independent files
 
 Once your `helmfile.yaml` got to contain too many releases,
@@ -907,7 +990,7 @@ like rollback, history, and so on? This section is for you!
 The combination of `hooks` and [helmify-kustomize](https://gist.github.com/mumoshu/f9d0bd98e0eb77f636f79fc2fb130690)
 enables you to integrate [kustomize](https://github.com/kubernetes-sigs/kustomize) into Helmfile.
 
-That is, you can use `kustommize` to build a local helm chart from a kustomize overlay.
+That is, you can use `kustomize` to build a local helm chart from a kustomize overlay.
 
 Let's assume you have a kustomize project named `foo-kustomize` like this:
 
@@ -992,6 +1075,22 @@ Those features are set using the environment variable `HELMFILE_EXPERIMENTAL`. H
 
 If you want to enable all experimental features set the env var to `HELMFILE_EXPERIMENTAL=true`
 
+## Azure ACR integration
+
+Azure offers helm repository [support for Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-helm-repos) as a preview feature.
+
+To use this you must first `az login` and then `az acr helm repo add -n <MyRegistry>`. This will extract a token for the given ACR and configure `helm` to use it, e.g. `helm repo update` should work straight away.
+
+To use `helmfile` with ACR, on the other hand, you must either include a username/password in the repository definition for the ACR in your `helmfile.yaml` or use the `--skip-deps` switch, e.g. `helmfile template --skip-deps`.
+
+An ACR repository definition in `helmfile.yaml` looks like this:
+
+```
+repositories:
+  - name: <MyRegistry>
+    url: https://<MyRegistry>.azurecr.io/helm/v1/repo
+```
+
 ## Examples
 
 For more examples, see the [examples/README.md](https://github.com/roboll/helmfile/blob/master/examples/README.md) or the [`helmfile`](https://github.com/cloudposse/helmfiles/tree/master/releases) distribution by [Cloud Posse](https://github.com/cloudposse/).

USERS.md

@@ -16,6 +16,8 @@ information to this file.
 | [Vlocity](https://vlocity.com/) | proof-of-concept | | Melbourne, Australia | March 2019 |
 | [transit](https://transit.app/) | production | [Blog post](https://medium.com/@naseem_60378/helmfile-its-like-a-helm-for-your-helm-74a908581599). | Montreal, Canada | March 2019 |
 | [uniqkey](https://uniqkey.eu/) | production | [Wiki Page](https://ocd-scm.github.io/ocd-meta/) | Copenhagen, Denmark | April 2019 |
-| [bitsofinfo](https://github.com/bitsofinfo/helmfile-deploy) | proof-of-concept, stage | Used with [helmfile-deploy](https://github.com/bitsofinfo/helmfile-deploy) to manage releases for dozens of apps | USA | July 2019 |
+| [bitsofinfo](https://github.com/bitsofinfo/helmfile-deploy) | production | Used with [helmfile-deploy](https://github.com/bitsofinfo/helmfile-deploy) to manage releases for dozens of apps | USA | July 2019 |
 | [kloeckner-i](https://www.kloeckner-i.com/) | production | We are deploying our standard tools via helmfile to all our clusters.  | Berlin, Germany | September 2019 |
+| [American Express](https://www.americanexpress.com) | proof-of-concept | Orchestration of both internal cluster workloads and local developer environments. | London, GB | January 2020 |
+| [Sportradar](https://www.sportradar.com) | production | Since mid-2019, we've been deploying our core infrastructure and several application stacks with Helmfile. | St. Gallen, Switzerland | March 2020 |
 <!-- TABLE_END -->

docs/advanced-features.md

@@ -0,0 +1,91 @@
+## Advanced Features
+
+- [Import Configuration Parameters into Helmfile](#import-configuration-parameters-into-helmfile)
+
+### Import Configuration Parameters into Helmfile
+
+Helmfile integrates [vals]() to import configuration parameters from following backends:
+
+- AWS SSM Parameter Store
+- AWS SecretsManager
+- Vault
+- SOPS
+
+See [Vals "Suported Backends"](https://github.com/variantdev/vals#suported-backends) for the full list of available backends.
+
+This feature was implemented in https://github.com/roboll/helmfile/pull/906.
+If you're curious how it's designed and how it works, please consult the pull request.
+
+### Deploy Kustomizations with Helmfile
+
+You can deploy [kustomize](https://github.com/kubernetes-sigs/kustomize) "kustomization"s with Helmfile.
+
+Most of Kustomize operations that is usually done with `kustomize edit` can be done declaratively via Helm values.yaml files.
+
+Under the hood, Helmfile transforms the kustomization into a local chart in a temporary directory so that it can be `helm upgrade --install`ed.
+
+The transformation is done by generating (1)a temporary kustomization from various options and (2)temporary chart from the temporary kustomization.
+
+An example pseudo code for the transformation logic can be written as:
+
+```console
+$ TMPCHART=/tmp/sometmpdir
+$ mkdir -p ${TMPCHART}/templates
+$ somehow_generate_chart_yaml ${TMPCHART}/Chart.yaml
+
+$ TMPKUSTOMIZATION=/tmp/sometmpdir2
+$ somehow_generate_temp_kustomization_yaml ${TMPKUSTOMIZATION}/kustomization.yaml
+$ kustomize build ${TMPKUSTOMIZATION}/kustomization.yaml > ${TMPCHART}/templates/all.yaml 
+```
+
+Let's say you have a `helmfile.yaml` that looks like the below:
+
+```yaml
+releases:
+- name: myapp
+  chart: mykustomization
+  values:
+  - values.yaml
+```
+
+Helmfile firstly generates a temporary `kustomization.yaml` that looks like:
+
+```yaml
+bases:
+- $(ABS_PATH_TO_HELMFILE_YAML}/mykustomization
+```
+
+Followed by the below steps:
+
+- Running `kustomize edit set image $IMAGE` for every `$IMAGE` generated from your values.yaml 
+- Running `kustomize edit set nameprefix $NAMEPREFIX` with the nameprefix specified in your values.yaml
+- Running `kustomize edit set namesuffix $NAMESUFFIX` with the namesuffix specified in your values.yaml
+- Running `kustomize edit set namespace $NS` with the namespace specified in your values.yaml
+
+A `values.yaml` file for kustomization would look like the below:
+
+```yaml
+images:
+# kustomize edit set image mysql=eu.gcr.io/my-project/mysql@canary
+- name: mysql
+  newName: eu.gcr.io/my-project/mysql
+  newTag: canary
+# kustomize edit set image myapp=my-registry/my-app@sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d3
+- name: myapp
+  digest: sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d3
+  newName: my-registry/my-app
+
+# kustomize edit set nameprefix foo-
+namePrefix: foo-
+
+# kustomize edit set namesuffix -bar
+nameSuffix: -bar
+
+# kustomize edit set namespace myapp
+namespace: myapp
+```
+
+At this point, Helmfile can generate a complete kustomization from the base kustomization you specified in `releases[].chart` of your helmfile.yaml and `values.yaml`,
+which can be included in the temporary chart.
+
+After all, Helmfile just installs the temporary chart like standard charts, which allows you to manage everything with Helmfile regardless of each app is declared using a Helm chart or a kustomization.

docs/remote-secrets.md

@@ -0,0 +1,58 @@
+# Secrets 
+
+helmfile can handle secrets using [helm-secrets](https://github.com/zendesk/helm-secrets) plugin or using remote secrets storage 
+(everything that package [vals](https://github.com/variantdev/vals) can handle vault, AWS SSM etc)  
+This section will describe the second use case. 
+
+# Remote secrets 
+
+This paragraph will describe how to use remote secrets storage (vault, SSM etc) in helmfile 
+
+## Fetching single key
+
+To fetch single key from remote secret storage you can use `fetchSecretValue` template function example below
+
+```yaml 
+# helmfile.yaml 
+
+repositories: 
+  - name: stable 
+    url: https://kubernetes-charts.storage.googleapis.com 
+
+environments: 
+  default: 
+    values:
+      - service:
+          password: ref+vault://svc/#pass
+          login: ref+vault://svc/#login
+releases:
+  - name: service 
+    namespace: default
+    labels:
+      cluster: services
+      secrets: vault
+    chart: stable/svc
+    version: 0.1.0
+    values:
+      - service:
+          login: {{ .Values.service.login | fetchSecretValue }} # this will resolve ref+vault://svc/#pass and fetch secret from vault
+          password: {{ .Values.service.password | fetchSecretValue | quote }}
+      # - values/service.yaml.gotmpl   # alternatively 
+```
+## Fetching multiple keys
+Alternatively you can use `expandSecretRefs` to fetch a map of secrets 
+```yaml
+# values/service.yaml.gotmpl
+service:
+{{ .Values.service | expandSecretRefs | toYaml | nindent 2 }}
+```
+
+This will produce
+```yaml
+# values/service.yaml
+service:
+  login: svc-login # fetched from vault
+  password: pass
+  
+```
+

docs/writing-helmfile.md

@@ -32,7 +32,7 @@ Now, you get `1` when there is no `eventApi.replicas` defined in environment val
 
 ## Release Template / Conventional Directory Structure
 
-Introducing helmfile into a large-scale project that involes dozens of releases often results in a lot of repetitions in `helmfile.yaml` files.
+Introducing helmfile into a large-scale project that involves dozens of releases often results in a lot of repetitions in `helmfile.yaml` files.
 
 The example below shows repetitions in `namespace`, `chart`, `values`, and `secrets`:
 
@@ -57,7 +57,7 @@ releases:
   values:
   - "./config/kubernetes-dashboard/values.yaml"
   - "./config/kubernetes-dashboard/{{ .Environment.Name }}.yaml"
-  values:
+  secrets:
   - "./config/kubernetes-dashboard/secrets.yaml"
   - "./config/kubernetes-dashboard/{{ .Environment.Name }}-secrets.yaml"
 ```
@@ -146,7 +146,7 @@ bases:
 - environments.yaml
 
 releases:
-- name: metricbaet
+- name: metricbeat
   chart: stable/metricbeat
 - name: myapp
   chart: mychart
@@ -173,7 +173,7 @@ environments:
 releases:
 - name: myapp
   chart: mychart
-- name: metricbaet
+- name: metricbeat
   chart: stable/metricbeat
 ```
 
@@ -186,7 +186,7 @@ environments:
   production:
 
 releases:
-- name: metricbaet
+- name: metricbeat
   chart: stable/metricbeat
 - name: myapp
   chart: mychart
@@ -204,7 +204,7 @@ Helmfile doesn't merge arrays across layers. That is, the below example doesn't
 
 ```yaml
 releases:
-- name: metricbaet
+- name: metricbeat
   chart: stable/metricbeat
 ---
 releases:
@@ -216,7 +216,7 @@ Helmfile overrides the `releases` array with the latest layer so the resulting s
 
 ```yaml
 releases:
-# metricbeat release disappeared! but that's how helmfiel works
+# metricbeat release disappeared! but that's how helmfile works
 - name: myapp
   chart: mychart
 ```
@@ -228,7 +228,7 @@ A work-around is to treat the state file as a go template and use `readFile` tem
 ```yaml
 templates:
   metricbeat: &metricbeat
-  - name: metricbeat
+    name: metricbeat
     chart: stable/metricbeat
 ```
 

go.mod

@@ -1,27 +1,38 @@
 module github.com/roboll/helmfile
 
-go 1.12
+go 1.14
 
 require (
+	github.com/Azure/azure-sdk-for-go v35.0.0+incompatible // indirect
 	github.com/Masterminds/goutils v1.1.0 // indirect
-	github.com/Masterminds/semver v1.4.1
-	github.com/Masterminds/sprig v2.20.0+incompatible
+	github.com/Masterminds/semver v1.4.2
+	github.com/Masterminds/sprig v2.22.0+incompatible
+	github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a
 	github.com/go-test/deep v1.0.3
-	github.com/google/go-cmp v0.3.0
-	github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c // indirect
+	github.com/golang/mock v1.4.3
+	github.com/google/go-cmp v0.4.0
 	github.com/gosuri/uitable v0.0.3
 	github.com/hashicorp/go-getter v1.3.0
+	github.com/hashicorp/go-retryablehttp v0.6.3 // indirect
+	github.com/hashicorp/go-version v1.2.0
+	github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c // indirect
 	github.com/huandu/xstrings v1.2.0 // indirect
-	github.com/imdario/mergo v0.3.6
-	github.com/pkg/errors v0.8.1 // indirect
+	github.com/imdario/mergo v0.3.9
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/pierrec/lz4 v2.3.0+incompatible // indirect
 	github.com/r3labs/diff v0.0.0-20190801153147-a71de73c46ad
 	github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939
-	github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28
-	go.uber.org/atomic v1.3.2 // indirect
+	github.com/urfave/cli v1.20.0
+	github.com/variantdev/chartify v0.3.0
+	github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363
+	github.com/variantdev/vals v0.4.1-0.20200501114609-9cebe482281c
+	go.mozilla.org/sops v0.0.0-20190912205235-14a22d7a7060 // indirect
 	go.uber.org/multierr v1.1.0
-	go.uber.org/zap v1.8.0
-	gopkg.in/yaml.v2 v2.2.1
+	go.uber.org/zap v1.9.1
+	golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf // indirect
+	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
+	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v2 v2.2.4
 	gotest.tools v2.2.0+incompatible
+	gotest.tools/v3 v3.0.3-0.20200410202438-4e4a41b7851a
 )
-
-replace git.apache.org/thrift.git => github.com/apache/thrift v0.12.0

hooks/build

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+echo "=> Building container"
+docker build \
+  --tag "$IMAGE_NAME" \
+  --file Dockerfile .

hooks/pre_build

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+go()
+{
+  docker run --rm -v "$(pwd):/workspace" -w /workspace golang:latest go $@;
+}
+
+go get -u github.com/tcnksm/ghr github.com/mitchellh/gox
+go mod vendor

hooks/push

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "=> Fetch unshallow origin"
+git fetch --unshallow origin || true
+
+echo "=> Tag image:"
+docker tag "${IMAGE_NAME}" "${DOCKER_REPO}:$(git describe --tags --abbrev=0 HEAD^)"
+docker tag "${IMAGE_NAME}" "${DOCKER_REPO}:$(git describe --tags --abbrev=0 HEAD)"
+
+echo "=> Push images"
+docker push "${DOCKER_REPO}"

main.go

@@ -5,31 +5,27 @@ import (
 	"os"
 	"strings"
 
+	"github.com/roboll/helmfile/pkg/app/version"
+
 	"github.com/roboll/helmfile/pkg/app"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/maputil"
 	"github.com/roboll/helmfile/pkg/state"
 	"github.com/urfave/cli"
 	"go.uber.org/zap"
-	"go.uber.org/zap/zapcore"
 )
 
-var Version string
-
 var logger *zap.SugaredLogger
 
 func configureLogging(c *cli.Context) error {
 	// Valid levels:
 	// https://github.com/uber-go/zap/blob/7e7e266a8dbce911a49554b945538c5b950196b8/zapcore/level.go#L126
 	logLevel := c.GlobalString("log-level")
-	if c.GlobalBool("quiet") {
+	if c.GlobalBool("debug") {
+		logLevel = "debug"
+	} else if c.GlobalBool("quiet") {
 		logLevel = "warn"
 	}
-	var level zapcore.Level
-	err := level.Set(logLevel)
-	if err != nil {
-		return err
-	}
 	logger = helmexec.NewLogger(os.Stderr, logLevel)
 	if c.App.Metadata == nil {
 		// Auto-initialised in 1.19.0
@@ -45,11 +41,12 @@ func main() {
 	cliApp := cli.NewApp()
 	cliApp.Name = "helmfile"
 	cliApp.Usage = ""
-	cliApp.Version = Version
+	cliApp.Version = version.Version
 	cliApp.Flags = []cli.Flag{
 		cli.StringFlag{
 			Name:  "helm-binary, b",
 			Usage: "path to helm binary",
+			Value: app.DefaultHelmBinary,
 		},
 		cli.StringFlag{
 			Name:  "file, f",
@@ -75,6 +72,10 @@ func main() {
 			Name:  "kube-context",
 			Usage: "Set kubectl context. Uses current context by default",
 		},
+		cli.BoolFlag{
+			Name:  "debug",
+			Usage: "Enable verbose output for Helm and set log-level to debug, this disables --quiet/-q effect",
+		},
 		cli.BoolFlag{
 			Name:  "no-color",
 			Usage: "Output without color",
@@ -190,6 +191,10 @@ func main() {
 					Name:  "detailed-exitcode",
 					Usage: "return a non-zero exit code when there are changes",
 				},
+				cli.BoolFlag{
+					Name:  "include-tests",
+					Usage: "enable the diffing of the helm test hooks",
+				},
 				cli.BoolFlag{
 					Name:  "suppress-secrets",
 					Usage: "suppress secrets in the output. highly recommended to specify on CI/CD use-cases",
@@ -235,6 +240,10 @@ func main() {
 					Value: 0,
 					Usage: "maximum number of concurrent downloads of release charts",
 				},
+				cli.BoolFlag{
+					Name:  "validate",
+					Usage: "validate your manifests against the Kubernetes cluster you are currently pointing at",
+				},
 				cli.BoolFlag{
 					Name:  "skip-deps",
 					Usage: "skip running `helm repo update` and `helm dependency build`",
@@ -328,15 +337,31 @@ func main() {
 					Value: 0,
 					Usage: "output NUM lines of context around changes",
 				},
+				cli.BoolFlag{
+					Name:  "detailed-exitcode",
+					Usage: "return a non-zero exit code 2 instead of 0 when there were changes detected AND the changes are synced successfully",
+				},
 				cli.StringFlag{
 					Name:  "args",
 					Value: "",
 					Usage: "pass args to helm exec",
 				},
+				cli.BoolFlag{
+					Name:  "retain-values-files",
+					Usage: "Stop cleaning up values files passed to Helm. Together with --log-level=debug, you can manually rerun helm commands as Helmfile did for debugging purpose",
+				},
+				cli.BoolFlag{
+					Name:  "include-tests",
+					Usage: "enable the diffing of the helm test hooks",
+				},
 				cli.BoolFlag{
 					Name:  "suppress-secrets",
 					Usage: "suppress secrets in the diff output. highly recommended to specify on CI/CD use-cases",
 				},
+				cli.BoolFlag{
+					Name:  "suppress-diff",
+					Usage: "suppress diff in the output. Usable in new installs",
+				},
 				cli.BoolFlag{
 					Name:  "skip-deps",
 					Usage: "skip running `helm repo update` and `helm dependency build`",
@@ -446,7 +471,13 @@ func main() {
 		{
 			Name:  "list",
 			Usage: "list releases defined in state file",
-			Flags: []cli.Flag{},
+			Flags: []cli.Flag{
+				cli.StringFlag{
+					Name:  "output",
+					Value: "",
+					Usage: "output releases list as a json string",
+				},
+			},
 			Action: action(func(run *app.App, c configImpl) error {
 				return run.ListReleases(c)
 			}),
@@ -483,10 +514,10 @@ func NewUrfaveCliConfigImpl(c *cli.Context) (configImpl, error) {
 			ops := strings.Split(optsSet[i], ",")
 			for j := range ops {
 				op := strings.SplitN(ops[j], "=", 2)
-				k := strings.Split(op[0], ".")
+				k := maputil.ParseKey(op[0])
 				v := op[1]
 
-				set = maputil.Set(set, k, v)
+				maputil.Set(set, k, v)
 			}
 		}
 		conf.set = set
@@ -508,13 +539,23 @@ func (c configImpl) Values() []string {
 }
 
 func (c configImpl) Args() string {
-	return c.c.String("args")
+	args := c.c.String("args")
+	enableHelmDebug := c.c.GlobalBool("debug")
+
+	if enableHelmDebug {
+		args = fmt.Sprintf("%s %s", args, "--debug")
+	}
+	return args
 }
 
 func (c configImpl) OutputDir() string {
 	return c.c.String("output-dir")
 }
 
+func (c configImpl) Validate() bool {
+	return c.c.Bool("validate")
+}
+
 func (c configImpl) Concurrency() int {
 	return c.c.Int("concurrency")
 }
@@ -533,10 +574,22 @@ func (c configImpl) DetailedExitcode() bool {
 	return c.c.Bool("detailed-exitcode")
 }
 
+func (c configImpl) RetainValuesFiles() bool {
+	return c.c.Bool("retain-values-files")
+}
+
+func (c configImpl) IncludeTests() bool {
+	return c.c.Bool("include-tests")
+}
+
 func (c configImpl) SuppressSecrets() bool {
 	return c.c.Bool("suppress-secrets")
 }
 
+func (c configImpl) SuppressDiff() bool {
+	return c.c.Bool("suppress-diff")
+}
+
 // DeleteConfig
 
 func (c configImpl) Purge() bool {
@@ -550,9 +603,18 @@ func (c configImpl) Cleanup() bool {
 }
 
 func (c configImpl) Timeout() int {
+	if !c.c.IsSet("timeout") {
+		return state.EmptyTimeout
+	}
 	return c.c.Int("timeout")
 }
 
+// ListConfig
+
+func (c configImpl) Output() string {
+	return c.c.String("output")
+}
+
 // GlobalConfig
 
 func (c configImpl) HelmBinary() string {
@@ -616,11 +678,11 @@ func action(do func(*app.App, configImpl) error) func(*cli.Context) error {
 
 		a := app.New(conf)
 
-		a.ErrorHandler = func(err error) error {
+		if err := do(a, conf); err != nil {
 			return toCliError(implCtx, err)
 		}
 
-		return do(a, conf)
+		return nil
 	}
 }
 

pkg/app/config.go

@@ -40,11 +40,18 @@ type ApplyConfigProvider interface {
 	Set() []string
 	SkipDeps() bool
 
+	IncludeTests() bool
+
 	SuppressSecrets() bool
+	SuppressDiff() bool
+
+	DetailedExitcode() bool
 
 	NoColor() bool
 	Context() int
 
+	RetainValuesFiles() bool
+
 	concurrencyConfig
 	interactive
 	loggingConfig
@@ -68,7 +75,10 @@ type DiffConfigProvider interface {
 	Set() []string
 	SkipDeps() bool
 
+	IncludeTests() bool
+
 	SuppressSecrets() bool
+	SuppressDiff() bool
 
 	DetailedExitcode() bool
 	NoColor() bool
@@ -119,6 +129,7 @@ type TemplateConfigProvider interface {
 
 	Values() []string
 	Set() []string
+	Validate() bool
 	SkipDeps() bool
 	OutputDir() string
 
@@ -145,3 +156,7 @@ type loggingConfig interface {
 type interactive interface {
 	Interactive() bool
 }
+
+type ListConfigProvider interface {
+	Output() string
+}

pkg/app/constants_test.go

@@ -1,9 +1,9 @@
 package app
 
 import (
-	"gotest.tools/assert"
-	is "gotest.tools/assert/cmp"
-	"gotest.tools/env"
+	"gotest.tools/v3/assert"
+	is "gotest.tools/v3/assert/cmp"
+	"gotest.tools/v3/env"
 	"os"
 	"testing"
 )

pkg/app/context.go

@@ -17,6 +17,15 @@ func NewContext() Context {
 func (ctx Context) SyncReposOnce(st *state.HelmState, helm state.RepoUpdater) []error {
 	var errs []error
 
+	hasInstalled := false
+	for _, release := range st.Releases {
+		hasInstalled = hasInstalled || release.Installed == nil || *release.Installed
+	}
+
+	if !hasInstalled {
+		return errs
+	}
+
 	allUpdated := true
 	for _, r := range st.Repositories {
 		_, exists := ctx.updatedRepos[r.Name]

pkg/app/desired_state_file_loader.go

@@ -4,19 +4,24 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"github.com/roboll/helmfile/pkg/helmexec"
 	"path/filepath"
 	"sort"
 
 	"github.com/imdario/mergo"
 	"github.com/roboll/helmfile/pkg/environment"
-	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/state"
+	"github.com/variantdev/vals"
 	"go.uber.org/zap"
 )
 
+const (
+	DefaultHelmBinary = state.DefaultHelmBinary
+)
+
 type desiredStateLoader struct {
-	KubeContext string
-	Reverse     bool
+	overrideKubeContext string
+	overrideHelmBinary  string
 
 	env       string
 	namespace string
@@ -25,9 +30,10 @@ type desiredStateLoader struct {
 	fileExists func(string) (bool, error)
 	abs        func(string) (string, error)
 	glob       func(string) ([]string, error)
+	getHelm    func(*state.HelmState) helmexec.Interface
 
-	logger *zap.SugaredLogger
-	helm   helmexec.Interface
+	logger      *zap.SugaredLogger
+	valsRuntime vals.Evaluator
 }
 
 func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, error) {
@@ -58,7 +64,7 @@ func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, e
 		return nil, err
 	}
 
-	if ld.Reverse {
+	if opts.Reverse {
 		rev := func(i, j int) bool {
 			return j < i
 		}
@@ -66,18 +72,18 @@ func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, e
 		sort.Slice(st.Helmfiles, rev)
 	}
 
-	if ld.KubeContext != "" {
+	if ld.overrideKubeContext != "" {
 		if st.HelmDefaults.KubeContext != "" {
 			return nil, errors.New("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.")
 		}
-		st.HelmDefaults.KubeContext = ld.KubeContext
+		st.HelmDefaults.KubeContext = ld.overrideKubeContext
 	}
 
 	if ld.namespace != "" {
-		if st.Namespace != "" {
+		if st.OverrideNamespace != "" {
 			return nil, errors.New("err: Cannot use option --namespace and set attribute namespace.")
 		}
-		st.Namespace = ld.namespace
+		st.OverrideNamespace = ld.namespace
 	}
 
 	return st, nil
@@ -124,11 +130,24 @@ func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *e
 		)
 	}
 
-	return self, err
+	if err != nil {
+		return nil, err
+	}
+
+	for i, h := range self.Helmfiles {
+		if h.Path == f {
+			return nil, fmt.Errorf("%s contains a recursion into the same sub-helmfile at helmfiles[%d]", f, i)
+		}
+		if h.Path == "." {
+			return nil, fmt.Errorf("%s contains a recursion into the the directory containing this helmfile at helmfiles[%d]", f, i)
+		}
+	}
+
+	return self, nil
 }
 
 func (a *desiredStateLoader) underlying() *state.StateCreator {
-	c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.helm)
+	c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.valsRuntime, a.getHelm, a.overrideHelmBinary)
 	c.LoadFile = a.loadFile
 	return c
 }
@@ -154,7 +173,9 @@ func (a *desiredStateLoader) load(yaml []byte, baseDir, file string, evaluateBas
 }
 
 func (ld *desiredStateLoader) renderAndLoad(env, overrodeEnv *environment.Environment, baseDir, filename string, content []byte, evaluateBases bool) (*state.HelmState, error) {
-	parts := bytes.Split(content, []byte("\n---\n"))
+	// Allows part-splitting to work with CLRF-ed content
+	normalizedContent := bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n"))
+	parts := bytes.Split(normalizedContent, []byte("\n---\n"))
 
 	var finalState *state.HelmState
 
@@ -188,6 +209,12 @@ func (ld *desiredStateLoader) renderAndLoad(env, overrodeEnv *environment.Enviro
 			return nil, err
 		}
 
+		for i, r := range currentState.Releases {
+			if r.Chart == "" {
+				return nil, fmt.Errorf("error during %s parsing: encountered empty chart while reading release %q at index %d", id, r.Name, i)
+			}
+		}
+
 		if finalState == nil {
 			finalState = currentState
 		} else {

pkg/app/destroy_test.go

@@ -0,0 +1,453 @@
+package app
+
+import (
+	"bufio"
+	"bytes"
+	"github.com/roboll/helmfile/pkg/exectest"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/testhelper"
+	"github.com/variantdev/vals"
+	"go.uber.org/zap"
+	"io"
+	"path/filepath"
+	"sync"
+	"testing"
+)
+
+type destroyConfig struct {
+	args        string
+	concurrency int
+	interactive bool
+	logger      *zap.SugaredLogger
+}
+
+func (d destroyConfig) Args() string {
+	return d.args
+}
+
+func (d destroyConfig) Interactive() bool {
+	return d.interactive
+}
+
+func (d destroyConfig) Logger() *zap.SugaredLogger {
+	return d.logger
+}
+
+func (d destroyConfig) Concurrency() int {
+	return d.concurrency
+}
+
+func TestDestroy(t *testing.T) {
+	testcases := []struct {
+		name        string
+		loc         string
+		ns          string
+		concurrency int
+		error       string
+		files       map[string]string
+		selectors   []string
+		lists       map[exectest.ListKey]string
+		diffs       map[exectest.DiffKey]error
+		upgraded    []exectest.Release
+		deleted     []exectest.Release
+		log         string
+	}{
+		//
+		// complex test cases for smoke testing
+		//
+		{
+			name: "smoke",
+			loc:  location(),
+			files: map[string]string{
+				"/path/to/helmfile.yaml": `
+releases:
+- name: database
+  chart: charts/mysql
+  needs:
+  - logging
+- name: frontend-v1
+  chart: charts/frontend
+  installed: false
+  needs:
+  - servicemesh
+  - logging
+  - backend-v1
+- name: frontend-v2
+  chart: charts/frontend
+  needs:
+  - servicemesh
+  - logging
+  - backend-v2
+- name: frontend-v3
+  chart: charts/frontend
+  needs:
+  - servicemesh
+  - logging
+  - backend-v2
+- name: backend-v1
+  chart: charts/backend
+  installed: false
+  needs:
+  - servicemesh
+  - logging
+  - database
+  - anotherbackend
+- name: backend-v2
+  chart: charts/backend
+  needs:
+  - servicemesh
+  - logging
+  - database
+  - anotherbackend
+- name: anotherbackend
+  chart: charts/anotherbackend
+  needs:
+  - servicemesh
+  - logging
+  - database
+- name: servicemesh
+  chart: charts/istio
+  needs:
+  - logging
+- name: logging
+  chart: charts/fluent-bit
+- name: front-proxy
+  chart: stable/envoy
+`,
+			},
+			diffs: map[exectest.DiffKey]error{},
+			lists: map[exectest.ListKey]string{
+				exectest.ListKey{Filter: "^frontend-v1$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+`,
+				exectest.ListKey{Filter: "^frontend-v2$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+frontend-v2 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	frontend-3.1.0	3.1.0      	default
+`,
+				exectest.ListKey{Filter: "^frontend-v3$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+frontend-v3 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	frontend-3.1.0	3.1.0      	default
+`,
+				exectest.ListKey{Filter: "^backend-v1$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+`,
+				exectest.ListKey{Filter: "^backend-v2$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+backend-v2 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	backend-3.1.0	3.1.0      	default
+`,
+				exectest.ListKey{Filter: "^logging$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+logging	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	fluent-bit-3.1.0	3.1.0      	default
+`,
+				exectest.ListKey{Filter: "^front-proxy$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+front-proxy 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	envoy-3.1.0	3.1.0      	default
+`,
+				exectest.ListKey{Filter: "^servicemesh$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+servicemesh 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	istio-3.1.0	3.1.0      	default
+`,
+				exectest.ListKey{Filter: "^database$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+database 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	mysql-3.1.0	3.1.0      	default
+`,
+				exectest.ListKey{Filter: "^anotherbackend$", Flags: "--kube-contextdefault--deployed--failed--pending"}: `NAME	REVISION	UPDATED                 	STATUS  	CHART        	APP VERSION	NAMESPACE
+anotherbackend 	4       	Fri Nov  1 08:40:07 2019	DEPLOYED	anotherbackend-3.1.0	3.1.0      	default
+`,
+			},
+			// Disable concurrency to avoid in-deterministic result
+			concurrency: 1,
+			upgraded:    []exectest.Release{},
+			deleted: []exectest.Release{
+				{Name: "frontend-v3", Flags: []string{}},
+				{Name: "frontend-v2", Flags: []string{}},
+				{Name: "frontend-v1", Flags: []string{}},
+				{Name: "backend-v2", Flags: []string{}},
+				{Name: "backend-v1", Flags: []string{}},
+				{Name: "anotherbackend", Flags: []string{}},
+				{Name: "database", Flags: []string{}},
+				{Name: "servicemesh", Flags: []string{}},
+				{Name: "front-proxy", Flags: []string{}},
+				{Name: "logging", Flags: []string{}},
+			},
+			log: `processing file "helmfile.yaml" in directory "."
+first-pass rendering starting for "helmfile.yaml.part.0": inherited=&{default map[] map[]}, overrode=<nil>
+first-pass uses: &{default map[] map[]}
+first-pass rendering output of "helmfile.yaml.part.0":
+ 0: 
+ 1: releases:
+ 2: - name: database
+ 3:   chart: charts/mysql
+ 4:   needs:
+ 5:   - logging
+ 6: - name: frontend-v1
+ 7:   chart: charts/frontend
+ 8:   installed: false
+ 9:   needs:
+10:   - servicemesh
+11:   - logging
+12:   - backend-v1
+13: - name: frontend-v2
+14:   chart: charts/frontend
+15:   needs:
+16:   - servicemesh
+17:   - logging
+18:   - backend-v2
+19: - name: frontend-v3
+20:   chart: charts/frontend
+21:   needs:
+22:   - servicemesh
+23:   - logging
+24:   - backend-v2
+25: - name: backend-v1
+26:   chart: charts/backend
+27:   installed: false
+28:   needs:
+29:   - servicemesh
+30:   - logging
+31:   - database
+32:   - anotherbackend
+33: - name: backend-v2
+34:   chart: charts/backend
+35:   needs:
+36:   - servicemesh
+37:   - logging
+38:   - database
+39:   - anotherbackend
+40: - name: anotherbackend
+41:   chart: charts/anotherbackend
+42:   needs:
+43:   - servicemesh
+44:   - logging
+45:   - database
+46: - name: servicemesh
+47:   chart: charts/istio
+48:   needs:
+49:   - logging
+50: - name: logging
+51:   chart: charts/fluent-bit
+52: - name: front-proxy
+53:   chart: stable/envoy
+54: 
+
+first-pass produced: &{default map[] map[]}
+first-pass rendering result of "helmfile.yaml.part.0": {default map[] map[]}
+vals:
+map[]
+defaultVals:[]
+second-pass rendering result of "helmfile.yaml.part.0":
+ 0: 
+ 1: releases:
+ 2: - name: database
+ 3:   chart: charts/mysql
+ 4:   needs:
+ 5:   - logging
+ 6: - name: frontend-v1
+ 7:   chart: charts/frontend
+ 8:   installed: false
+ 9:   needs:
+10:   - servicemesh
+11:   - logging
+12:   - backend-v1
+13: - name: frontend-v2
+14:   chart: charts/frontend
+15:   needs:
+16:   - servicemesh
+17:   - logging
+18:   - backend-v2
+19: - name: frontend-v3
+20:   chart: charts/frontend
+21:   needs:
+22:   - servicemesh
+23:   - logging
+24:   - backend-v2
+25: - name: backend-v1
+26:   chart: charts/backend
+27:   installed: false
+28:   needs:
+29:   - servicemesh
+30:   - logging
+31:   - database
+32:   - anotherbackend
+33: - name: backend-v2
+34:   chart: charts/backend
+35:   needs:
+36:   - servicemesh
+37:   - logging
+38:   - database
+39:   - anotherbackend
+40: - name: anotherbackend
+41:   chart: charts/anotherbackend
+42:   needs:
+43:   - servicemesh
+44:   - logging
+45:   - database
+46: - name: servicemesh
+47:   chart: charts/istio
+48:   needs:
+49:   - logging
+50: - name: logging
+51:   chart: charts/fluent-bit
+52: - name: front-proxy
+53:   chart: stable/envoy
+54: 
+
+merged environment: &{default map[] map[]}
+10 release(s) found in helmfile.yaml
+
+processing 5 groups of releases in this order:
+GROUP RELEASES
+1     frontend-v3, frontend-v2, frontend-v1
+2     backend-v2, backend-v1
+3     anotherbackend
+4     database, servicemesh
+5     front-proxy, logging
+
+processing releases in group 1/5: frontend-v3, frontend-v2, frontend-v1
+release "frontend-v3" processed
+release "frontend-v2" processed
+release "frontend-v1" processed
+processing releases in group 2/5: backend-v2, backend-v1
+release "backend-v2" processed
+release "backend-v1" processed
+processing releases in group 3/5: anotherbackend
+release "anotherbackend" processed
+processing releases in group 4/5: database, servicemesh
+release "database" processed
+release "servicemesh" processed
+processing releases in group 5/5: front-proxy, logging
+release "front-proxy" processed
+release "logging" processed
+
+DELETED RELEASES:
+NAME
+frontend-v3
+frontend-v2
+frontend-v1
+backend-v2
+backend-v1
+anotherbackend
+database
+servicemesh
+front-proxy
+logging
+`,
+		},
+	}
+
+	for i := range testcases {
+		tc := testcases[i]
+		t.Run(tc.name, func(t *testing.T) {
+			wantUpgrades := tc.upgraded
+			wantDeletes := tc.deleted
+
+			var helm = &exectest.Helm{
+				FailOnUnexpectedList: true,
+				FailOnUnexpectedDiff: true,
+				Lists:                tc.lists,
+				Diffs:                tc.diffs,
+				DiffMutex:            &sync.Mutex{},
+				ChartsMutex:          &sync.Mutex{},
+				ReleasesMutex:        &sync.Mutex{},
+			}
+
+			bs := &bytes.Buffer{}
+
+			func() {
+				logReader, logWriter := io.Pipe()
+
+				logFlushed := &sync.WaitGroup{}
+				// Ensure all the log is consumed into `bs` by calling `logWriter.Close()` followed by `logFlushed.Wait()`
+				logFlushed.Add(1)
+				go func() {
+					scanner := bufio.NewScanner(logReader)
+					for scanner.Scan() {
+						bs.Write(scanner.Bytes())
+						bs.WriteString("\n")
+					}
+					logFlushed.Done()
+				}()
+
+				defer func() {
+					// This is here to avoid data-trace on bytes buffer `bs` to capture logs
+					if err := logWriter.Close(); err != nil {
+						panic(err)
+					}
+					logFlushed.Wait()
+				}()
+
+				logger := helmexec.NewLogger(logWriter, "debug")
+
+				valsRuntime, err := vals.New(vals.Options{CacheSize: 32})
+				if err != nil {
+					t.Errorf("unexpected error creating vals runtime: %v", err)
+				}
+
+				app := appWithFs(&App{
+					OverrideHelmBinary:  DefaultHelmBinary,
+					glob:                filepath.Glob,
+					abs:                 filepath.Abs,
+					OverrideKubeContext: "default",
+					Env:                 "default",
+					Logger:              logger,
+					helms: map[helmKey]helmexec.Interface{
+						createHelmKey("helm", "default"): helm,
+					},
+					valsRuntime: valsRuntime,
+				}, tc.files)
+
+				if tc.ns != "" {
+					app.Namespace = tc.ns
+				}
+
+				if tc.selectors != nil {
+					app.Selectors = tc.selectors
+				}
+
+				destroyErr := app.Destroy(destroyConfig{
+					// if we check log output, concurrency must be 1. otherwise the test becomes non-deterministic.
+					concurrency: tc.concurrency,
+					logger:      logger,
+				})
+
+				if tc.error == "" && destroyErr != nil {
+					t.Fatalf("unexpected error for data defined at %s: %v", tc.loc, destroyErr)
+				} else if tc.error != "" && destroyErr == nil {
+					t.Fatalf("expected error did not occur for data defined at %s", tc.loc)
+				} else if tc.error != "" && destroyErr != nil && tc.error != destroyErr.Error() {
+					t.Fatalf("invalid error: expected %q, got %q", tc.error, destroyErr.Error())
+				}
+
+				if len(wantUpgrades) > len(helm.Releases) {
+					t.Fatalf("insufficient number of upgrades: got %d, want %d", len(helm.Releases), len(wantUpgrades))
+				}
+
+				for relIdx := range wantUpgrades {
+					if wantUpgrades[relIdx].Name != helm.Releases[relIdx].Name {
+						t.Errorf("releases[%d].name: got %q, want %q", relIdx, helm.Releases[relIdx].Name, wantUpgrades[relIdx].Name)
+					}
+					for flagIdx := range wantUpgrades[relIdx].Flags {
+						if wantUpgrades[relIdx].Flags[flagIdx] != helm.Releases[relIdx].Flags[flagIdx] {
+							t.Errorf("releaes[%d].flags[%d]: got %v, want %v", relIdx, flagIdx, helm.Releases[relIdx].Flags[flagIdx], wantUpgrades[relIdx].Flags[flagIdx])
+						}
+					}
+				}
+
+				if len(wantDeletes) > len(helm.Deleted) {
+					t.Fatalf("insufficient number of deletes: got %d, want %d", len(helm.Deleted), len(wantDeletes))
+				}
+
+				for relIdx := range wantDeletes {
+					if wantDeletes[relIdx].Name != helm.Deleted[relIdx].Name {
+						t.Errorf("releases[%d].name: got %q, want %q", relIdx, helm.Deleted[relIdx].Name, wantDeletes[relIdx].Name)
+					}
+					for flagIdx := range wantDeletes[relIdx].Flags {
+						if wantDeletes[relIdx].Flags[flagIdx] != helm.Deleted[relIdx].Flags[flagIdx] {
+							t.Errorf("releaes[%d].flags[%d]: got %v, want %v", relIdx, flagIdx, helm.Deleted[relIdx].Flags[flagIdx], wantDeletes[relIdx].Flags[flagIdx])
+						}
+					}
+				}
+			}()
+
+			if tc.log != "" {
+				actual := bs.String()
+
+				diff, exists := testhelper.Diff(tc.log, actual, 3)
+				if exists {
+					t.Errorf("unexpected log for data defined %s:\nDIFF\n%s\nEOD", tc.loc, diff)
+				}
+			}
+		})
+	}
+}

pkg/app/formatters.go

@@ -0,0 +1,33 @@
+package app
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/gosuri/uitable"
+)
+
+func FormatAsTable(releases []*HelmRelease) error {
+	table := uitable.New()
+	table.AddRow("NAME", "NAMESPACE", "ENABLED", "LABELS")
+
+	for _, r := range releases {
+		table.AddRow(r.Name, r.Namespace, fmt.Sprintf("%t", r.Enabled), r.Labels)
+	}
+
+	fmt.Println(table.String())
+
+	return nil
+}
+
+func FormatAsJson(releases []*HelmRelease) error {
+	output, err := json.Marshal(releases)
+
+	if err != nil {
+		return fmt.Errorf("error generating json: %v", err)
+	}
+
+	fmt.Println(string(output))
+
+	return nil
+}

pkg/app/load_opts.go

@@ -9,8 +9,12 @@ type LoadOpts struct {
 	Selectors   []string
 	Environment state.SubhelmfileEnvironmentSpec
 
+	RetainValuesFiles bool
+
 	// CalleePath is the absolute path to the file being loaded
 	CalleePath string
+
+	Reverse bool
 }
 
 func (o LoadOpts) DeepCopy() LoadOpts {

pkg/app/mocks_test.go

@@ -0,0 +1,103 @@
+package app
+
+import "github.com/roboll/helmfile/pkg/helmexec"
+
+type noCallHelmExec struct {
+}
+
+type versionOnlyHelmExec struct {
+	*noCallHelmExec
+	isHelm3 bool
+}
+
+func (helm *versionOnlyHelmExec) IsHelm3() bool {
+	return helm.isHelm3
+}
+
+func (helm *noCallHelmExec) doPanic() {
+	panic("unexpected call to helm")
+}
+
+func (helm *noCallHelmExec) TemplateRelease(name, chart string, flags ...string) error {
+	helm.doPanic()
+	return nil
+}
+
+func (helm *noCallHelmExec) UpdateDeps(chart string) error {
+	helm.doPanic()
+	return nil
+}
+
+func (helm *noCallHelmExec) BuildDeps(name, chart string) error {
+	helm.doPanic()
+	return nil
+}
+
+func (helm *noCallHelmExec) SetExtraArgs(args ...string) {
+	helm.doPanic()
+	return
+}
+func (helm *noCallHelmExec) SetHelmBinary(bin string) {
+	helm.doPanic()
+	return
+}
+func (helm *noCallHelmExec) AddRepo(name, repository, cafile, certfile, keyfile, username, password string) error {
+	helm.doPanic()
+	return nil
+}
+func (helm *noCallHelmExec) UpdateRepo() error {
+	helm.doPanic()
+	return nil
+}
+func (helm *noCallHelmExec) SyncRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
+	helm.doPanic()
+	return nil
+}
+func (helm *noCallHelmExec) DiffRelease(context helmexec.HelmContext, name, chart string, suppressDiff bool, flags ...string) error {
+	helm.doPanic()
+	return nil
+}
+func (helm *noCallHelmExec) ReleaseStatus(context helmexec.HelmContext, release string, flags ...string) error {
+	helm.doPanic()
+	return nil
+}
+func (helm *noCallHelmExec) DeleteRelease(context helmexec.HelmContext, name string, flags ...string) error {
+	helm.doPanic()
+	return nil
+}
+
+func (helm *noCallHelmExec) List(context helmexec.HelmContext, filter string, flags ...string) (string, error) {
+	helm.doPanic()
+	return "", nil
+}
+
+func (helm *noCallHelmExec) DecryptSecret(context helmexec.HelmContext, name string, flags ...string) (string, error) {
+	helm.doPanic()
+	return "", nil
+}
+func (helm *noCallHelmExec) TestRelease(context helmexec.HelmContext, name string, flags ...string) error {
+	helm.doPanic()
+	return nil
+}
+func (helm *noCallHelmExec) Fetch(chart string, flags ...string) error {
+	helm.doPanic()
+	return nil
+}
+func (helm *noCallHelmExec) Lint(name, chart string, flags ...string) error {
+	helm.doPanic()
+	return nil
+}
+func (helm *noCallHelmExec) IsHelm3() bool {
+	helm.doPanic()
+	return false
+}
+
+func (helm *noCallHelmExec) GetVersion() helmexec.Version {
+	helm.doPanic()
+	return helmexec.Version{}
+}
+
+func (helm *noCallHelmExec) IsVersionAtLeast(major int, minor int) bool {
+	helm.doPanic()
+	return false
+}

pkg/app/run.go

@@ -2,11 +2,13 @@ package app
 
 import (
 	"fmt"
-	"strings"
-
 	"github.com/roboll/helmfile/pkg/argparser"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/state"
+	"io/ioutil"
+	"os"
+	"sort"
+	"strings"
 )
 
 type Run struct {
@@ -14,6 +16,8 @@ type Run struct {
 	helm  helmexec.Interface
 	ctx   Context
 
+	ReleaseToChart map[string]string
+
 	Ask func(string) bool
 }
 
@@ -28,6 +32,37 @@ func (r *Run) askForConfirmation(msg string) bool {
 	return AskForConfirmation(msg)
 }
 
+func (r *Run) withPreparedCharts(forceDownload bool, f func()) error {
+	if r.ReleaseToChart != nil {
+		panic("Run.PrepareCharts can be called only once")
+	}
+
+	// Create tmp directory and bail immediately if it fails
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(dir)
+
+	releaseToChart, errs := state.PrepareCharts(r.helm, r.state, dir, 2, "template", forceDownload)
+
+	if len(errs) > 0 {
+		return fmt.Errorf("%v", errs)
+	}
+
+	for i := range r.state.Releases {
+		rel := &r.state.Releases[i]
+
+		rel.Chart = releaseToChart[rel.Name]
+	}
+
+	r.ReleaseToChart = releaseToChart
+
+	f()
+
+	return nil
+}
+
 func (r *Run) Deps(c DepsConfigProvider) []error {
 	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
 
@@ -64,173 +99,36 @@ func (r *Run) Status(c StatusesConfigProvider) []error {
 	return r.state.ReleaseStatuses(r.helm, workers)
 }
 
-func (r *Run) Delete(c DeleteConfigProvider) []error {
-	affectedReleases := state.AffectedReleases{}
-	purge := c.Purge()
-
-	errs := []error{}
-
-	names := make([]string, len(r.state.Releases))
-	for i, r := range r.state.Releases {
-		names[i] = fmt.Sprintf("  %s (%s)", r.Name, r.Chart)
-	}
-
-	msg := fmt.Sprintf(`Affected releases are:
-%s
-
-Do you really want to delete?
-  Helmfile will delete all your releases, as shown above.
-
-`, strings.Join(names, "\n"))
-	interactive := c.Interactive()
-	if !interactive || interactive && r.askForConfirmation(msg) {
-		r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
-
-		errs = r.state.DeleteReleases(&affectedReleases, r.helm, c.Concurrency(), purge)
-	}
-	affectedReleases.DisplayAffectedReleases(c.Logger())
-	return errs
-}
-
-func (r *Run) Destroy(c DestroyConfigProvider) []error {
-	errs := []error{}
-	affectedReleases := state.AffectedReleases{}
-
-	names := make([]string, len(r.state.Releases))
-	for i, r := range r.state.Releases {
-		names[i] = fmt.Sprintf("  %s (%s)", r.Name, r.Chart)
-	}
-
-	msg := fmt.Sprintf(`Affected releases are:
-%s
-
-Do you really want to delete?
-  Helmfile will delete all your releases, as shown above.
-
-`, strings.Join(names, "\n"))
-	interactive := c.Interactive()
-	if !interactive || interactive && r.askForConfirmation(msg) {
-		r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
-
-		errs = r.state.DeleteReleases(&affectedReleases, r.helm, c.Concurrency(), true)
-	}
-	affectedReleases.DisplayAffectedReleases(c.Logger())
-	return errs
-}
-
-func (r *Run) Apply(c ApplyConfigProvider) []error {
+func (r *Run) Diff(c DiffConfigProvider) (*string, bool, bool, []error) {
 	st := r.state
 	helm := r.helm
 	ctx := r.ctx
 
-	affectedReleases := state.AffectedReleases{}
-	if !c.SkipDeps() {
-		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
-			return errs
-		}
-		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
-			return errs
-		}
-	}
-	if errs := st.PrepareReleases(helm, "apply"); errs != nil && len(errs) > 0 {
-		return errs
-	}
+	allReleases := st.GetReleasesWithOverrides()
 
-	// helm must be 2.11+ and helm-diff should be provided `--detailed-exitcode` in order for `helmfile apply` to work properly
-	detailedExitCode := true
-
-	diffOpts := &state.DiffOpts{
-		NoColor: c.NoColor(),
-		Context: c.Context(),
-		Set:     c.Set(),
-	}
-
-	releases, errs := st.DiffReleases(helm, c.Values(), c.Concurrency(), detailedExitCode, c.SuppressSecrets(), false, diffOpts)
-
-	releasesToBeDeleted, err := st.DetectReleasesToBeDeleted(helm)
+	toDiff, err := st.GetSelectedReleasesWithOverrides()
 	if err != nil {
-		errs = append(errs, err)
+		return nil, false, false, []error{err}
 	}
 
-	fatalErrs := []error{}
-
-	noError := true
-	for _, e := range errs {
-		switch err := e.(type) {
-		case *state.ReleaseError:
-			if err.Code != 2 {
-				noError = false
-				fatalErrs = append(fatalErrs, e)
-			}
-		default:
-			noError = false
-			fatalErrs = append(fatalErrs, e)
-		}
+	if len(toDiff) == 0 {
+		return nil, false, false, nil
 	}
 
-	// sync only when there are changes
-	if noError {
-		if len(releases) == 0 && len(releasesToBeDeleted) == 0 {
-			// TODO better way to get the logger
-			logger := c.Logger()
-			logger.Infof("")
-			logger.Infof("No affected releases")
-		} else {
-			names := []string{}
-			for _, r := range releases {
-				names = append(names, fmt.Sprintf("  %s (%s) UPDATED", r.Name, r.Chart))
-			}
-			for _, r := range releasesToBeDeleted {
-				names = append(names, fmt.Sprintf("  %s (%s) DELETED", r.Name, r.Chart))
-			}
-
-			msg := fmt.Sprintf(`Affected releases are:
-%s
-
-Do you really want to apply?
-  Helmfile will apply all your changes, as shown above.
-
-`, strings.Join(names, "\n"))
-			interactive := c.Interactive()
-			if !interactive || interactive && r.askForConfirmation(msg) {
-				rs := []state.ReleaseSpec{}
-				for _, r := range releases {
-					rs = append(rs, *r)
-				}
-				for _, r := range releasesToBeDeleted {
-					rs = append(rs, *r)
-				}
-
-				r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
-
-				st.Releases = rs
-				syncOpts := &state.SyncOpts{
-					Set: c.Set(),
-				}
-				return st.SyncReleases(&affectedReleases, helm, c.Values(), c.Concurrency(), syncOpts)
-			}
-		}
-	}
-
-	affectedReleases.DisplayAffectedReleases(c.Logger())
-	return fatalErrs
-}
-
-func (r *Run) Diff(c DiffConfigProvider) []error {
-	st := r.state
-	helm := r.helm
-	ctx := r.ctx
+	// Do build deps and prepare only on selected releases so that we won't waste time
+	// on running various helm commands on unnecessary releases
+	st.Releases = toDiff
 
 	if !c.SkipDeps() {
 		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
-			return errs
+			return nil, false, false, errs
 		}
 		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
-			return errs
+			return nil, false, false, errs
 		}
 	}
 	if errs := st.PrepareReleases(helm, "diff"); errs != nil && len(errs) > 0 {
-		return errs
+		return nil, false, false, errs
 	}
 
 	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
@@ -240,60 +138,28 @@ func (r *Run) Diff(c DiffConfigProvider) []error {
 		NoColor: c.NoColor(),
 		Set:     c.Set(),
 	}
-	_, errs := st.DiffReleases(helm, c.Values(), c.Concurrency(), c.DetailedExitcode(), c.SuppressSecrets(), true, opts)
-	return errs
-}
 
-func (r *Run) Sync(c SyncConfigProvider) []error {
-	st := r.state
-	helm := r.helm
-	ctx := r.ctx
+	// Validate all releases for missing `needs` targets
+	st.Releases = allReleases
 
-	affectedReleases := state.AffectedReleases{}
-	if !c.SkipDeps() {
-		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
-			return errs
-		}
-		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
-			return errs
-		}
-	}
-	if errs := st.PrepareReleases(helm, "sync"); errs != nil && len(errs) > 0 {
-		return errs
+	if _, err := st.PlanReleases(false); err != nil {
+		return nil, false, false, []error{err}
 	}
 
-	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
+	// Diff only targeted releases
 
-	opts := &state.SyncOpts{
-		Set: c.Set(),
-	}
-	errs := st.SyncReleases(&affectedReleases, helm, c.Values(), c.Concurrency(), opts)
-	affectedReleases.DisplayAffectedReleases(c.Logger())
-	return errs
-}
+	st.Releases = toDiff
 
-func (r *Run) Template(c TemplateConfigProvider) []error {
-	st := r.state
-	helm := r.helm
-	ctx := r.ctx
-
-	if !c.SkipDeps() {
-		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
-			return errs
-		}
-		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
-			return errs
-		}
-	}
-	if errs := st.PrepareReleases(helm, "template"); errs != nil && len(errs) > 0 {
-		return errs
+	filtered := &Run{
+		state: st,
+		helm:  r.helm,
+		ctx:   r.ctx,
+		Ask:   r.Ask,
 	}
 
-	args := argparser.GetArgs(c.Args(), st)
-	opts := &state.TemplateOpts{
-		Set: c.Set(),
-	}
-	return st.TemplateReleases(helm, c.OutputDir(), c.Values(), args, c.Concurrency(), opts)
+	infoMsg, updated, deleted, errs := filtered.diff(true, c.DetailedExitcode(), c, opts)
+
+	return infoMsg, true, len(deleted) > 0 || len(updated) > 0, errs
 }
 
 func (r *Run) Test(c TestConfigProvider) []error {
@@ -330,3 +196,83 @@ func (r *Run) Lint(c LintConfigProvider) []error {
 	}
 	return st.LintReleases(helm, values, args, workers, opts)
 }
+
+func (run *Run) diff(triggerCleanupEvent bool, detailedExitCode bool, c DiffConfigProvider, diffOpts *state.DiffOpts) (*string, map[string]state.ReleaseSpec, map[string]state.ReleaseSpec, []error) {
+	st := run.state
+	helm := run.helm
+
+	var changedReleases []state.ReleaseSpec
+	var deletingReleases []state.ReleaseSpec
+	var planningErrs []error
+
+	// TODO Better way to detect diff on only filtered releases
+	{
+		changedReleases, planningErrs = st.DiffReleases(helm, c.Values(), c.Concurrency(), detailedExitCode, c.IncludeTests(), c.SuppressSecrets(), c.SuppressDiff(), triggerCleanupEvent, diffOpts)
+
+		var err error
+		deletingReleases, err = st.DetectReleasesToBeDeletedForSync(helm, st.Releases)
+		if err != nil {
+			planningErrs = append(planningErrs, err)
+		}
+	}
+
+	fatalErrs := []error{}
+
+	for _, e := range planningErrs {
+		switch err := e.(type) {
+		case *state.ReleaseError:
+			if err.Code != 2 {
+				fatalErrs = append(fatalErrs, e)
+			}
+		default:
+			fatalErrs = append(fatalErrs, e)
+		}
+	}
+
+	if len(fatalErrs) > 0 {
+		return nil, nil, nil, fatalErrs
+	}
+
+	releasesToBeDeleted := map[string]state.ReleaseSpec{}
+	for _, r := range deletingReleases {
+		id := state.ReleaseToID(&r)
+		releasesToBeDeleted[id] = r
+	}
+
+	releasesToBeUpdated := map[string]state.ReleaseSpec{}
+	for _, r := range changedReleases {
+		id := state.ReleaseToID(&r)
+
+		// If `helm-diff` detected changes but it is not being `helm delete`ed, we should run `helm upgrade`
+		if _, ok := releasesToBeDeleted[id]; !ok {
+			releasesToBeUpdated[id] = r
+		}
+	}
+
+	// sync only when there are changes
+	if len(releasesToBeUpdated) == 0 && len(releasesToBeDeleted) == 0 {
+		var msg *string
+		if c.DetailedExitcode() {
+			// TODO better way to get the logger
+			m := "No affected releases"
+			msg = &m
+		}
+		return msg, nil, nil, nil
+	}
+
+	names := []string{}
+	for _, r := range releasesToBeUpdated {
+		names = append(names, fmt.Sprintf("  %s (%s) UPDATED", r.Name, r.Chart))
+	}
+	for _, r := range releasesToBeDeleted {
+		names = append(names, fmt.Sprintf("  %s (%s) DELETED", r.Name, r.Chart))
+	}
+	// Make the output deterministic for testing purpose
+	sort.Strings(names)
+
+	infoMsg := fmt.Sprintf(`Affected releases are:
+%s
+`, strings.Join(names, "\n"))
+
+	return &infoMsg, releasesToBeUpdated, releasesToBeDeleted, nil
+}

pkg/app/two_pass_renderer.go

@@ -31,12 +31,15 @@ func (r *desiredStateLoader) renderPrestate(firstPassEnv *environment.Environmen
 	yamlBuf, err := firstPassRenderer.RenderTemplateContentToBuffer(content)
 	if err != nil && r.logger != nil {
 		r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", filename, prependLineNumbers(string(content)))
+		r.logger.Debugf("template syntax error: %v", err)
 		if yamlBuf == nil { // we have a template syntax error, let the second parse report
-			r.logger.Debugf("template syntax error: %v", err)
 			return firstPassEnv, nil
 		}
 	}
 	yamlData := yamlBuf.String()
+	if r.logger != nil {
+		r.logger.Debugf("first-pass rendering output of \"%s\":\n%s", filename, prependLineNumbers(yamlData))
+	}
 
 	// Work-around for https://github.com/golang/go/issues/24963
 	sanitized := strings.ReplaceAll(yamlData, "<no value>", "")
@@ -53,7 +56,7 @@ func (r *desiredStateLoader) renderPrestate(firstPassEnv *environment.Environmen
 	if err != nil && r.logger != nil {
 		switch err.(type) {
 		case *state.StateLoadError:
-			r.logger.Infof("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err)
+			r.logger.Debugf("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err)
 		}
 		r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String()))
 	}
@@ -113,15 +116,13 @@ func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *en
 		r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, *finalEnv)
 	}
 
-	vals := map[string]interface{}{}
+	vals, err := finalEnv.GetMergedValues()
+	if err != nil {
+		return nil, err
+	}
+
 	if prestate != nil {
 		prestate.Env = *finalEnv
-		vals, err = prestate.Values()
-		if err != nil {
-			return nil, err
-		}
-	}
-	if prestate != nil {
 		r.logger.Debugf("vals:\n%v\ndefaultVals:%v", vals, prestate.DefaultValues)
 	}
 

pkg/app/version/version.go

@@ -0,0 +1,3 @@
+package version
+
+var Version string

pkg/environment/environment.go

@@ -58,9 +58,28 @@ func (e *Environment) Merge(other *Environment) (*Environment, error) {
 	}
 	copy := e.DeepCopy()
 	if other != nil {
-		if err := mergo.Merge(&copy, other, mergo.WithOverride); err != nil {
+		if err := mergo.Merge(&copy, other, mergo.WithOverride, mergo.WithOverwriteWithEmptyValue); err != nil {
 			return nil, err
 		}
 	}
 	return &copy, nil
 }
+
+func (e *Environment) GetMergedValues() (map[string]interface{}, error) {
+	vals := map[string]interface{}{}
+
+	if err := mergo.Merge(&vals, e.Defaults, mergo.WithOverride, mergo.WithOverwriteWithEmptyValue); err != nil {
+		return nil, err
+	}
+
+	if err := mergo.Merge(&vals, e.Values, mergo.WithOverride, mergo.WithOverwriteWithEmptyValue); err != nil {
+		return nil, err
+	}
+
+	vals, err := maputil.CastKeysToStrings(vals)
+	if err != nil {
+		return nil, err
+	}
+
+	return vals, nil
+}

pkg/environment/environment_test.go

@@ -0,0 +1,98 @@
+package environment
+
+import (
+	"github.com/google/go-cmp/cmp"
+	"testing"
+)
+
+// See https://github.com/roboll/helmfile/issues/1150
+func TestMerge_OverwriteNilValue_Issue1150(t *testing.T) {
+	dst := &Environment{
+		Name: "dst",
+		Values: map[string]interface{}{
+			"components": map[string]interface{}{
+				"etcd-operator": nil,
+			},
+		},
+		Defaults: nil,
+	}
+
+	src := &Environment{
+		Name: "src",
+		Values: map[string]interface{}{
+			"components": map[string]interface{}{
+				"etcd-operator": map[string]interface{}{
+					"version": "0.10.3",
+				},
+			},
+		},
+		Defaults: nil,
+	}
+
+	merged, err := dst.Merge(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	actual := merged.Values
+
+	expected := map[string]interface{}{
+		"components": map[string]interface{}{
+			"etcd-operator": map[string]interface{}{
+				"version": "0.10.3",
+			},
+		},
+	}
+
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf(diff)
+	}
+}
+
+// See https://github.com/roboll/helmfile/issues/1154
+func TestMerge_OverwriteWithNilValue_Issue1154(t *testing.T) {
+	dst := &Environment{
+		Name: "dst",
+		Values: map[string]interface{}{
+			"components": map[string]interface{}{
+				"etcd-operator": map[string]interface{}{
+					"version": "0.10.0",
+				},
+			},
+		},
+		Defaults: nil,
+	}
+
+	src := &Environment{
+		Name: "src",
+		Values: map[string]interface{}{
+			"components": map[string]interface{}{
+				"etcd-operator": map[string]interface{}{
+					"version": "0.10.3",
+				},
+				"prometheus": nil,
+			},
+		},
+		Defaults: nil,
+	}
+
+	merged, err := dst.Merge(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	actual := merged.Values
+
+	expected := map[string]interface{}{
+		"components": map[string]interface{}{
+			"etcd-operator": map[string]interface{}{
+				"version": "0.10.3",
+			},
+			"prometheus": nil,
+		},
+	}
+
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf(diff)
+	}
+}

pkg/event/bus.go

@@ -20,7 +20,8 @@ type Hook struct {
 }
 
 type event struct {
-	Name string
+	Name  string
+	Error error
 }
 
 type Bus struct {
@@ -37,7 +38,7 @@ type Bus struct {
 	Logger   *zap.SugaredLogger
 }
 
-func (bus *Bus) Trigger(evt string, context map[string]interface{}) (bool, error) {
+func (bus *Bus) Trigger(evt string, evtErr error, context map[string]interface{}) (bool, error) {
 	if bus.Runner == nil {
 		bus.Runner = helmexec.ShellRunner{
 			Dir: bus.BasePath,
@@ -68,7 +69,8 @@ func (bus *Bus) Trigger(evt string, context map[string]interface{}) (bool, error
 			"Environment": bus.Env,
 			"Namespace":   bus.Namespace,
 			"Event": event{
-				Name: evt,
+				Name:  evt,
+				Error: evtErr,
 			},
 		}
 		for k, v := range context {

pkg/event/bus_test.go

@@ -104,7 +104,7 @@ func TestTrigger(t *testing.T) {
 			"Release":         "myrel",
 			"HelmfileCommand": "mycmd",
 		}
-		ok, err := bus.Trigger(c.triggeredEvt, data)
+		ok, err := bus.Trigger(c.triggeredEvt, nil, data)
 
 		if ok != c.expectedResult {
 			t.Errorf("unexpected result for case \"%s\": expected=%v, actual=%v", c.name, c.expectedResult, ok)

pkg/exectest/helm.go

@@ -0,0 +1,188 @@
+package exectest
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/roboll/helmfile/pkg/helmexec"
+)
+
+type ListKey struct {
+	Filter string
+	Flags  string
+}
+
+type DiffKey struct {
+	Name  string
+	Chart string
+	Flags string
+}
+
+type Helm struct {
+	Charts               []string
+	Repo                 []string
+	Releases             []Release
+	Deleted              []Release
+	Lists                map[ListKey]string
+	Diffs                map[DiffKey]error
+	Diffed               []Release
+	FailOnUnexpectedDiff bool
+	FailOnUnexpectedList bool
+	Version              *helmexec.Version
+
+	UpdateDepsCallbacks map[string]func(string) error
+
+	DiffMutex     *sync.Mutex
+	ChartsMutex   *sync.Mutex
+	ReleasesMutex *sync.Mutex
+}
+
+type Release struct {
+	Name  string
+	Flags []string
+}
+
+type Affected struct {
+	Upgraded []*Release
+	Deleted  []*Release
+	Failed   []*Release
+}
+
+func (helm *Helm) UpdateDeps(chart string) error {
+	if strings.Contains(chart, "error") {
+		return fmt.Errorf("simulated UpdateDeps failure for chart: %s", chart)
+	}
+	helm.Charts = append(helm.Charts, chart)
+
+	if helm.UpdateDepsCallbacks != nil {
+		callback, exists := helm.UpdateDepsCallbacks[chart]
+		if exists {
+			if err := callback(chart); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (helm *Helm) BuildDeps(name, chart string) error {
+	if strings.Contains(chart, "error") {
+		return errors.New("error")
+	}
+	helm.Charts = append(helm.Charts, chart)
+	return nil
+}
+
+func (helm *Helm) SetExtraArgs(args ...string) {
+	return
+}
+func (helm *Helm) SetHelmBinary(bin string) {
+	return
+}
+func (helm *Helm) AddRepo(name, repository, cafile, certfile, keyfile, username, password string) error {
+	helm.Repo = []string{name, repository, cafile, certfile, keyfile, username, password}
+	return nil
+}
+func (helm *Helm) UpdateRepo() error {
+	return nil
+}
+func (helm *Helm) SyncRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
+	if strings.Contains(name, "error") {
+		return errors.New("error")
+	}
+	helm.sync(helm.ReleasesMutex, func() {
+		helm.Releases = append(helm.Releases, Release{Name: name, Flags: flags})
+	})
+	helm.sync(helm.ChartsMutex, func() {
+		helm.Charts = append(helm.Charts, chart)
+	})
+
+	return nil
+}
+func (helm *Helm) DiffRelease(context helmexec.HelmContext, name, chart string, suppressDiff bool, flags ...string) error {
+	if helm.DiffMutex != nil {
+		helm.DiffMutex.Lock()
+	}
+	helm.Diffed = append(helm.Diffed, Release{Name: name, Flags: flags})
+	if helm.DiffMutex != nil {
+		helm.DiffMutex.Unlock()
+	}
+	key := DiffKey{Name: name, Chart: chart, Flags: strings.Join(flags, "")}
+	err, ok := helm.Diffs[key]
+	if !ok && helm.FailOnUnexpectedDiff {
+		return fmt.Errorf("unexpected diff with key: %v", key)
+	}
+	return err
+}
+func (helm *Helm) ReleaseStatus(context helmexec.HelmContext, release string, flags ...string) error {
+	if strings.Contains(release, "error") {
+		return errors.New("error")
+	}
+	helm.Releases = append(helm.Releases, Release{Name: release, Flags: flags})
+	return nil
+}
+func (helm *Helm) DeleteRelease(context helmexec.HelmContext, name string, flags ...string) error {
+	if strings.Contains(name, "error") {
+		return errors.New("error")
+	}
+	helm.Deleted = append(helm.Deleted, Release{Name: name, Flags: flags})
+	return nil
+}
+func (helm *Helm) List(context helmexec.HelmContext, filter string, flags ...string) (string, error) {
+	key := ListKey{Filter: filter, Flags: strings.Join(flags, "")}
+	res, ok := helm.Lists[key]
+	if !ok && helm.FailOnUnexpectedList {
+		return "", fmt.Errorf("unexpected list key: %v", key)
+	}
+	return res, nil
+}
+func (helm *Helm) DecryptSecret(context helmexec.HelmContext, name string, flags ...string) (string, error) {
+	return "", nil
+}
+func (helm *Helm) TestRelease(context helmexec.HelmContext, name string, flags ...string) error {
+	if strings.Contains(name, "error") {
+		return errors.New("error")
+	}
+	helm.Releases = append(helm.Releases, Release{Name: name, Flags: flags})
+	return nil
+}
+func (helm *Helm) Fetch(chart string, flags ...string) error {
+	return nil
+}
+func (helm *Helm) Lint(name, chart string, flags ...string) error {
+	return nil
+}
+func (helm *Helm) TemplateRelease(name, chart string, flags ...string) error {
+	return nil
+}
+
+func (helm *Helm) IsHelm3() bool {
+	return false
+}
+
+func (helm *Helm) GetVersion() helmexec.Version {
+	if helm.Version != nil {
+		return *helm.Version
+	}
+
+	return helmexec.Version{}
+}
+
+func (helm *Helm) IsVersionAtLeast(major int, minor int) bool {
+	if helm.Version == nil {
+		return false
+	}
+
+	return helm.Version.Major >= major && minor >= helm.Version.Minor
+}
+
+func (helm *Helm) sync(m *sync.Mutex, f func()) {
+	if m != nil {
+		m.Lock()
+		defer m.Unlock()
+	}
+
+	f()
+}

pkg/helmexec/context.go

@@ -8,15 +8,16 @@ import (
 type HelmContext struct {
 	Tillerless      bool
 	TillerNamespace string
+	HistoryMax      int
 	WorkerIndex     int
 }
 
-func (context *HelmContext) GetTillerlessArgs(helmBinary string) []string {
-	if context.Tillerless {
+func (context *HelmContext) GetTillerlessArgs(helm *execer) []string {
+	if context.Tillerless && !helm.IsHelm3() {
 		if context.TillerNamespace != "" {
-			return []string{"tiller", "run", context.TillerNamespace, "--", helmBinary}
+			return []string{"tiller", "run", context.TillerNamespace, "--", helm.helmBinary}
 		} else {
-			return []string{"tiller", "run", "--", helmBinary}
+			return []string{"tiller", "run", "--", helm.helmBinary}
 		}
 	} else {
 		return []string{}

pkg/helmexec/exec.go

@@ -6,6 +6,8 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -13,10 +15,6 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
-const (
-	command = "helm"
-)
-
 type decryptedSecret struct {
 	mutex sync.RWMutex
 	bytes []byte
@@ -24,6 +22,7 @@ type decryptedSecret struct {
 
 type execer struct {
 	helmBinary           string
+	version              Version
 	runner               Runner
 	logger               *zap.SugaredLogger
 	kubeContext          string
@@ -49,10 +48,62 @@ func NewLogger(writer io.Writer, logLevel string) *zap.SugaredLogger {
 	return zap.New(core).Sugar()
 }
 
+func getHelmVersion(helmBinary string, logger *zap.SugaredLogger, runner Runner) Version {
+
+	// Autodetect from `helm verison`
+	bytes, err := runner.Execute(helmBinary, []string{"version", "--client", "--short"}, nil)
+	if err != nil {
+		panic(err)
+	}
+
+	if bytes == nil || len(bytes) == 0 {
+		return Version{}
+	}
+
+	re := regexp.MustCompile("v(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)")
+	matches := re.FindStringSubmatch(string(bytes))
+
+	result := make(map[string]string)
+	for i, name := range re.SubexpNames() {
+		result[name] = matches[i]
+	}
+
+	major, err := strconv.Atoi(result["major"])
+	if err != nil {
+		panic(err)
+	}
+
+	minor, err := strconv.Atoi(result["minor"])
+	if err != nil {
+		panic(err)
+	}
+
+	patch, err := strconv.Atoi(result["patch"])
+	if err != nil {
+		panic(err)
+	}
+
+	// Support explicit helm3 opt-in via environment variable
+	if os.Getenv("HELMFILE_HELM3") != "" && major < 3 {
+		return Version{
+			Major: 3,
+			Minor: 0,
+			Patch: 0,
+		}
+	}
+
+	return Version{
+		Major: major,
+		Minor: minor,
+		Patch: patch,
+	}
+}
+
 // New for running helm commands
-func New(logger *zap.SugaredLogger, kubeContext string, runner Runner) *execer {
+func New(helmBinary string, logger *zap.SugaredLogger, kubeContext string, runner Runner) *execer {
 	return &execer{
-		helmBinary:       command,
+		helmBinary:       helmBinary,
+		version:          getHelmVersion(helmBinary, logger, runner),
 		logger:           logger,
 		kubeContext:      kubeContext,
 		runner:           runner,
@@ -70,6 +121,10 @@ func (helm *execer) SetHelmBinary(bin string) {
 
 func (helm *execer) AddRepo(name, repository, cafile, certfile, keyfile, username, password string) error {
 	var args []string
+	if name == "" && repository != "" {
+		helm.logger.Infof("empty field name\n")
+		return fmt.Errorf("empty field name")
+	}
 	args = append(args, "repo", "add", name, repository)
 	if certfile != "" && keyfile != "" {
 		args = append(args, "--cert-file", certfile, "--key-file", keyfile)
@@ -109,8 +164,15 @@ func (helm *execer) UpdateDeps(chart string) error {
 
 func (helm *execer) SyncRelease(context HelmContext, name, chart string, flags ...string) error {
 	helm.logger.Infof("Upgrading release=%v, chart=%v", name, chart)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
+
+	if helm.IsHelm3() {
+		flags = append(flags, "--history-max", strconv.Itoa(context.HistoryMax))
+	} else {
+		env["HELM_TILLER_HISTORY_MAX"] = strconv.Itoa(context.HistoryMax)
+	}
+
 	out, err := helm.exec(append(append(preArgs, "upgrade", "--install", "--reset-values", name, chart), flags...), env)
 	helm.write(out)
 	return err
@@ -118,7 +180,7 @@ func (helm *execer) SyncRelease(context HelmContext, name, chart string, flags .
 
 func (helm *execer) ReleaseStatus(context HelmContext, name string, flags ...string) error {
 	helm.logger.Infof("Getting status %v", name)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
 	out, err := helm.exec(append(append(preArgs, "status", name), flags...), env)
 	helm.write(out)
@@ -127,16 +189,27 @@ func (helm *execer) ReleaseStatus(context HelmContext, name string, flags ...str
 
 func (helm *execer) List(context HelmContext, filter string, flags ...string) (string, error) {
 	helm.logger.Infof("Listing releases matching %v", filter)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
 	var args []string
-	if helm.isHelm3() {
+	if helm.IsHelm3() {
 		args = []string{"list", "--filter", filter}
 	} else {
 		args = []string{"list", filter}
 	}
 
 	out, err := helm.exec(append(append(preArgs, args...), flags...), env)
+	// In v2 we have been expecting `helm list FILTER` prints nothing.
+	// In v3 helm still prints the header like `NAME	NAMESPACE	REVISION	UPDATED	STATUS	CHART	APP VERSION`,
+	// which confuses helmfile's existing logic that treats any non-empty output from `helm list` is considered as the indication
+	// of the release to exist.
+	//
+	// This fixes it by removing the header from the v3 output, so that the output is formatted the same as that of v2.
+	if helm.IsHelm3() {
+		lines := strings.Split(string(out), "\n")
+		lines = lines[1:]
+		out = []byte(strings.Join(lines, "\n"))
+	}
 	helm.write(out)
 	return string(out), err
 }
@@ -163,7 +236,7 @@ func (helm *execer) DecryptSecret(context HelmContext, name string, flags ...str
 		helm.decryptedSecretMutex.Unlock()
 
 		helm.logger.Infof("Decrypting secret %v", absPath)
-		preArgs := context.GetTillerlessArgs(helm.helmBinary)
+		preArgs := context.GetTillerlessArgs(helm)
 		env := context.getTillerlessEnv()
 		out, err := helm.exec(append(append(preArgs, "secrets", "dec", absPath), flags...), env)
 		helm.info(out)
@@ -212,7 +285,7 @@ func (helm *execer) DecryptSecret(context HelmContext, name string, flags ...str
 func (helm *execer) TemplateRelease(name string, chart string, flags ...string) error {
 	helm.logger.Infof("Templating release=%v, chart=%v", name, chart)
 	var args []string
-	if helm.isHelm3() {
+	if helm.IsHelm3() {
 		args = []string{"template", name, chart}
 	} else {
 		args = []string{"template", chart, "--name", name}
@@ -223,9 +296,9 @@ func (helm *execer) TemplateRelease(name string, chart string, flags ...string)
 	return err
 }
 
-func (helm *execer) DiffRelease(context HelmContext, name, chart string, flags ...string) error {
+func (helm *execer) DiffRelease(context HelmContext, name, chart string, suppressDiff bool, flags ...string) error {
 	helm.logger.Infof("Comparing release=%v, chart=%v", name, chart)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
 	out, err := helm.exec(append(append(preArgs, "diff", "upgrade", "--reset-values", "--allow-unreleased", name, chart), flags...), env)
 	// Do our best to write STDOUT only when diff existed
@@ -241,11 +314,13 @@ func (helm *execer) DiffRelease(context HelmContext, name, chart string, flags .
 		switch e := err.(type) {
 		case ExitError:
 			if e.ExitStatus() == 2 {
-				helm.write(out)
+				if !(suppressDiff) {
+					helm.write(out)
+				}
 				return err
 			}
 		}
-	} else {
+	} else if !(suppressDiff) {
 		helm.write(out)
 	}
 	return err
@@ -267,7 +342,7 @@ func (helm *execer) Fetch(chart string, flags ...string) error {
 
 func (helm *execer) DeleteRelease(context HelmContext, name string, flags ...string) error {
 	helm.logger.Infof("Deleting %v", name)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
 	out, err := helm.exec(append(append(preArgs, "delete", name), flags...), env)
 	helm.write(out)
@@ -276,14 +351,9 @@ func (helm *execer) DeleteRelease(context HelmContext, name string, flags ...str
 
 func (helm *execer) TestRelease(context HelmContext, name string, flags ...string) error {
 	helm.logger.Infof("Testing %v", name)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
+	preArgs := context.GetTillerlessArgs(helm)
 	env := context.getTillerlessEnv()
-	var args []string
-	if helm.isHelm3() {
-		args = []string{"test", "run", name}
-	} else {
-		args = []string{"test", name}
-	}
+	args := []string{"test", name}
 	out, err := helm.exec(append(append(preArgs, args...), flags...), env)
 	helm.write(out)
 	return err
@@ -316,6 +386,14 @@ func (helm *execer) write(out []byte) {
 	}
 }
 
-func (helm *execer) isHelm3() bool {
-	return os.Getenv("HELMFILE_HELM3") != ""
+func (helm *execer) IsHelm3() bool {
+	return helm.version.Major == 3
+}
+
+func (helm *execer) GetVersion() Version {
+	return helm.version
+}
+
+func (helm *execer) IsVersionAtLeast(major int, minor int) bool {
+	return helm.version.Major >= major && helm.version.Minor >= minor
 }

pkg/helmexec/exec_test.go

@@ -20,11 +20,11 @@ type mockRunner struct {
 }
 
 func (mock *mockRunner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) {
-	return []byte{}, nil
+	return mock.output, mock.err
 }
 
 func MockExecer(logger *zap.SugaredLogger, kubeContext string) *execer {
-	execer := New(logger, kubeContext, &mockRunner{})
+	execer := New("helm", logger, kubeContext, &mockRunner{})
 	return execer
 }
 
@@ -32,10 +32,7 @@ func MockExecer(logger *zap.SugaredLogger, kubeContext string) *execer {
 
 func TestNewHelmExec(t *testing.T) {
 	buffer := bytes.NewBufferString("something")
-	logger := NewLogger(buffer, "debug")
-	helm := New(logger, "dev", &ShellRunner{
-		Logger: logger,
-	})
+	helm := MockExecer(NewLogger(buffer, "debug"), "dev")
 	if helm.kubeContext != "dev" {
 		t.Error("helmexec.New() - kubeContext")
 	}
@@ -48,11 +45,7 @@ func TestNewHelmExec(t *testing.T) {
 }
 
 func Test_SetExtraArgs(t *testing.T) {
-	buffer := bytes.NewBufferString("something")
-	logger := NewLogger(buffer, "debug")
-	helm := New(NewLogger(os.Stdout, "info"), "dev", &ShellRunner{
-		Logger: logger,
-	})
+	helm := MockExecer(NewLogger(os.Stdout, "info"), "dev")
 	helm.SetExtraArgs()
 	if len(helm.extra) != 0 {
 		t.Error("helmexec.SetExtraArgs() - passing no arguments should not change extra field")
@@ -68,11 +61,7 @@ func Test_SetExtraArgs(t *testing.T) {
 }
 
 func Test_SetHelmBinary(t *testing.T) {
-	buffer := bytes.NewBufferString("something")
-	logger := NewLogger(buffer, "debug")
-	helm := New(NewLogger(os.Stdout, "info"), "dev", &ShellRunner{
-		Logger: logger,
-	})
+	helm := MockExecer(NewLogger(os.Stdout, "info"), "dev")
 	if helm.helmBinary != "helm" {
 		t.Error("helmexec.command - default command is not helm")
 	}
@@ -120,6 +109,15 @@ exec: helm repo add myRepo https://repo.example.com/ --kube-context dev:
 	expected = `Adding repo myRepo https://repo.example.com/
 exec: helm repo add myRepo https://repo.example.com/ --username example_user --password example_password --kube-context dev
 exec: helm repo add myRepo https://repo.example.com/ --username example_user --password example_password --kube-context dev: 
+`
+	if buffer.String() != expected {
+		t.Errorf("helmexec.AddRepo()\nactual = %v\nexpect = %v", buffer.String(), expected)
+	}
+
+	buffer.Reset()
+	helm.AddRepo("", "https://repo.example.com/", "", "", "", "", "")
+	expected = `empty field name
+
 `
 	if buffer.String() != expected {
 		t.Errorf("helmexec.AddRepo()\nactual = %v\nexpect = %v", buffer.String(), expected)
@@ -257,7 +255,7 @@ func Test_DiffRelease(t *testing.T) {
 	var buffer bytes.Buffer
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
-	helm.DiffRelease(HelmContext{}, "release", "chart", "--timeout 10", "--wait")
+	helm.DiffRelease(HelmContext{}, "release", "chart", false, "--timeout 10", "--wait")
 	expected := `Comparing release=release, chart=chart
 exec: helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev
 exec: helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev: 
@@ -267,7 +265,7 @@ exec: helm diff upgrade --reset-values --allow-unreleased release chart --timeou
 	}
 
 	buffer.Reset()
-	helm.DiffRelease(HelmContext{}, "release", "chart")
+	helm.DiffRelease(HelmContext{}, "release", "chart", false)
 	expected = `Comparing release=release, chart=chart
 exec: helm diff upgrade --reset-values --allow-unreleased release chart --kube-context dev
 exec: helm diff upgrade --reset-values --allow-unreleased release chart --kube-context dev: 
@@ -281,7 +279,7 @@ func Test_DiffReleaseTillerless(t *testing.T) {
 	var buffer bytes.Buffer
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
-	helm.DiffRelease(HelmContext{Tillerless: true}, "release", "chart", "--timeout 10", "--wait")
+	helm.DiffRelease(HelmContext{Tillerless: true}, "release", "chart", false, "--timeout 10", "--wait")
 	expected := `Comparing release=release, chart=chart
 exec: helm tiller run -- helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev
 exec: helm tiller run -- helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev: 
@@ -517,3 +515,58 @@ exec: helm template path/to/chart --name release --values file.yml --kube-contex
 		t.Errorf("helmexec.Template()\nactual = %v\nexpect = %v", buffer.String(), expected)
 	}
 }
+
+func Test_IsHelm3(t *testing.T) {
+	helm2Runner := mockRunner{output: []byte("Client: v2.16.0+ge13bc94\n")}
+	helm := New("helm", NewLogger(os.Stdout, "info"), "dev", &helm2Runner)
+	if helm.IsHelm3() {
+		t.Error("helmexec.IsHelm3() - Detected Helm 3 with Helm 2 version")
+	}
+
+	helm3Runner := mockRunner{output: []byte("v3.0.0+ge29ce2a\n")}
+	helm = New("helm", NewLogger(os.Stdout, "info"), "dev", &helm3Runner)
+	if !helm.IsHelm3() {
+		t.Error("helmexec.IsHelm3() - Failed to detect Helm 3")
+	}
+
+	os.Setenv("HELMFILE_HELM3", "1")
+	helm2Runner = mockRunner{output: []byte("Client: v2.16.0+ge13bc94\n")}
+	helm = New("helm", NewLogger(os.Stdout, "info"), "dev", &helm2Runner)
+	if !helm.IsHelm3() {
+		t.Error("helmexec.IsHelm3() - Helm3 not detected when HELMFILE_HELM3 is set")
+	}
+	os.Setenv("HELMFILE_HELM3", "")
+}
+
+func Test_GetVersion(t *testing.T) {
+	helm2Runner := mockRunner{output: []byte("Client: v2.16.1+ge13bc94\n")}
+	helm := New("helm", NewLogger(os.Stdout, "info"), "dev", &helm2Runner)
+	ver := helm.GetVersion()
+	if ver.Major != 2 || ver.Minor != 16 || ver.Patch != 1 {
+		t.Error(fmt.Sprintf("helmexec.GetVersion - did not detect correct Helm2 version; it was: %+v", ver))
+	}
+
+	helm3Runner := mockRunner{output: []byte("v3.2.4+ge29ce2a\n")}
+	helm = New("helm", NewLogger(os.Stdout, "info"), "dev", &helm3Runner)
+	ver = helm.GetVersion()
+	if ver.Major != 3 || ver.Minor != 2 || ver.Patch != 4 {
+		t.Error(fmt.Sprintf("helmexec.GetVersion - did not detect correct Helm3 version; it was: %+v", ver))
+	}
+}
+
+func Test_IsVersionAtLeast(t *testing.T) {
+	helm2Runner := mockRunner{output: []byte("Client: v2.16.1+ge13bc94\n")}
+	helm := New("helm", NewLogger(os.Stdout, "info"), "dev", &helm2Runner)
+	if !helm.IsVersionAtLeast(2, 1) {
+		t.Error("helmexec.IsVersionAtLeast - 2.16.1 not atleast 2.1")
+	}
+
+	if helm.IsVersionAtLeast(2, 19) {
+		t.Error("helmexec.IsVersionAtLeast - 2.16.1 is atleast 2.19")
+	}
+
+	if helm.IsVersionAtLeast(3, 2) {
+		t.Error("helmexec.IsVersionAtLeast - 2.16.1 is atleast 3.2")
+	}
+
+}

pkg/helmexec/exit_error.go

@@ -2,14 +2,13 @@ package helmexec
 
 import (
 	"fmt"
-	"path/filepath"
 	"strings"
 )
 
 func newExitError(helmCmdPath string, exitStatus int, errorMessage string) ExitError {
 	return ExitError{
-		msg:        fmt.Sprintf("%s exited with status %d:\n%s", filepath.Base(helmCmdPath), exitStatus, indent(strings.TrimSpace(errorMessage))),
-		exitStatus: exitStatus,
+		Message: fmt.Sprintf("the following cmd exited with status %d:\n%s\n\n%s", exitStatus, indent(strings.TrimSpace(helmCmdPath)), indent(strings.TrimSpace(errorMessage))),
+		Code:    exitStatus,
 	}
 }
 
@@ -23,14 +22,14 @@ func indent(text string) string {
 
 // ExitError is created whenever your shell command exits with a non-zero exit status
 type ExitError struct {
-	msg        string
-	exitStatus int
+	Message string
+	Code    int
 }
 
 func (e ExitError) Error() string {
-	return e.msg
+	return e.Message
 }
 
 func (e ExitError) ExitStatus() int {
-	return e.exitStatus
+	return e.Code
 }

pkg/helmexec/helmexec.go

@@ -1,5 +1,12 @@
 package helmexec
 
+// Version represents the version of helm
+type Version struct {
+	Major int
+	Minor int
+	Patch int
+}
+
 // Interface for executing helm commands
 type Interface interface {
 	SetExtraArgs(args ...string)
@@ -10,7 +17,7 @@ type Interface interface {
 	BuildDeps(name, chart string) error
 	UpdateDeps(chart string) error
 	SyncRelease(context HelmContext, name, chart string, flags ...string) error
-	DiffRelease(context HelmContext, name, chart string, flags ...string) error
+	DiffRelease(context HelmContext, name, chart string, suppressDiff bool, flags ...string) error
 	TemplateRelease(name, chart string, flags ...string) error
 	Fetch(chart string, flags ...string) error
 	Lint(name, chart string, flags ...string) error
@@ -19,8 +26,12 @@ type Interface interface {
 	TestRelease(context HelmContext, name string, flags ...string) error
 	List(context HelmContext, filter string, flags ...string) (string, error)
 	DecryptSecret(context HelmContext, name string, flags ...string) (string, error)
+	IsHelm3() bool
+	GetVersion() Version
+	IsVersionAtLeast(major int, minor int) bool
 }
 
 type DependencyUpdater interface {
 	UpdateDeps(chart string) error
+	IsHelm3() bool
 }

pkg/helmexec/runner.go

@@ -64,7 +64,8 @@ func combinedOutput(c *exec.Cmd, logger *zap.SugaredLogger) ([]byte, error) {
 			// so that helmfile could return its own exit code accordingly
 			waitStatus := ee.Sys().(syscall.WaitStatus)
 			exitStatus := waitStatus.ExitStatus()
-			err = newExitError(c.Path, exitStatus, string(e))
+			cmd := fmt.Sprintf("%s %s", c.Path, strings.Join(c.Args, " "))
+			err = newExitError(cmd, exitStatus, string(e))
 		default:
 			panic(fmt.Sprintf("unexpected error: %v", err))
 		}

pkg/maputil/maputil.go

@@ -1,6 +1,9 @@
 package maputil
 
-import "fmt"
+import (
+	"fmt"
+	"strings"
+)
 
 func CastKeysToStrings(s interface{}) (map[string]interface{}, error) {
 	new := map[string]interface{}{}
@@ -60,32 +63,129 @@ func recursivelyStringifyMapKey(v interface{}) (interface{}, error) {
 	return casted_v, nil
 }
 
-func Set(m map[string]interface{}, key []string, value string) map[string]interface{} {
+type arg interface {
+	getMap(map[string]interface{}) map[string]interface{}
+	set(map[string]interface{}, string)
+}
+
+type keyArg struct {
+	key string
+}
+
+func (a keyArg) getMap(m map[string]interface{}) map[string]interface{} {
+	_, ok := m[a.key]
+	if !ok {
+		m[a.key] = map[string]interface{}{}
+	}
+	switch t := m[a.key].(type) {
+	case map[string]interface{}:
+		return t
+	default:
+		panic(fmt.Errorf("unexpected type: %v(%T)", t, t))
+	}
+}
+
+func (a keyArg) set(m map[string]interface{}, value string) {
+	m[a.key] = value
+}
+
+type indexedKeyArg struct {
+	key   string
+	index int
+}
+
+func (a indexedKeyArg) getArray(m map[string]interface{}) []interface{} {
+	_, ok := m[a.key]
+	if !ok {
+		m[a.key] = make([]interface{}, a.index+1)
+	}
+	switch t := m[a.key].(type) {
+	case []interface{}:
+		if len(t) <= a.index {
+			t2 := make([]interface{}, a.index+1)
+			copy(t, t2)
+			t = t2
+		}
+		return t
+	default:
+		panic(fmt.Errorf("unexpected type: %v(%T)", t, t))
+	}
+}
+
+func (a indexedKeyArg) getMap(m map[string]interface{}) map[string]interface{} {
+	t := a.getArray(m)
+	if t[a.index] == nil {
+		t[a.index] = map[string]interface{}{}
+	}
+	switch t := t[a.index].(type) {
+	case map[string]interface{}:
+		return t
+	default:
+		panic(fmt.Errorf("unexpected type: %v(%T)", t, t))
+	}
+}
+
+func (a indexedKeyArg) set(m map[string]interface{}, value string) {
+	t := a.getArray(m)
+	t[a.index] = value
+	m[a.key] = t
+}
+
+func getCursor(key string) arg {
+	key = strings.ReplaceAll(key, "[", " ")
+	key = strings.ReplaceAll(key, "]", " ")
+	k := key
+	idx := 0
+
+	n, err := fmt.Sscanf(key, "%s %d", &k, &idx)
+
+	if n == 2 && err == nil {
+		return indexedKeyArg{
+			key:   k,
+			index: idx,
+		}
+	}
+
+	return keyArg{
+		key: key,
+	}
+}
+
+func ParseKey(key string) []string {
+	r := []string{}
+	part := ""
+	escaped := false
+	for _, rune := range key {
+		split := false
+		switch {
+		case escaped == false && rune == '\\':
+			escaped = true
+			continue
+		case rune == '.':
+			split = escaped == false
+		}
+		escaped = false
+		if split {
+			r = append(r, part)
+			part = ""
+		} else {
+			part = part + string(rune)
+		}
+	}
+	if len(part) > 0 {
+		r = append(r, part)
+	}
+	return r
+}
+
+func Set(m map[string]interface{}, key []string, value string) {
 	if len(key) == 0 {
 		panic(fmt.Errorf("bug: unexpected length of key: %d", len(key)))
 	}
 
-	k := key[0]
-
-	if len(key) == 1 {
-		m[k] = value
-		return m
+	for len(key) > 1 {
+		m, key = getCursor(key[0]).getMap(m), key[1:]
 	}
 
-	remain := key[1:]
-
-	nested, ok := m[k]
-	if !ok {
-		nested = map[string]interface{}{}
-	}
-	switch t := nested.(type) {
-	case map[string]interface{}:
-		nested = Set(t, remain, value)
-	default:
-		panic(fmt.Errorf("unexpected type: %v(%T)", t, t))
-	}
-
-	m[k] = nested
-
-	return m
+	getCursor(key[0]).set(m, value)
 }

pkg/maputil/maputil_test.go

@@ -59,3 +59,72 @@ func TestMapUtil_IFKeys(t *testing.T) {
 		t.Errorf("unexpected c: expected=C, got=%s", c)
 	}
 }
+
+func TestMapUtil_KeyArg(t *testing.T) {
+	m := map[string]interface{}{}
+
+	key := []string{"a", "b", "c"}
+
+	Set(m, key, "C")
+
+	c := (((m["a"].(map[string]interface{}))["b"]).(map[string]interface{}))["c"]
+
+	if c != "C" {
+		t.Errorf("unexpected c: expected=C, got=%s", c)
+	}
+}
+
+func TestMapUtil_IndexedKeyArg(t *testing.T) {
+	m := map[string]interface{}{}
+
+	key := []string{"a", "b[0]", "c"}
+
+	Set(m, key, "C")
+
+	c := (((m["a"].(map[string]interface{}))["b"].([]interface{}))[0].(map[string]interface{}))["c"]
+
+	if c != "C" {
+		t.Errorf("unexpected c: expected=C, got=%s", c)
+	}
+}
+
+type parseKeyTc struct {
+	key    string
+	result map[int]string
+}
+
+func TestMapUtil_ParseKey(t *testing.T) {
+	tcs := []parseKeyTc{
+		{
+			key: `a.b.c`,
+			result: map[int]string{
+				0: "a",
+				1: "b",
+				2: "c",
+			},
+		},
+		{
+			key: `a\.b.c`,
+			result: map[int]string{
+				0: "a.b",
+				1: "c",
+			},
+		},
+		{
+			key: `a\.b\.c`,
+			result: map[int]string{
+				0: "a.b.c",
+			},
+		},
+	}
+
+	for _, tc := range tcs {
+		parts := ParseKey(tc.key)
+
+		for index, value := range tc.result {
+			if parts[index] != value {
+				t.Errorf("unexpected key part[%d]: expected=%s, got=%s", index, value, parts[index])
+			}
+		}
+	}
+}

pkg/plugins/vals.go

@@ -0,0 +1,23 @@
+package plugins
+
+import (
+	"github.com/variantdev/vals"
+	"sync"
+)
+
+const (
+	// cache size for improving performance of ref+.* secrets rendering
+	valsCacheSize = 512
+)
+
+var instance *vals.Runtime
+var once sync.Once
+
+func ValsInstance() (*vals.Runtime, error) {
+	var err error
+	once.Do(func() {
+		instance, err = vals.New(vals.Options{CacheSize: valsCacheSize})
+	})
+
+	return instance, err
+}

pkg/plugins/vals_test.go

@@ -0,0 +1,17 @@
+package plugins
+
+import "testing"
+
+func TestValsInstance(t *testing.T) {
+	i, err := ValsInstance()
+
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	i2, _ := ValsInstance()
+
+	if i != i2 {
+		t.Error("Instances should be equal")
+	}
+}

pkg/state/chart_dependency.go

@@ -3,7 +3,9 @@ package state
 import (
 	"fmt"
 	"github.com/Masterminds/semver"
+	goversion "github.com/hashicorp/go-version"
 	"github.com/r3labs/diff"
+	"github.com/roboll/helmfile/pkg/app/version"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
@@ -50,6 +52,7 @@ type ChartRequirements struct {
 }
 
 type ChartLockedRequirements struct {
+	Version              string                    `yaml:"version"`
 	ResolvedDependencies []ResolvedChartDependency `yaml:"dependencies"`
 	Digest               string                    `yaml:"digest"`
 	Generated            string                    `yaml:"generated"`
@@ -195,6 +198,7 @@ func (st *HelmState) updateDependenciesInTempDir(shell helmexec.DependencyUpdate
 	}
 
 	if len(unresolved.deps) == 0 {
+		st.logger.Warnf("There are no repositories defined in your helmfile.yaml.\nThis means helmfile cannot update your dependencies or create a lock file.\nSee https://github.com/roboll/helmfile/issues/878 for more information.")
 		return st, nil
 	}
 
@@ -279,7 +283,7 @@ func (m *chartDependencyManager) lockFileName() string {
 }
 
 func (m *chartDependencyManager) Update(shell helmexec.DependencyUpdater, wd string, unresolved *UnresolvedDependencies) (*ResolvedDependencies, error) {
-	if isHelm3() {
+	if shell.IsHelm3() {
 		return m.updateHelm3(shell, wd, unresolved)
 	}
 	return m.updateHelm2(shell, wd, unresolved)
@@ -287,7 +291,7 @@ func (m *chartDependencyManager) Update(shell helmexec.DependencyUpdater, wd str
 
 func (m *chartDependencyManager) updateHelm3(shell helmexec.DependencyUpdater, wd string, unresolved *UnresolvedDependencies) (*ResolvedDependencies, error) {
 	// Generate `Chart.yaml` of the temporary local chart
-	chartMetaContent := fmt.Sprintf("name: %s\nversion: 1.0.0\n", m.Name)
+	chartMetaContent := fmt.Sprintf("name: %s\nversion: 1.0.0\napiVersion: v2\n", m.Name)
 
 	// Generate `requirements.yaml` of the temporary local chart from the helmfile state
 	reqsContent, err := yaml.Marshal(unresolved.ToChartRequirements())
@@ -329,7 +333,7 @@ func (m *chartDependencyManager) doUpdate(chartLockFile string, unresolved *Unre
 		return nil, err
 	}
 
-	if isHelm3() && originalLockFileContent != nil {
+	if shell.IsHelm3() && originalLockFileContent != nil {
 		if err := m.writeBytes(filepath.Join(wd, chartLockFile), originalLockFileContent); err != nil {
 			return nil, err
 		}
@@ -356,7 +360,7 @@ func (m *chartDependencyManager) doUpdate(chartLockFile string, unresolved *Unre
 	})
 
 	// Don't update lock file if no dependency updated.
-	if !isHelm3() && originalLockFileContent != nil {
+	if !shell.IsHelm3() && originalLockFileContent != nil {
 		originalLockedReqs := &ChartLockedRequirements{}
 		if err := yaml.Unmarshal(originalLockFileContent, originalLockedReqs); err != nil {
 			return nil, err
@@ -373,6 +377,8 @@ func (m *chartDependencyManager) doUpdate(chartLockFile string, unresolved *Unre
 		}
 	}
 
+	lockedReqs.Version = version.Version
+
 	updatedLockFileContent, err = yaml.Marshal(lockedReqs)
 
 	if err != nil {
@@ -403,6 +409,25 @@ func (m *chartDependencyManager) Resolve(unresolved *UnresolvedDependencies) (*R
 		return nil, false, err
 	}
 
+	// Make sure go run main.go works and compatible with old lock files.
+	if version.Version != "" && lockedReqs.Version != "" {
+		lockedVersion, err := goversion.NewVersion(lockedReqs.Version)
+
+		if err != nil {
+			return nil, false, err
+		}
+
+		currentVersion, err := goversion.NewVersion(version.Version)
+
+		if err != nil {
+			return nil, false, err
+		}
+
+		if currentVersion.LessThan(lockedVersion) {
+			return nil, false, fmt.Errorf("the lockfile was created by Helmfile %s, which is newer than current %s; Please upgrade to Helmfile %s or greater", lockedVersion.Original(), currentVersion.Original(), lockedVersion.Original())
+		}
+	}
+
 	resolved := &ResolvedDependencies{deps: map[string][]ResolvedChartDependency{}}
 	for _, d := range lockedReqs.ResolvedDependencies {
 		if err := resolved.add(d); err != nil {

pkg/state/create.go

@@ -4,17 +4,22 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"github.com/roboll/helmfile/pkg/helmexec"
 	"io"
 	"os"
 
 	"github.com/imdario/mergo"
 	"github.com/roboll/helmfile/pkg/environment"
-	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/maputil"
+	"github.com/variantdev/vals"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
 )
 
+const (
+	DefaultHelmBinary = "helm"
+)
+
 type StateLoadError struct {
 	msg   string
 	Cause error
@@ -33,27 +38,34 @@ func (e *UndefinedEnvError) Error() string {
 }
 
 type StateCreator struct {
-	logger     *zap.SugaredLogger
-	readFile   func(string) ([]byte, error)
-	fileExists func(string) (bool, error)
-	abs        func(string) (string, error)
-	glob       func(string) ([]string, error)
-	helm       helmexec.Interface
+	logger      *zap.SugaredLogger
+	readFile    func(string) ([]byte, error)
+	fileExists  func(string) (bool, error)
+	abs         func(string) (string, error)
+	glob        func(string) ([]string, error)
+	valsRuntime vals.Evaluator
 
 	Strict bool
 
 	LoadFile func(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*HelmState, error)
+
+	getHelm func(*HelmState) helmexec.Interface
+
+	overrideHelmBinary string
 }
 
-func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), helm helmexec.Interface) *StateCreator {
+func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), valsRuntime vals.Evaluator, getHelm func(*HelmState) helmexec.Interface, overrideHelmBinary string) *StateCreator {
 	return &StateCreator{
-		logger:     logger,
-		readFile:   readFile,
-		fileExists: fileExists,
-		abs:        abs,
-		glob:       glob,
-		Strict:     true,
-		helm:       helm,
+		logger:      logger,
+		readFile:    readFile,
+		fileExists:  fileExists,
+		abs:         abs,
+		glob:        glob,
+		Strict:      true,
+		valsRuntime: valsRuntime,
+		getHelm:     getHelm,
+
+		overrideHelmBinary: overrideHelmBinary,
 	}
 }
 
@@ -63,7 +75,6 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState,
 
 	state.FilePath = file
 	state.basePath = baseDir
-	state.helm = c.helm
 
 	decoder := yaml.NewDecoder(bytes.NewReader(content))
 	if !c.Strict {
@@ -101,21 +112,29 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState,
 		state.HelmDefaults.KubeContext = state.DeprecatedContext
 	}
 
+	if c.overrideHelmBinary != "" && c.overrideHelmBinary != DefaultHelmBinary {
+		state.DefaultHelmBinary = c.overrideHelmBinary
+	} else if state.DefaultHelmBinary == "" {
+		// Let `helmfile --helm-binary ""` not break this helmfile run
+		state.DefaultHelmBinary = DefaultHelmBinary
+	}
+
 	state.logger = c.logger
 
 	state.readFile = c.readFile
 	state.removeFile = os.Remove
 	state.fileExists = c.fileExists
 	state.glob = c.glob
+	state.valsRuntime = c.valsRuntime
 
 	return &state, nil
 }
 
 // LoadEnvValues loads environment values files relative to the `baseDir`
-func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *environment.Environment) (*HelmState, error) {
+func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *environment.Environment, failOnMissingEnv bool) (*HelmState, error) {
 	state := *target
 
-	e, err := state.loadEnvValues(env, ctxEnv, c.readFile, c.glob)
+	e, err := c.loadEnvValues(&state, env, failOnMissingEnv, ctxEnv, c.readFile, c.glob)
 	if err != nil {
 		return nil, &StateLoadError{fmt.Sprintf("failed to read %s", state.FilePath), err}
 	}
@@ -131,6 +150,7 @@ func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *envi
 }
 
 // Parses YAML into HelmState, while loading environment values files relative to the `baseDir`
+// evaluateBases=true means that this is NOT a base helmfile
 func (c *StateCreator) ParseAndLoad(content []byte, baseDir, file string, envName string, evaluateBases bool, envValues *environment.Environment) (*HelmState, error) {
 	state, err := c.Parse(content, baseDir, file)
 	if err != nil {
@@ -141,14 +161,14 @@ func (c *StateCreator) ParseAndLoad(content []byte, baseDir, file string, envNam
 		if len(state.Bases) > 0 {
 			return nil, errors.New("nested `base` helmfile is unsupported. please submit a feature request if you need this!")
 		}
+	} else {
+		state, err = c.loadBases(envValues, state, baseDir)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	state, err = c.loadBases(envValues, state, baseDir)
-	if err != nil {
-		return nil, err
-	}
-
-	state, err = c.LoadEnvValues(state, envName, envValues)
+	state, err = c.LoadEnvValues(state, envName, envValues, evaluateBases)
 	if err != nil {
 		return nil, err
 	}
@@ -178,7 +198,7 @@ func (c *StateCreator) loadBases(envValues *environment.Environment, st *HelmSta
 	return layers[0], nil
 }
 
-func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, readFile func(string) ([]byte, error), glob func(string) ([]string, error)) (*environment.Environment, error) {
+func (c *StateCreator) loadEnvValues(st *HelmState, name string, failOnMissingEnv bool, ctxEnv *environment.Environment, readFile func(string) ([]byte, error), glob func(string) ([]string, error)) (*environment.Environment, error) {
 	envVals := map[string]interface{}{}
 	envSpec, ok := st.Environments[name]
 	if ok {
@@ -202,11 +222,11 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 
 				envSecretFiles = append(envSecretFiles, resolved...)
 			}
-			if err = st.scatterGatherEnvSecretFiles(envSecretFiles, envVals, readFile); err != nil {
+			if err = c.scatterGatherEnvSecretFiles(st, envSecretFiles, envVals, readFile); err != nil {
 				return nil, err
 			}
 		}
-	} else if ctxEnv == nil && name != DefaultEnv {
+	} else if ctxEnv == nil && name != DefaultEnv && failOnMissingEnv {
 		return nil, &UndefinedEnvError{msg: fmt.Sprintf("environment \"%s\" is not defined", name)}
 	}
 
@@ -215,7 +235,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 	if ctxEnv != nil {
 		intEnv := *ctxEnv
 
-		if err := mergo.Merge(&intEnv, newEnv, mergo.WithOverride); err != nil {
+		if err := mergo.Merge(&intEnv, newEnv, mergo.WithOverride, mergo.WithOverwriteWithEmptyValue); err != nil {
 			return nil, fmt.Errorf("error while merging environment values for \"%s\": %v", name, err)
 		}
 
@@ -225,7 +245,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 	return newEnv, nil
 }
 
-func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, envVals map[string]interface{}, readFile func(string) ([]byte, error)) error {
+func (c *StateCreator) scatterGatherEnvSecretFiles(st *HelmState, envSecretFiles []string, envVals map[string]interface{}, readFile func(string) ([]byte, error)) error {
 	var errs []error
 
 	inputs := envSecretFiles
@@ -239,7 +259,6 @@ func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, envVal
 
 	secrets := make(chan string, inputsSize)
 	results := make(chan secretResult, inputsSize)
-	helm := st.helm
 
 	st.scatterGather(0, inputsSize,
 		func() {
@@ -252,7 +271,7 @@ func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, envVal
 			for path := range secrets {
 				release := &ReleaseSpec{}
 				flags := st.appendConnectionFlags([]string{}, release)
-				decFile, err := helm.DecryptSecret(st.createHelmContext(release, 0), path, flags...)
+				decFile, err := c.getHelm(st).DecryptSecret(st.createHelmContext(release, 0), path, flags...)
 				if err != nil {
 					results <- secretResult{nil, err, path}
 					continue
@@ -284,7 +303,7 @@ func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, envVal
 				if result.err != nil {
 					errs = append(errs, result.err)
 				} else {
-					if err := mergo.Merge(&envVals, &result.result, mergo.WithOverride); err != nil {
+					if err := mergo.Merge(&envVals, &result.result, mergo.WithOverride, mergo.WithOverwriteWithEmptyValue); err != nil {
 						errs = append(errs, fmt.Errorf("failed to load environment secrets file \"%s\": %v", result.path, err))
 					}
 				}

pkg/state/create_test.go

@@ -9,8 +9,8 @@ import (
 	"github.com/roboll/helmfile/pkg/testhelper"
 	"go.uber.org/zap"
 
-	. "gotest.tools/assert"
-	"gotest.tools/assert/cmp"
+	. "gotest.tools/v3/assert"
+	"gotest.tools/v3/assert/cmp"
 )
 
 func createFromYaml(content []byte, file string, env string, logger *zap.SugaredLogger) (*HelmState, error) {
@@ -20,7 +20,7 @@ func createFromYaml(content []byte, file string, env string, logger *zap.Sugared
 		abs:      filepath.Abs,
 		Strict:   true,
 	}
-	return c.ParseAndLoad(content, filepath.Dir(file), file, env, false, nil)
+	return c.ParseAndLoad(content, filepath.Dir(file), file, env, true, nil)
 }
 
 func TestReadFromYaml(t *testing.T) {
@@ -46,7 +46,7 @@ func TestReadFromYaml(t *testing.T) {
 	}
 }
 
-func TestReadFromYaml_InexistentEnv(t *testing.T) {
+func TestReadFromYaml_NonexistentEnv(t *testing.T) {
 	yamlFile := "example/path/to/yaml/file"
 	yamlContent := []byte(`releases:
 - name: myrelease
@@ -108,7 +108,7 @@ bar: {{ readFile "bar.txt" }}
 	})
 	testFs.Cwd = "/example/path/to"
 
-	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", false, nil)
+	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil, nil, "").ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", true, nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}

pkg/state/envvals_loader.go

@@ -73,7 +73,7 @@ func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *str
 			if err != nil {
 				return nil, err
 			}
-			if err := mergo.Merge(&result, &vals, mergo.WithOverride); err != nil {
+			if err := mergo.Merge(&result, &vals, mergo.WithOverride, mergo.WithOverwriteWithEmptyValue); err != nil {
 				return nil, fmt.Errorf("failed to merge %v: %v", m, err)
 			}
 		}

pkg/state/envvals_loader_test.go

@@ -0,0 +1,116 @@
+package state
+
+import (
+	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+)
+
+func newLoader() *EnvironmentValuesLoader {
+	log, err := zap.NewDevelopment(zap.AddStacktrace(zap.DebugLevel))
+	if err != nil {
+		panic(err)
+	}
+
+	sugar := log.Sugar()
+
+	storage := &Storage{
+		FilePath: "./helmfile.yaml",
+		basePath: ".",
+		glob:     filepath.Glob,
+		logger:   sugar,
+	}
+
+	return NewEnvironmentValuesLoader(storage, ioutil.ReadFile, sugar)
+}
+
+// See https://github.com/roboll/helmfile/pull/1169
+func TestEnvValsLoad_SingleValuesFile(t *testing.T) {
+	l := newLoader()
+
+	actual, err := l.LoadEnvironmentValues(nil, []interface{}{"testdata/values.5.yaml"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := map[string]interface{}{
+		"affinity": map[string]interface{}{},
+	}
+
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf(diff)
+	}
+}
+
+// See https://github.com/roboll/helmfile/issues/1150
+func TestEnvValsLoad_OverwriteNilValue_Issue1150(t *testing.T) {
+	l := newLoader()
+
+	actual, err := l.LoadEnvironmentValues(nil, []interface{}{"testdata/values.1.yaml", "testdata/values.2.yaml"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := map[string]interface{}{
+		"components": map[string]interface{}{
+			"etcd-operator": map[string]interface{}{
+				"version": "0.10.3",
+			},
+		},
+	}
+
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf(diff)
+	}
+}
+
+// See https://github.com/roboll/helmfile/issues/1154
+func TestEnvValsLoad_OverwriteWithNilValue_Issue1154(t *testing.T) {
+	l := newLoader()
+
+	actual, err := l.LoadEnvironmentValues(nil, []interface{}{"testdata/values.3.yaml", "testdata/values.4.yaml"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := map[string]interface{}{
+		"components": map[string]interface{}{
+			"etcd-operator": map[string]interface{}{
+				"version": "0.10.3",
+			},
+			"prometheus": nil,
+		},
+	}
+
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf(diff)
+	}
+}
+
+// See https://github.com/roboll/helmfile/issues/1168
+func TestEnvValsLoad_OverwriteEmptyValue_Issue1168(t *testing.T) {
+	l := newLoader()
+
+	actual, err := l.LoadEnvironmentValues(nil, []interface{}{"testdata/issues/1168/addons.yaml", "testdata/issues/1168/addons2.yaml"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := map[string]interface{}{
+		"addons": map[string]interface{}{
+			"mychart": map[string]interface{}{
+				"skip":      false,
+				"name":      "mychart",
+				"namespace": "kube-system",
+				"chart":     "stable/mychart",
+				"version":   "1.0.0",
+			},
+		},
+	}
+
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf(diff)
+	}
+}

pkg/state/helmx.go

@@ -1,5 +1,10 @@
 package state
 
+import (
+	"github.com/variantdev/chartify"
+	"strings"
+)
+
 type Dependency struct {
 	Chart   string `yaml:"chart"`
 	Version string `yaml:"version"`
@@ -7,49 +12,73 @@ type Dependency struct {
 }
 
 func (st *HelmState) appendHelmXFlags(flags []string, release *ReleaseSpec) ([]string, error) {
+	for _, adopt := range release.Adopt {
+		flags = append(flags, "--adopt", adopt)
+	}
+
+	return flags, nil
+}
+
+func (st *HelmState) PrepareChartify(release *ReleaseSpec) (bool, *chartify.ChartifyOpts) {
+	var opts chartify.ChartifyOpts
+
+	var shouldRun bool
+
 	for _, d := range release.Dependencies {
 		var dep string
+
 		if d.Alias != "" {
 			dep += d.Alias + "="
+		} else {
+			a := strings.Split(d.Chart, "/")
+
+			chart := a[len(a)-1]
+
+			dep += chart + "="
 		}
+
 		dep += d.Chart
+
 		if d.Version != "" {
 			dep += ":" + d.Version
 		}
-		flags = append(flags, "--dependency", dep)
-	}
 
-	for _, adopt := range release.Adopt {
-		flags = append(flags, "--adopt", adopt)
+		opts.AdhocChartDependencies = append(opts.AdhocChartDependencies, dep)
+
+		shouldRun = true
 	}
 
 	jsonPatches := release.JSONPatches
 	if len(jsonPatches) > 0 {
 		generatedFiles, err := st.generateTemporaryValuesFiles(jsonPatches, release.MissingFileHandler)
 		if err != nil {
-			return nil, err
+			return false, nil
 		}
 
 		for _, f := range generatedFiles {
-			flags = append(flags, "--json-patch", f)
+			opts.JsonPatches = append(opts.JsonPatches, f)
 		}
 
 		release.generatedValues = append(release.generatedValues, generatedFiles...)
+
+		shouldRun = true
 	}
 
 	strategicMergePatches := release.StrategicMergePatches
 	if len(strategicMergePatches) > 0 {
 		generatedFiles, err := st.generateTemporaryValuesFiles(strategicMergePatches, release.MissingFileHandler)
 		if err != nil {
-			return nil, err
+			return false, nil
 		}
 
 		for _, f := range generatedFiles {
-			flags = append(flags, "--strategic-merge-patch", f)
+			opts.StrategicMergePatches = append(opts.StrategicMergePatches, f)
 		}
 
 		release.generatedValues = append(release.generatedValues, generatedFiles...)
+
+		shouldRun = true
 	}
 
-	return flags, nil
+	return shouldRun, &opts
 }

pkg/state/release.go

@@ -93,7 +93,7 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
 		result.Labels[key] = s.String()
 	}
 
-	if result.ValuesTemplate != nil && len(result.ValuesTemplate) > 0 {
+	if len(result.ValuesTemplate) > 0 {
 		for i, t := range result.ValuesTemplate {
 			switch ts := t.(type) {
 			case map[interface{}]interface{}:
@@ -117,7 +117,9 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
 			}
 		}
 
-		result.Values = result.ValuesTemplate
+		var newvals []interface{}
+		newvals = append(newvals, result.ValuesTemplate...)
+		result.Values = append(newvals, result.Values...)
 	}
 
 	for i, t := range result.Values {
@@ -139,7 +141,7 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
 		result.Secrets[i] = s.String()
 	}
 
-	if result.SetValuesTemplate != nil && len(result.SetValuesTemplate) > 0 {
+	if len(result.SetValuesTemplate) > 0 {
 		for i, val := range result.SetValuesTemplate {
 			{
 				// name
@@ -178,7 +180,9 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
 			}
 		}
 
-		result.SetValues = result.SetValuesTemplate
+		var newvals []SetValue
+		newvals = append(newvals, result.SetValuesTemplate...)
+		result.SetValues = append(newvals, result.SetValues...)
 	}
 
 	return result, nil

pkg/state/release_error.go

@@ -13,9 +13,19 @@ type ReleaseError struct {
 }
 
 func (e *ReleaseError) Error() string {
-	return fmt.Sprintf("failed processing release %s: %v", e.Name, e.err.Error())
+	return e.err.Error()
 }
 
-func newReleaseError(release *ReleaseSpec, err error) *ReleaseError {
-	return &ReleaseError{release, err, ReleaseErrorCodeFailure}
+func NewReleaseError(release *ReleaseSpec, err error, code int) *ReleaseError {
+	return &ReleaseError{
+		ReleaseSpec: release,
+		err:         err,
+		Code:        code,
+	}
+}
+
+func newReleaseFailedError(release *ReleaseSpec, err error) *ReleaseError {
+	wrappedErr := fmt.Errorf("failed processing release %s: %v", release.Name, err.Error())
+
+	return NewReleaseError(release, wrappedErr, ReleaseErrorCodeFailure)
 }

pkg/state/state_exec_tmpl.go

@@ -4,28 +4,12 @@ import (
 	"fmt"
 	"reflect"
 
-	"github.com/imdario/mergo"
-	"github.com/roboll/helmfile/pkg/maputil"
 	"github.com/roboll/helmfile/pkg/tmpl"
 	"gopkg.in/yaml.v2"
 )
 
 func (st *HelmState) Values() (map[string]interface{}, error) {
-	vals := map[string]interface{}{}
-
-	if err := mergo.Merge(&vals, st.Env.Defaults, mergo.WithOverride); err != nil {
-		return nil, err
-	}
-	if err := mergo.Merge(&vals, st.Env.Values, mergo.WithOverride); err != nil {
-		return nil, err
-	}
-
-	vals, err := maputil.CastKeysToStrings(vals)
-	if err != nil {
-		return nil, err
-	}
-
-	return vals, nil
+	return st.Env.GetMergedValues()
 }
 
 func (st *HelmState) mustLoadVals() map[string]interface{} {
@@ -39,7 +23,7 @@ func (st *HelmState) mustLoadVals() map[string]interface{} {
 func (st *HelmState) valuesFileTemplateData() EnvironmentTemplateData {
 	return EnvironmentTemplateData{
 		Environment: st.Env,
-		Namespace:   st.Namespace,
+		Namespace:   st.OverrideNamespace,
 		Values:      st.mustLoadVals(),
 	}
 }

pkg/state/state_exec_tmpl_test.go

@@ -138,9 +138,9 @@ func TestHelmState_executeTemplates(t *testing.T) {
 				HelmDefaults: HelmSpec{
 					KubeContext: "test_context",
 				},
-				Env:          environment.Environment{Name: "test_env"},
-				Namespace:    "test-namespace_",
-				Repositories: nil,
+				Env:               environment.Environment{Name: "test_env"},
+				OverrideNamespace: "test-namespace_",
+				Repositories:      nil,
 				Releases: []ReleaseSpec{
 					tt.input,
 				},
@@ -238,9 +238,9 @@ func TestHelmState_recursiveRefsTemplates(t *testing.T) {
 				HelmDefaults: HelmSpec{
 					KubeContext: "test_context",
 				},
-				Env:          environment.Environment{Name: "test_env"},
-				Namespace:    "test-namespace_",
-				Repositories: nil,
+				Env:               environment.Environment{Name: "test_env"},
+				OverrideNamespace: "test-namespace_",
+				Repositories:      nil,
 				Releases: []ReleaseSpec{
 					tt.input,
 				},

pkg/state/state_run.go

@@ -2,9 +2,11 @@ package state
 
 import (
 	"fmt"
+	"sort"
 	"sync"
 
 	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/variantdev/dag/pkg/dag"
 )
 
 type result struct {
@@ -36,9 +38,7 @@ func (st *HelmState) scatterGather(concurrency int, items int, produceInputs fun
 
 	for w := 1; w <= concurrency; w++ {
 		go func(id int) {
-			st.logger.Debugf("worker %d/%d started", id, concurrency)
 			receiveInputsAndProduceIntermediates(id)
-			st.logger.Debugf("worker %d/%d finished", id, concurrency)
 			waitGroup.Done()
 		}(w)
 	}
@@ -51,9 +51,14 @@ func (st *HelmState) scatterGather(concurrency int, items int, produceInputs fun
 
 func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency int,
 	do func(ReleaseSpec, int) error) []error {
+
+	return st.iterateOnReleases(helm, concurrency, st.Releases, do)
+}
+
+func (st *HelmState) iterateOnReleases(helm helmexec.Interface, concurrency int, inputs []ReleaseSpec,
+	do func(ReleaseSpec, int) error) []error {
 	var errs []error
 
-	inputs := st.Releases
 	inputsSize := len(inputs)
 
 	releases := make(chan ReleaseSpec)
@@ -71,21 +76,16 @@ func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency
 		func(id int) {
 			for release := range releases {
 				err := do(release, id)
-				st.logger.Debugf("sending result for release: %s\n", release.Name)
+				st.logger.Debugf("release %q processed", release.Name)
 				results <- result{release: release, err: err}
-				st.logger.Debugf("sent result for release: %s\n", release.Name)
 			}
 		},
 		func() {
-			for i := range inputs {
-				st.logger.Debugf("receiving result %d", i)
+			for range inputs {
 				r := <-results
 				if r.err != nil {
 					errs = append(errs, fmt.Errorf("release \"%s\" failed: %v", r.release.Name, r.err))
-				} else {
-					st.logger.Debugf("received result for release \"%s\"", r.release.Name)
 				}
-				st.logger.Debugf("received result for %d", i)
 			}
 		},
 	)
@@ -96,3 +96,105 @@ func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency
 
 	return nil
 }
+
+type PlanOpts struct {
+	Reverse bool
+}
+
+func (st *HelmState) PlanReleases(reverse bool) ([][]Release, error) {
+	marked, err := st.SelectReleasesWithOverrides()
+	if err != nil {
+		return nil, err
+	}
+
+	groups, err := SortedReleaseGroups(marked, reverse)
+	if err != nil {
+		return nil, err
+	}
+
+	return groups, nil
+}
+
+func SortedReleaseGroups(releases []Release, reverse bool) ([][]Release, error) {
+	groups, err := GroupReleasesByDependency(releases)
+	if err != nil {
+		return nil, err
+	}
+
+	if reverse {
+		sort.Slice(groups, func(i, j int) bool {
+			return j < i
+		})
+	}
+
+	return groups, nil
+}
+
+func GroupReleasesByDependency(releases []Release) ([][]Release, error) {
+	idToReleases := map[string][]Release{}
+	idToIndex := map[string]int{}
+
+	d := dag.New()
+	for i, r := range releases {
+
+		id := ReleaseToID(&r.ReleaseSpec)
+
+		idToReleases[id] = append(idToReleases[id], r)
+		idToIndex[id] = i
+
+		// Only compute dependencies from non-filtered releases
+		if !r.Filtered {
+			d.Add(id, dag.Dependencies(r.Needs))
+		}
+	}
+
+	for _, r := range releases {
+		if !r.Filtered {
+			for _, n := range r.Needs {
+				if _, ok := idToReleases[n]; !ok {
+					id := ReleaseToID(&r.ReleaseSpec)
+					return nil, fmt.Errorf("%q depends on nonexistent release %q", id, n)
+				}
+			}
+		}
+	}
+
+	plan, err := d.Plan()
+	if err != nil {
+		return nil, err
+	}
+
+	var result [][]Release
+
+	for groupIndex := 0; groupIndex < len(plan); groupIndex++ {
+		dagNodesInGroup := plan[groupIndex]
+
+		var idsInGroup []string
+		var releasesInGroup []Release
+
+		for _, node := range dagNodesInGroup {
+			idsInGroup = append(idsInGroup, node.Id)
+		}
+
+		// Make the helmfile behavior deterministic for reproducibility and ease of testing
+		// We try to keep the order of definitions to keep backward-compatibility
+		// See https://github.com/roboll/helmfile/issues/988
+		sort.Slice(idsInGroup, func(i, j int) bool {
+			ii := idToIndex[idsInGroup[i]]
+			ij := idToIndex[idsInGroup[j]]
+			return ii < ij
+		})
+
+		for _, id := range idsInGroup {
+			releases, ok := idToReleases[id]
+			if !ok {
+				panic(fmt.Errorf("bug: unexpectedly failed to get releases for id %q", id))
+			}
+			releasesInGroup = append(releasesInGroup, releases...)
+		}
+
+		result = append(result, releasesInGroup)
+	}
+
+	return result, nil
+}

pkg/state/testdata/issues/1168/addons.yaml

@@ -0,0 +1,7 @@
+addons:
+  mychart:
+    skip: true
+    name: mychart
+    namespace: kube-system
+    chart: stable/mychart
+    version: 1.0.0

pkg/state/testdata/issues/1168/addons2.yaml

@@ -0,0 +1,3 @@
+addons:
+  mychart:
+    skip: false

pkg/state/testdata/values.1.yaml

@@ -0,0 +1,2 @@
+components:
+  etcd-operator:

pkg/state/testdata/values.2.yaml

@@ -0,0 +1,3 @@
+components:
+  etcd-operator:
+    version: 0.10.3

pkg/state/testdata/values.3.yaml

@@ -0,0 +1,3 @@
+components:
+  etcd-operator:
+    version: 0.10.0

pkg/state/testdata/values.4.yaml

@@ -0,0 +1,4 @@
+components:
+  etcd-operator:
+    version: 0.10.3
+  prometheus:

pkg/state/testdata/values.5.yaml

@@ -0,0 +1 @@
+affinity: {}

pkg/testhelper/diff.go

@@ -0,0 +1,103 @@
+package testhelper
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math"
+	"strings"
+
+	"github.com/aryann/difflib"
+)
+
+func Diff(want, got string, context int) (string, bool) {
+	records := difflib.Diff(
+		strings.Split(want, "\n"),
+		strings.Split(got, "\n"),
+	)
+
+	w := &bytes.Buffer{}
+
+	changed := checkAndPrintRecords(w, records, context)
+
+	return w.String(), changed
+}
+
+func checkAndPrintRecords(w io.Writer, records []difflib.DiffRecord, context int) bool {
+	var changed bool
+	if context >= 0 {
+		distances := calculateDistances(records)
+		omitting := false
+		for i, diff := range records {
+			if diff.Delta != difflib.Common {
+				changed = true
+			}
+			if distances[i] > context {
+				if !omitting {
+					fmt.Fprintln(w, "...")
+					omitting = true
+				}
+			} else {
+				omitting = false
+				fmt.Fprintln(w, formatRecord(diff))
+			}
+		}
+	} else {
+		for _, diff := range records {
+			if diff.Delta != difflib.Common {
+				changed = true
+			}
+			fmt.Fprintln(w, formatRecord(diff))
+		}
+	}
+	return changed
+}
+
+func formatRecord(diff difflib.DiffRecord) string {
+	var prefix string
+	switch diff.Delta {
+	case difflib.RightOnly:
+		prefix = "+ "
+	case difflib.LeftOnly:
+		prefix = "- "
+	case difflib.Common:
+		prefix = "  "
+	}
+
+	return prefix + diff.Payload
+}
+
+// Shamelessly and thankfully copied from https://github.com/databus23/helm-diff/blob/99b8474af7726ca6f57b37b0b8b8f3cd36c991e8/diff/diff.go#L116
+func calculateDistances(diffs []difflib.DiffRecord) map[int]int {
+	distances := map[int]int{}
+
+	// Iterate forwards through diffs, set 'distance' based on closest 'change' before this line
+	change := -1
+	for i, diff := range diffs {
+		if diff.Delta != difflib.Common {
+			change = i
+		}
+		distance := math.MaxInt32
+		if change != -1 {
+			distance = i - change
+		}
+		distances[i] = distance
+	}
+
+	// Iterate backwards through diffs, reduce 'distance' based on closest 'change' after this line
+	change = -1
+	for i := len(diffs) - 1; i >= 0; i-- {
+		diff := diffs[i]
+		if diff.Delta != difflib.Common {
+			change = i
+		}
+		if change != -1 {
+			distance := change - i
+			if distance < distances[i] {
+				distances[i] = distance
+			}
+		}
+	}
+
+	return distances
+}

pkg/tmpl/context_funcs.go

@@ -2,6 +2,7 @@ package tmpl
 
 import (
 	"fmt"
+	"golang.org/x/sync/errgroup"
 	"gopkg.in/yaml.v2"
 	"io"
 	"os"
@@ -16,15 +17,18 @@ type Values = map[string]interface{}
 
 func (c *Context) createFuncMap() template.FuncMap {
 	funcMap := template.FuncMap{
-		"exec":           c.Exec,
-		"readFile":       c.ReadFile,
-		"toYaml":         ToYaml,
-		"fromYaml":       FromYaml,
-		"setValueAtPath": SetValueAtPath,
-		"requiredEnv":    RequiredEnv,
-		"get":            get,
-		"getOrNil":       getOrNil,
-		"tpl":            c.Tpl,
+		"exec":             c.Exec,
+		"readFile":         c.ReadFile,
+		"toYaml":           ToYaml,
+		"fromYaml":         FromYaml,
+		"setValueAtPath":   SetValueAtPath,
+		"requiredEnv":      RequiredEnv,
+		"get":              get,
+		"getOrNil":         getOrNil,
+		"tpl":              c.Tpl,
+		"required":         Required,
+		"fetchSecretValue": fetchSecretValue,
+		"expandSecretRefs": fetchSecretValues,
 	}
 	if c.preRender {
 		// disable potential side-effect template calls
@@ -58,60 +62,95 @@ func (c *Context) Exec(command string, args []interface{}, inputs ...string) (st
 	cmd := exec.Command(command, strArgs...)
 	cmd.Dir = c.basePath
 
-	writeErrs := make(chan error)
-	cmdErrs := make(chan error)
-	cmdOuts := make(chan []byte)
+	g := errgroup.Group{}
 
 	if len(input) > 0 {
 		stdin, err := cmd.StdinPipe()
 		if err != nil {
 			return "", err
 		}
-		go func(input string, stdin io.WriteCloser) {
+
+		g.Go(func() error {
 			defer stdin.Close()
-			defer close(writeErrs)
 
 			size := len(input)
 
-			var n int
-			var err error
 			i := 0
+
 			for {
-				n, err = io.WriteString(stdin, input[i:])
+				n, err := io.WriteString(stdin, input[i:])
 				if err != nil {
-					writeErrs <- fmt.Errorf("failed while writing %d bytes to stdin of \"%s\": %v", len(input), command, err)
-					break
+					return fmt.Errorf("failed while writing %d bytes to stdin of \"%s\": %v", len(input), command, err)
 				}
+
 				i += n
-				if n == size {
-					break
+
+				if i == size {
+					return nil
 				}
 			}
-		}(input, stdin)
+		})
 	}
 
-	go func() {
-		defer close(cmdOuts)
-		defer close(cmdErrs)
+	var bytes []byte
 
-		bytes, err := cmd.Output()
+	g.Go(func() error {
+		// We use CombinedOutput to produce helpful error messages
+		// See https://github.com/roboll/helmfile/issues/1158
+		bs, err := cmd.CombinedOutput()
 		if err != nil {
-			cmdErrs <- fmt.Errorf("exec cmd=%s args=[%s] failed: %v", command, strings.Join(strArgs, ", "), err)
-		} else {
-			cmdOuts <- bytes
-		}
-	}()
+			args := strings.Join(strArgs, ", ")
+			shownCmd := []string{command}
+			if len(args) > 0 {
+				shownCmd = append(shownCmd, args)
+			}
 
-	for {
-		select {
-		case bytes := <-cmdOuts:
-			return string(bytes), nil
-		case err := <-cmdErrs:
-			return "", err
-		case err := <-writeErrs:
-			return "", err
+			var out string
+
+			out += fmt.Sprintf("\n\nCOMMAND:\n%s", Indent(strings.Join(shownCmd, " "), "  "))
+
+			out += fmt.Sprintf("\n\nERROR:\n%s", Indent(err.Error(), "  "))
+
+			if len(bs) > 0 {
+				out += fmt.Sprintf("\n\nCOMBINED OUTPUT:\n%s", Indent(string(bs), "  "))
+			}
+
+			return fmt.Errorf("%v%s", err, out)
+		}
+
+		bytes = bs
+
+		return nil
+	})
+
+	if err := g.Wait(); err != nil {
+		return "", err
+	}
+
+	return string(bytes), nil
+}
+
+// indents a block of text with an indent string
+func Indent(text, indent string) string {
+	var b strings.Builder
+
+	b.Grow(len(text) * 2)
+
+	lines := strings.Split(text, "\n")
+
+	last := len(lines) - 1
+
+	for i, j := range lines {
+		if i > 0 && i < last && j != "" {
+			b.WriteString("\n")
+		}
+
+		if j != "" {
+			b.WriteString(indent + j)
 		}
 	}
+
+	return b.String()
 }
 
 func (c *Context) ReadFile(filename string) (string, error) {
@@ -206,3 +245,15 @@ func RequiredEnv(name string) (string, error) {
 
 	return "", fmt.Errorf("required env var `%s` is not set", name)
 }
+
+func Required(warn string, val interface{}) (interface{}, error) {
+	if val == nil {
+		return nil, fmt.Errorf(warn)
+	} else if _, ok := val.(string); ok {
+		if val == "" {
+			return nil, fmt.Errorf(warn)
+		}
+	}
+
+	return val, nil
+}

pkg/tmpl/context_funcs_test.go

@@ -133,3 +133,47 @@ func TestTpl(t *testing.T) {
 		t.Errorf("unexpected result: expected=%v, actual=%v", expected, actual)
 	}
 }
+
+func TestRequired(t *testing.T) {
+	type args struct {
+		warn string
+		val  interface{}
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    interface{}
+		wantErr bool
+	}{
+		{
+			name:    "required val is nil",
+			args:    args{warn: "This value is required", val: nil},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "required val is empty string",
+			args:    args{warn: "This value is required", val: ""},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "required val is existed",
+			args:    args{warn: "This value is required", val: "foo"},
+			want:    "foo",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := Required(tt.args.warn, tt.args.val)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Required() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Required() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/tmpl/context_tmpl.go

@@ -6,7 +6,7 @@ import (
 	"text/template"
 )
 
-func (c *Context) stringTemplate() *template.Template {
+func (c *Context) newTemplate() *template.Template {
 	funcMap := sprig.TxtFuncMap()
 	for name, f := range c.createFuncMap() {
 		funcMap[name] = f
@@ -21,7 +21,7 @@ func (c *Context) stringTemplate() *template.Template {
 }
 
 func (c *Context) RenderTemplateToBuffer(s string, data ...interface{}) (*bytes.Buffer, error) {
-	var t, parseErr = c.stringTemplate().Parse(s)
+	var t, parseErr = c.newTemplate().Parse(s)
 	if parseErr != nil {
 		return nil, parseErr
 	}

pkg/tmpl/context_tmpl_test.go

@@ -250,3 +250,81 @@ func Test_renderTemplateToString(t *testing.T) {
 		})
 	}
 }
+
+func TestRenderTemplate_Required(t *testing.T) {
+	tests := []struct {
+		name    string
+		s       string
+		data    map[string]interface{}
+		want    string
+		wantErr bool
+	}{
+		{
+			name: ".foo is existed",
+			s:    `{{ required ".foo.bar is required" .foo }}`,
+			data: map[string]interface{}{
+				"foo": "bar",
+			},
+			want:    "bar",
+			wantErr: false,
+		},
+		{
+			name: ".foo.bar is existed",
+			s:    `{{ required "foo.bar is required" .foo.bar }}`,
+			data: map[string]interface{}{
+				"foo": map[string]interface{}{
+					"bar": "FOO_BAR",
+				},
+			},
+			want:    "FOO_BAR",
+			wantErr: false,
+		},
+		{
+			name: ".foo.bar is existed but value is nil",
+			s:    `{{ required "foo.bar is required" .foo.bar }}`,
+			data: map[string]interface{}{
+				"foo": map[string]interface{}{
+					"bar": nil,
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: ".foo.bar is existed but value is empty string",
+			s:    `{{ required "foo.bar is required" .foo.bar }}`,
+			data: map[string]interface{}{
+				"foo": map[string]interface{}{
+					"bar": "",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: ".foo is nil",
+			s:    `{{ required "foo is required" .foo }}`,
+			data: map[string]interface{}{
+				"foo": nil,
+			},
+			wantErr: true,
+		},
+		{
+			name: ".foo is a empty string",
+			s:    `{{ required "foo is required" .foo }}`,
+			data: map[string]interface{}{
+				"foo": "",
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		got, err := renderTemplateToString(tt.s, tt.data)
+		if (err != nil) != tt.wantErr {
+			t.Errorf("renderTemplateToString() for %s error = %v, wantErr %v", tt.name, err, tt.wantErr)
+			return
+		}
+		if got != tt.want {
+			t.Errorf("renderTemplateToString() for %s = %v, want %v", tt.name, got, tt.want)
+		}
+	}
+}

pkg/tmpl/expand_secret_ref.go

@@ -0,0 +1,63 @@
+package tmpl
+
+import (
+	"errors"
+	"fmt"
+	"github.com/roboll/helmfile/pkg/plugins"
+	"github.com/variantdev/vals"
+	"sync"
+)
+
+//to generate mock run mockgen -source=expand_secret_ref.go -destination=expand_secrets_mock.go -package=tmpl
+type valClient interface {
+	Eval(template map[string]interface{}) (map[string]interface{}, error)
+}
+
+var once sync.Once
+var secretsClient valClient
+
+func fetchSecretValue(path string) (string, error) {
+	tmpMap := make(map[string]interface{})
+	tmpMap["key"] = path
+	resultMap, err := fetchSecretValues(tmpMap)
+	if err != nil {
+		return "", err
+	}
+
+	rendered, ok := resultMap["key"]
+	if !ok {
+		return "", errors.New(fmt.Sprintf("unexpected error occurred, %v doesn't have 'key' key", resultMap))
+	}
+
+	result, ok := rendered.(string)
+	if !ok {
+		return "", errors.New(fmt.Sprintf("expected %v to be string", rendered))
+	}
+
+	return result, nil
+}
+
+func fetchSecretValues(values map[string]interface{}) (map[string]interface{}, error) {
+	var err error
+	// below lines are for tests
+	once.Do(func() {
+		var valRuntime *vals.Runtime
+		if secretsClient == nil {
+			valRuntime, err = plugins.ValsInstance()
+			if err != nil {
+				return
+			}
+			secretsClient = valRuntime
+		}
+	})
+	if secretsClient == nil {
+		return nil, err
+	}
+
+	resultMap, err := secretsClient.Eval(values)
+	if err != nil {
+		return nil, err
+	}
+
+	return resultMap, nil
+}

pkg/tmpl/expand_secret_ref_test.go

@@ -0,0 +1,78 @@
+package tmpl
+
+import (
+	"errors"
+	"github.com/golang/mock/gomock"
+	"gotest.tools/assert"
+	"testing"
+)
+
+func Test_fetchSecretValue(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+	c := NewMockvalClient(controller)
+	secretsClient = c
+
+	secretPath := "ref+vault://key/#path"
+	expectArg := make(map[string]interface{})
+	expectArg["key"] = secretPath
+
+	valsResult := make(map[string]interface{})
+	valsResult["key"] = "key_value"
+	c.EXPECT().Eval(expectArg).Return(valsResult, nil)
+	result, err := fetchSecretValue(secretPath)
+	assert.NilError(t, err)
+	assert.Equal(t, result, "key_value")
+}
+
+func Test_fetchSecretValue_error(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+	c := NewMockvalClient(controller)
+	secretsClient = c
+
+	secretPath := "ref+vault://key/#path"
+	expectArg := make(map[string]interface{})
+	expectArg["key"] = secretPath
+
+	expectedErr := errors.New("some error")
+	c.EXPECT().Eval(expectArg).Return(nil, expectedErr)
+	result, err := fetchSecretValue(secretPath)
+	assert.Equal(t, err, expectedErr)
+	assert.Equal(t, result, "")
+}
+
+func Test_fetchSecretValue_no_key(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+	c := NewMockvalClient(controller)
+	secretsClient = c
+
+	secretPath := "ref+vault://key/#path"
+	expectArg := make(map[string]interface{})
+	expectArg["key"] = secretPath
+
+	valsResult := make(map[string]interface{})
+	c.EXPECT().Eval(expectArg).Return(valsResult, nil)
+	result, err := fetchSecretValue(secretPath)
+	assert.Error(t, err, "unexpected error occurred, map[] doesn't have 'key' key")
+	assert.Equal(t, result, "")
+}
+
+func Test_fetchSecretValue_invalid_type(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+	c := NewMockvalClient(controller)
+	secretsClient = c
+
+	secretPath := "ref+vault://key/#path"
+	expectArg := make(map[string]interface{})
+	expectArg["key"] = secretPath
+
+	valsResult := make(map[string]interface{})
+	valsResult["key"] = 10
+	c.EXPECT().Eval(expectArg).Return(valsResult, nil)
+	result, err := fetchSecretValue(secretPath)
+	assert.Error(t, err, "expected 10 to be string")
+	assert.Equal(t, result, "")
+}

pkg/tmpl/expand_secrets_mock.go

@@ -0,0 +1,45 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: expand_secret_ref.go
+
+package tmpl
+
+import (
+	gomock "github.com/golang/mock/gomock"
+	reflect "reflect"
+)
+
+// MockvalClient is a mock of valClient interface
+type MockvalClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockvalClientMockRecorder
+}
+
+// MockvalClientMockRecorder is the mock recorder for MockvalClient
+type MockvalClientMockRecorder struct {
+	mock *MockvalClient
+}
+
+// NewMockvalClient creates a new mock instance
+func NewMockvalClient(ctrl *gomock.Controller) *MockvalClient {
+	mock := &MockvalClient{ctrl: ctrl}
+	mock.recorder = &MockvalClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (_m *MockvalClient) EXPECT() *MockvalClientMockRecorder {
+	return _m.recorder
+}
+
+// Eval mocks base method
+func (_m *MockvalClient) Eval(template map[string]interface{}) (map[string]interface{}, error) {
+	ret := _m.ctrl.Call(_m, "Eval", template)
+	ret0, _ := ret[0].(map[string]interface{})
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Eval indicates an expected call of Eval
+func (_mr *MockvalClientMockRecorder) Eval(arg0 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Eval", reflect.TypeOf((*MockvalClient)(nil).Eval), arg0)
+}

test/integration/run.sh

@@ -45,9 +45,15 @@ function retry() {
 
 set -e
 info "Using namespace: ${test_ns}"
-info "Using Helm version: $(helm version --short --client | grep -o v.*$)"
-${helm} init --wait --override spec.template.spec.automountServiceAccountToken=true
-${helm} plugin install https://github.com/databus23/helm-diff --version master
+# helm v2
+if helm version --client 2>/dev/null | grep '"v2\.'; then
+  info "Using Helm version: $(helm version --short --client | grep -o v.*$)"
+  ${helm} init --wait --override spec.template.spec.automountServiceAccountToken=true
+# helm v3
+else
+  info "Using Helm version: $(helm version --short | grep -o v.*$)"
+fi
+${helm} plugin install https://github.com/databus23/helm-diff --version v3.0.0-rc.7
 ${kubectl} get namespace ${test_ns} &> /dev/null && warn "Namespace ${test_ns} exists, from a previous test run?"
 $kubectl create namespace ${test_ns} || fail "Could not create namespace ${test_ns}"
 trap "{ $kubectl delete namespace ${test_ns}; }" EXIT # remove namespace whenever we exit this script
@@ -71,6 +77,9 @@ ${helmfile} -f ${dir}/happypath.yaml template
 code=$?
 [ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile template: ${code}"
 
+info "Applying ${dir}/happypath.yaml"
+bash -c "${helmfile} -f ${dir}/happypath.yaml apply --detailed-exitcode; code="'$?'"; echo Code: "'$code'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile apply"
+
 info "Syncing ${dir}/happypath.yaml"
 ${helmfile} -f ${dir}/happypath.yaml sync
 wait_deploy_ready httpbin-httpbin
@@ -78,9 +87,9 @@ retry 5 "curl --fail $(minikube service --url --namespace=${test_ns} httpbin-htt
 [ ${retry_result} -eq 0 ] || fail "httpbin failed to return 200 OK"
 
 info "Applying ${dir}/happypath.yaml"
-${helmfile} -f ${dir}/happypath.yaml apply
+${helmfile} -f ${dir}/happypath.yaml apply --detailed-exitcode
 code=$?
-[ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile apply: ${code}"
+[ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile apply: want 0, got ${code}"
 
 info "Locking dependencies"
 ${helmfile} -f ${dir}/happypath.yaml deps
@@ -91,6 +100,7 @@ info "Applying ${dir}/happypath.yaml with locked dependencies"
 ${helmfile} -f ${dir}/happypath.yaml apply
 code=$?
 [ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile apply: ${code}"
+${helm} list --namespace=${test_ns} || fail "unable to list releases"
 
 info "Deleting release"
 ${helmfile} -f ${dir}/happypath.yaml delete