Log "could not deduce environment: block" message at debug level (#967)

* Allow combining valuesTemplate and values, same for setTemplate

* Add testcase for valuesTemplate and setTemplate

.circleci/config.yml

@@ -4,7 +4,7 @@ jobs:
 
   build:
     docker:
-      - image: circleci/golang:1.12.4
+      - image: circleci/golang:1.13.4
     environment:
       GOPATH: /go
     working_directory: /home/circleci/workspace/helmfile
@@ -24,12 +24,45 @@ jobs:
           root: ./
           paths:
             - .
+  image:
+    docker:
+    - image: circleci/golang:1.13.4
+    working_directory: /home/circleci/workspace/helmfile
+    steps:
+    - checkout
+    - restore_cache:
+        keys:
+        - go-mod-cache-v1-{{ checksum "./go.sum" }}
+        - go-mod-cache-v1-
+    - setup_remote_docker
+    - run:
+        command: |
+          make tools
+          go mod vendor
+          ORG=roboll BUILD_URL="$CIRCLE_BUILD_URL" make image
+
+  image_helm3:
+    docker:
+    - image: circleci/golang:1.13.4
+    working_directory: /home/circleci/workspace/helmfile
+    steps:
+    - checkout
+    - restore_cache:
+        keys:
+        - go-mod-cache-v1-{{ checksum "./go.sum" }}
+        - go-mod-cache-v1-
+    - setup_remote_docker
+    - run:
+        command: |
+          make tools
+          go mod vendor
+          ORG=roboll BUILD_URL="$CIRCLE_BUILD_URL" make image/helm3
 
   test:
     environment:
       GOPATH: /go
     docker:
-      - image: circleci/golang:1.12.4
+      - image: circleci/golang:1.13.4
     working_directory: /home/circleci/workspace/helmfile
     steps:
       - checkout
@@ -58,7 +91,7 @@ jobs:
       - run:
           name: Install helm
           environment:
-            HELM_VERSION: v2.13.0
+            HELM_VERSION: v2.15.2
           command: |
             HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
             curl -Lo ${HELM_FILENAME} "https://kubernetes-helm.storage.googleapis.com/${HELM_FILENAME}"
@@ -87,11 +120,56 @@ jobs:
             export TERM=xterm
             make integration
 
+  integration_tests_helm3:
+    machine:
+      image: circleci/classic:201808-01
+    environment:
+      CHANGE_MINIKUBE_NONE_USER: true
+      MINIKUBE_WANTUPDATENOTIFICATION: false
+      MINIKUBE_WANTREPORTERRORPROMPT: false
+    steps:
+    - checkout
+    - run: mkdir ~/build
+    - attach_workspace:
+        at: ~/build
+    - run: cp ~/build/helmfile ~/project/helmfile
+    - run:
+        name: Install helm
+        environment:
+          HELM_VERSION: v3.0.0
+        command: |
+          HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
+          curl -Lo ${HELM_FILENAME} "https://get.helm.sh/${HELM_FILENAME}"
+          tar zxf ${HELM_FILENAME} linux-amd64/helm
+          chmod +x linux-amd64/helm
+          sudo mv linux-amd64/helm /usr/local/bin/
+    - run:
+        name: Deploy minikube
+        environment:
+          CHANGE_MINIKUBE_NONE_USER: true
+          K8S_VERSION: v1.12.3
+          MINIKUBE_VERSION: v0.30.0
+        command: |
+          curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl
+          chmod +x kubectl && sudo mv kubectl /usr/local/bin/
+          curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
+          chmod +x minikube && sudo mv minikube /usr/local/bin/
+          sudo -E minikube start --vm-driver=none --kubernetes-version=${K8S_VERSION}
+          sudo -E minikube update-context
+    - run:
+        name: Wait for nodes to become ready
+        command: JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done
+    - run:
+        name: Execute integration tests
+        command: |
+          export TERM=xterm
+          HELMFILE_HELM3=1 make integration
+
 # GITHUB_TOKEN env var must be setup in circleci console
 
   release:
     docker:
-      - image: circleci/golang:1.12.4
+      - image: circleci/golang:1.13.4
     working_directory: /home/circleci/workspace/helmfile
     steps:
     - checkout
@@ -105,22 +183,22 @@ jobs:
           docker login -u="$DOCKER_USER" -p="$DOCKER_PASS" quay.io
           make tools
           go mod vendor
-          ORG=roboll BUILD_URL="$CIRCLE_BUILD_URL" make push release
+          ORG=roboll BUILD_URL="$CIRCLE_BUILD_URL" make push push/helm3 release
 
 workflows:
   version: 2
   build_and_test:
     jobs:
       - build
+      - image
+      - image_helm3
       - test
       - integration_tests:
           requires:
             - build
-          filters:
-            branches:
-              only:
-                - master
-                - /pull.*/
+      - integration_tests_helm3:
+          requires:
+            - build
       - release:
           filters:
             branches:

.gitignore

@@ -1,3 +1,5 @@
 dist/
 .idea/
-helmfile
+helmfile
+helmfile.lock
+vendor/

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.12.4-alpine3.9 as builder
+FROM golang:1.13.4-alpine3.10 as builder
 
 RUN apk add --no-cache make git
 WORKDIR /workspace/helmfile
@@ -7,22 +7,37 @@ RUN make static-linux
 
 # -----------------------------------------------------------------------------
 
-FROM alpine:3.8
+FROM alpine:3.10
 
-RUN apk add --no-cache ca-certificates git bash curl
+RUN apk add --no-cache ca-certificates git bash curl jq
 
-ARG HELM_VERSION=v2.13.0
+ARG HELM_VERSION=v2.15.2
 ARG HELM_LOCATION="https://kubernetes-helm.storage.googleapis.com"
 ARG HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
-ARG HELM_SHA256="15eca6ad225a8279de80c7ced42305e24bc5ac60bb7d96f2d2fa4af86e02c794"
+ARG HELM_SHA256="a9d2db920bd4b3d824729bbe1ff3fa57ad27760487581af6e5d3156d1b3c2511"
 RUN wget ${HELM_LOCATION}/${HELM_FILENAME} && \
+    echo Verifying ${HELM_FILENAME}... && \
     sha256sum ${HELM_FILENAME} | grep -q "${HELM_SHA256}" && \
-    tar zxf ${HELM_FILENAME} && mv /linux-amd64/helm /usr/local/bin/ && \
+    echo Extracting ${HELM_FILENAME}... && \
+    tar zxvf ${HELM_FILENAME} && mv /linux-amd64/helm /usr/local/bin/ && \
     rm ${HELM_FILENAME} && rm -r /linux-amd64
 
-RUN mkdir -p "$(helm home)/plugins"
+# using the install documentation found at https://kubernetes.io/docs/tasks/tools/install-kubectl/
+# for now but in a future version of alpine (in the testing version at the time of writing)
+# we should be able to install using apk add.
+ENV KUBECTL_VERSION="v1.14.5"
+ENV KUBECTL_SHA256="26681319de56820a8467c9407e9203d5b15fb010ffc75ac5b99c9945ad0bd28c"
+RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && \
+    sha256sum kubectl | grep ${KUBECTL_SHA256} && \
+    chmod +x kubectl && \
+    mv kubectl /usr/local/bin/kubectl
+
+RUN ["helm", "init", "--client-only"]
 RUN helm plugin install https://github.com/databus23/helm-diff && \
-    helm plugin install https://github.com/futuresimple/helm-secrets
+    helm plugin install https://github.com/futuresimple/helm-secrets && \
+    helm plugin install https://github.com/hypnoglow/helm-s3.git && \
+    helm plugin install https://github.com/aslafy-z/helm-git.git && \
+    helm plugin install https://github.com/rimusz/helm-tiller
 
 COPY --from=builder /workspace/helmfile/dist/helmfile_linux_amd64 /usr/local/bin/helmfile
 

Dockerfile.helm3

@@ -0,0 +1,45 @@
+FROM golang:1.13.4-alpine3.10 as builder
+
+RUN apk add --no-cache make git
+WORKDIR /workspace/helmfile
+COPY . /workspace/helmfile
+RUN make static-linux
+
+# -----------------------------------------------------------------------------
+
+FROM alpine:3.10
+
+RUN apk add --no-cache ca-certificates git bash curl jq
+
+ARG HELM_VERSION=v3.0.0
+ARG HELM_LOCATION="https://get.helm.sh"
+ARG HELM_FILENAME="helm-${HELM_VERSION}-linux-amd64.tar.gz"
+ARG HELM_SHA256="10e1fdcca263062b1d7b2cb93a924be1ef3dd6c381263d8151dd1a20a3d8c0dc"
+RUN wget ${HELM_LOCATION}/${HELM_FILENAME} && \
+    echo Verifying ${HELM_FILENAME}... && \
+    sha256sum ${HELM_FILENAME} | grep -q "${HELM_SHA256}" && \
+    echo Extracting ${HELM_FILENAME}... && \
+    tar zxvf ${HELM_FILENAME} && mv /linux-amd64/helm /usr/local/bin/ && \
+    rm ${HELM_FILENAME} && rm -r /linux-amd64
+
+# using the install documentation found at https://kubernetes.io/docs/tasks/tools/install-kubectl/
+# for now but in a future version of alpine (in the testing version at the time of writing)
+# we should be able to install using apk add.
+ENV KUBECTL_VERSION="v1.14.5"
+ENV KUBECTL_SHA256="26681319de56820a8467c9407e9203d5b15fb010ffc75ac5b99c9945ad0bd28c"
+RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && \
+    sha256sum kubectl | grep ${KUBECTL_SHA256} && \
+    chmod +x kubectl && \
+    mv kubectl /usr/local/bin/kubectl
+
+RUN helm plugin install https://github.com/databus23/helm-diff && \
+    helm plugin install https://github.com/futuresimple/helm-secrets && \
+    helm plugin install https://github.com/hypnoglow/helm-s3.git && \
+    helm plugin install https://github.com/aslafy-z/helm-git.git && \
+    helm plugin install https://github.com/rimusz/helm-tiller
+
+COPY --from=builder /workspace/helmfile/dist/helmfile_linux_amd64 /usr/local/bin/helmfile
+
+ENV HELMFILE_HELM3="1"
+
+CMD ["/usr/local/bin/helmfile"]

Makefile

@@ -26,12 +26,12 @@ integration:
 .PHONY: integration
 
 cross:
-	env CGO_ENABLED=0 gox -os '!freebsd !netbsd' -arch '!arm' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS}
+	env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le !s390x' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS}
 .PHONY: cross
 
 static-linux:
-	env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "dist/helmfile_linux_amd64" -ldflags '-X main.Version=${TAG}' ${TARGETS}
-.PHONY: linux
+	env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -o "dist/helmfile_linux_amd64" -ldflags '-X main.Version=${TAG}' ${TARGETS}
+.PHONY: static-linux
 
 install:
 	env CGO_ENABLED=0 go install -ldflags '-X main.Version=${TAG}' ${TARGETS}
@@ -59,6 +59,13 @@ run: image
 push: image
 	docker push quay.io/${ORG}/helmfile:${TAG}
 
+
+image/helm3:
+	docker build -f Dockerfile.helm3 -t quay.io/${ORG}/helmfile:helm3-${TAG} .
+
+push/helm3: image/helm3
+	docker push quay.io/${ORG}/helmfile:helm3-${TAG}
+
 tools:
 	go get -u github.com/tcnksm/ghr github.com/mitchellh/gox
 .PHONY: tools

README.md

@@ -1,19 +1,19 @@
-# helmfile [![CircleCI](https://circleci.com/gh/roboll/helmfile.svg?style=svg)](https://circleci.com/gh/roboll/helmfile)
+# Helmfile [![CircleCI](https://circleci.com/gh/roboll/helmfile.svg?style=svg)](https://circleci.com/gh/roboll/helmfile)
 
 Deploy Kubernetes Helm Charts
 
 [![Docker Repository on Quay](https://quay.io/repository/roboll/helmfile/status "Docker Repository on Quay")](https://quay.io/repository/roboll/helmfile)
 [![Slack Community #helmfile](https://slack.sweetops.com/badge.svg)](https://slack.sweetops.com)
 
-## status
+## Status
 
 Even though Helmfile is used in production environments [across multiple organizations](USERS.md), it is still in its early stage of development, hence versioned 0.x.
 
 Helmfile complies to Semantic Versioning 2.0.0 in which v0.x means that there could be backward-incompatible changes for every release.
 
-Note that we will try our best to document any backward incompatibility.
+Note that we will try our best to document any backward incompatibility. And in reality, helmfile had no breaking change for a year or so.
 
-## about
+## About
 
 Helmfile is a declarative spec for deploying helm charts. It lets you...
 
@@ -23,20 +23,49 @@ Helmfile is a declarative spec for deploying helm charts. It lets you...
 
 To avoid upgrades for each iteration of `helm`, the `helmfile` executable delegates to `helm` - as a result, `helm` must be installed.
 
-## configuration syntax
+## Highlights
 
-**CAUTION**: This documentation is for the development version of Helmfile. If you are looking for the documentation for any of releases, please switch to the corresponding release tag like [v0.31.0](https://github.com/roboll/helmfile/tree/v0.31.0).
+**Declarative**: Write, version-control, apply the desired state file for visibility and reproducibility.
+
+**Modules**: Modularize common patterns of your infrastructure, distribute it via Git, S3, etc. to be reused across the entire company (See [#648](https://github.com/roboll/helmfile/pull/648))
+
+**Versatility**: Manage your cluster consisting of charts, [kustomizations](https://github.com/kubernetes-sigs/kustomize), and directories of Kubernetes resources, turning everything to Helm releases (See [#673](https://github.com/roboll/helmfile/pull/673))
+
+**Patch**: JSON/Strategic-Merge Patch Kubernetes resources before `helm-install`ing, without forking upstream charts (See [#673](https://github.com/roboll/helmfile/pull/673))
+
+## Configuration
+
+**CAUTION**: This documentation is for the development version of Helmfile. If you are looking for the documentation for any of releases, please switch to the corresponding release tag like [v0.79.1](https://github.com/roboll/helmfile/tree/v0.79.1).
 
 The default helmfile is `helmfile.yaml`:
 
 ```yaml
+# Chart repositories used from within this state file
+#
+# Use `helm-s3` and `helm-git` and whatever Helm Downloader plugins
+# to use repositories other than the official repository or one backend by chartmuseum.
 repositories:
-  - name: roboll
-    url: http://roboll.io/charts
-    certFile: optional_client_cert
-    keyFile: optional_client_key
-    username: optional_username
-    password: optional_password
+# To use official "stable" charts a.k.a https://github.com/helm/charts/tree/master/stable
+- name: stable
+  url: https://kubernetes-charts.storage.googleapis.com
+# To use official "incubator" charts a.k.a https://github.com/helm/charts/tree/master/incubator
+- name: incubator
+  url: https://kubernetes-charts-incubator.storage.googleapis.com
+# helm-git powered repository: You can treat any Git repository as a charts repository
+- name: polaris
+  url: git+https://github.com/reactiveops/polaris@deploy/helm?ref=master
+# Advanced configuration: You can setup basic or tls auth
+- name: roboll
+  url: http://roboll.io/charts
+  certFile: optional_client_cert
+  keyFile: optional_client_key
+  username: optional_username
+  password: optional_password
+# Advanced configuration: You can use a ca bundle to use an https repo
+# with a self-signed certificate
+- name: insecure
+   url: https://charts.my-insecure-domain.com
+   caFile: optional_ca_crt
 
 # context: kube-context # this directive is deprecated, please consider using helmDefaults.kubeContext
 
@@ -63,6 +92,9 @@ helmDefaults:
   # path to TLS key file (default "$HELM_HOME/key.pem")
   tlsKey: "path/to/key.pem"
 
+# The desired states of Helm releases.
+#
+# Helmfile runs various helm commands to converge the current state in the live cluster to the desired state defined here.
 releases:
   # Published chart example
   - name: vault                            # name of this release
@@ -71,12 +103,21 @@ releases:
       foo: bar
     chart: roboll/vault-secret-manager     # the chart being installed to create this release, referenced by `repository/chart` syntax
     version: ~1.24.1                       # the semver of the chart. range constraint is supported
-    missingFileHandler: warn # set to either "Error" or "Warn". "Error" instructs helmfile to fail when unable to find a values or secrets file. When "Warn", it prints the file and continues.
+    missingFileHandler: Warn # set to either "Error" or "Warn". "Error" instructs helmfile to fail when unable to find a values or secrets file. When "Warn", it prints the file and continues.
+    # Values files used for rendering the chart
     values:
-      # value files passed via --values
+      # Value files passed via --values
       - vault.yaml
-      # inline values, passed via a temporary values file and --values
+      # Inline values, passed via a temporary values file and --values, so that it doesn't suffer from type issues like --set
       - address: https://vault.example.com
+      # Go template available in inline values and values files.
+      - image:
+          # The end result is more or less YAML. So do `quote` to prevent number-like strings from accidentally parsed into numbers!
+          # See https://github.com/roboll/helmfile/issues/608
+          tag: {{ requiredEnv "IMAGE_TAG" | quote }}
+          # Otherwise:
+          #   tag: "{{ requiredEnv "IMAGE_TAG" }}"
+          #   tag: !!string {{ requiredEnv "IMAGE_TAG" }}
         db:
           username: {{ requiredEnv "DB_USERNAME" }}
           # value taken from environment variable. Quotes are necessary. Will throw an error if the environment variable is not set. $DB_PASSWORD needs to be set in the calling environment ex: export DB_PASSWORD='password1'
@@ -85,6 +126,8 @@ releases:
           # Interpolate environment variable with a fixed string
           domain: {{ requiredEnv "PLATFORM_ID" }}.my-domain.com
           scheme: {{ env "SCHEME" | default "https" }}
+    # Use `values` whenever possible!
+    # `set` translates to helm's `--set key=val`, that is known to suffer from type issues like https://github.com/roboll/helmfile/issues/608
     set:
     # single value loaded from a local file, translates to --set-file foo.config=path/to/file
     - name: foo.config
@@ -100,6 +143,8 @@ releases:
     # will attempt to decrypt it using helm-secrets plugin
     secrets:
       - vault_secret.yaml
+    # verify the chart before upgrading (only works with packaged charts not directories)
+    verify: true
     # wait for k8s resources via --wait. Defaults to `false`
     wait: true
     # time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks, and waits on pod/pvc/svc/deployment readiness) (default 300)
@@ -124,6 +169,10 @@ releases:
     tlsCert: "path/to/cert.pem"
     # path to TLS key file (default "$HELM_HOME/key.pem")
     tlsKey: "path/to/key.pem"
+    # --kube-context to be passed to helm commands
+    # CAUTION: this doesn't work as expected for `tilerless: true`.
+    # See https://github.com/roboll/helmfile/issues/642
+    kubeContext: kube-context
 
   # Local chart example
   - name: grafana                            # name of this release
@@ -134,6 +183,82 @@ releases:
     - ./values/{{ requiredEnv "PLATFORM_ENV" }}/config.yaml # Values file taken from path with environment variable. $PLATFORM_ENV must be set in the calling environment.
     wait: true
 
+#
+# Advanced Configuration: Nested States
+#
+helmfiles:
+- # Path to the helmfile state file being processed BEFORE releases in this state file
+  path: path/to/subhelmfile.yaml
+  # Label selector used for filtering releases in the nested state.
+  # For example, `name=prometheus` in this context is equivalent to processing the nested state like
+  #   helmfile -f path/to/subhelmfile.yaml -l name=prometheus sync
+  selectors:
+  - name=prometheus
+  # Override state values
+  values:
+  # Values files merged into the nested state's values
+  - additional.values.yaml
+  # One important aspect of using values here is that they first need to be defined in the values section
+  # of the origin helmfile, so in this example key1 needs to be in the values or environments.NAME.values of path/to/subhelmfile.yaml
+  # Inline state values merged into the nested state's values
+  - key1: val1
+- # All the nested state files under `helmfiles:` is processed in the order of definition.
+  # So it can be used for preparation for your main `releases`. An example would be creating CRDs required by `releases` in the parent state file.
+  path: path/to/mycrd.helmfile.yaml
+- # Terraform-module-like URL for importing a remote directory and use a file in it as a nested-state file
+  # The nested-state file is locally checked-out along with the remote directory containing it.
+  # Therefore all the local paths in the file are resolved relative to the file
+  path: git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0
+
+#
+# Advanced Configuration: Environments
+#
+
+# The list of environments managed by helmfile.
+#
+# The default is `environments: {"default": {}}` which implies:
+#
+# - `{{ .Environment.Name }}` evaluates to "default"
+# - `{{ .Values }}` being empty
+environments:
+  # The "default" environment is available and used when `helmfile` is run without `--environment NAME`.
+  default:
+    # Everything from the values.yaml is available via `{{ .Values.KEY }}`.
+    # Suppose `{"foo": {"bar": 1}}` contained in the values.yaml below,
+    # `{{ .Values.foo.bar }}` is evaluated to `1`.
+    values:
+    - environments/default/values.yaml
+    # Each entry in values can be either a file path or inline values.
+    # The below is an example of inline values, which is merged to the `.Values`
+    - myChartVer: 1.0.0-dev
+  # Any environment other than `default` is used only when `helmfile` is run with `--environment NAME`.
+  # That is, the "production" env below is used when and only when it is run like `helmfile --environment production sync`.
+  production:
+    values:
+    - environment/production/values.yaml
+    - myChartVer: 1.0.0
+    ## `secrets.yaml` is decrypted by `helm-secrets` and available via `{{ .Environment.Secrets.KEY }}`
+    secrets:
+    - environment/production/secrets.yaml
+    # Instructs helmfile to fail when unable to find a environment values file listed under `environments.NAME.values`.
+    #
+    # Possible values are  "Error", "Warn", "Info", "Debug". The default is "Error".
+    #
+    # Use "Warn", "Info", or "Debug" if you want helmfile to not fail when a values file is missing, while just leaving
+    # a message about the missing file at the log-level.
+    missingFileHandler: Error
+
+#
+# Advanced Configuration: Layering
+#
+# Helmfile merges all the "base" state files and this state file before processing.
+#
+# Assuming this state file is named `helmfile.yaml`, all the files are merged in the order of:
+#   environments.yaml <- defaults.yaml <- templates.yaml <- helmfile.yaml
+bases:
+- environments.yaml
+- defaults.yaml
+- templates.yaml
 ```
 
 ## Templating
@@ -220,9 +345,10 @@ USAGE:
    helmfile [global options] command [command options] [arguments...]
 
 VERSION:
-   v0.52.0
+   v0.70.0
 
 COMMANDS:
+     deps      update charts based on the contents of requirements.yaml
      repos     sync repositories from state file (helm repo add && helm repo update)
      charts    DEPRECATED: sync releases from state file (helm upgrade --install)
      diff      diff releases from state file against env (helm diff)
@@ -239,6 +365,8 @@ GLOBAL OPTIONS:
    --helm-binary value, -b value           path to helm binary
    --file helmfile.yaml, -f helmfile.yaml  load config from file or directory. defaults to helmfile.yaml or `helmfile.d`(means `helmfile.d/*.yaml`) in this preference
    --environment default, -e default       specify the environment name. defaults to default
+   --state-values-set value                set state values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)
+   --state-values-file value               specify state values in a YAML file
    --quiet, -q                             Silence output. Equivalent to log-level warn
    --kube-context value                    Set kubectl context. Uses current context by default
    --log-level value                       Set log level, default info
@@ -247,6 +375,7 @@ GLOBAL OPTIONS:
                                            A release must match all labels in a group in order to be used. Multiple groups can be specified at once.
                                            --selector tier=frontend,tier!=proxy --selector tier=backend. Will match all frontend, non-proxy releases AND all backend releases.
                                            The name of a release can be used as a label. --selector name=myrelease
+   --allow-no-matching-release             Do not exit with an error code if the provided selector has no matching releases.
    --interactive, -i                       Request confirmation before attempting to modify clusters
    --help, -h                              show help
    --version, -v                           print the version
@@ -254,13 +383,27 @@ GLOBAL OPTIONS:
 
 ### sync
 
-The `helmfile sync` sub-command sync your cluster state as described in your `helmfile`. The default helmfile is `helmfile.yaml`, but any yaml file can be passed by specifying a `--file path/to/your/yaml/file` flag.
+The `helmfile sync` sub-command sync your cluster state as described in your `helmfile`. The default helmfile is `helmfile.yaml`, but any YAML file can be passed by specifying a `--file path/to/your/yaml/file` flag.
 
 Under the covers, Helmfile executes `helm upgrade --install` for each `release` declared in the manifest, by optionally decrypting [secrets](#secrets) to be consumed as helm chart values. It also updates specified chart repositories and updates the
 dependencies of any referenced local charts.
 
 For Helm 2.9+ you can use a username and password to authenticate to a remote repository.
 
+### deps
+
+The `helmfile deps` sub-command locks your helmfile state and local charts dependencies.
+
+It basically runs `helm dependency update` on your helmfile state file and all the referenced local charts, so that you get a "lock" file per each helmfile state or local chart.
+
+All the other `helmfile` sub-commands like `sync` use chart versions recorded in the lock files, so that e.g. untested chart versions won't suddenly get deployed to the production environment.
+
+For example, the lock file for a helmfile state file named `helmfile.1.yaml` will be `helmfile.1.lock`. The lock file for a local chart would be `requirements.lock`, which is the same as `helm`.
+
+It is recommended to version-control all the lock files, so that they can be used in the production deployment pipeline for extra reproducibility.
+
+To bring in chart updates systematically, it would also be a good idea to run `helmfile deps` regularly, test it, and then update the lock files in the version-control system.
+
 ### diff
 
 The `helmfile diff` sub-command executes the [helm-diff](https://github.com/databus23/helm-diff) plugin across all of
@@ -292,6 +435,7 @@ The `helmfile delete` sub-command deletes all the releases defined in the manife
 
 Note that `delete` doesn't purge releases. So `helmfile delete && helmfile sync` results in sync failed due to that releases names are not deleted but preserved for future references. If you really want to remove releases for reuse, add `--purge` flag to run it like `helmfile delete --purge`.
 
+
 ### secrets
 
 The `secrets` parameter in a `helmfile.yaml` causes the [helm-secrets](https://github.com/futuresimple/helm-secrets) plugin to be executed to decrypt the file.
@@ -415,10 +559,10 @@ releases:
     namespace: {{ requiredEnv "NAME" }}
     chart: roboll/vault-secret-manager
     values:
-    - values.yaml.tmpl
+    - values.yaml.gotmpl
 ```
 
-`values.yaml.tmpl`:
+`values.yaml.gotmpl`:
 
 ```yaml
 db:
@@ -488,7 +632,7 @@ releaseName: prod
 `values.yaml.gotmpl`
 
 ```yaml
-domain: {{ .Environment.Values | getOrNil "my.domain" | default "dev.example.com" }}
+domain: {{ .Values | getOrNil "my.domain" | default "dev.example.com" }}
 ```
 
 `helmfile sync` installs `myapp` with the value `domain=dev.example.com`,
@@ -515,17 +659,26 @@ environments:
     - other.yaml.gotmpl  #  template directives with potential side-effects like `exec` and `readFile` will be honoured
 
 releases:
-- name: myapp-{{ .Environment.Values.releaseName }} # release name will be one of `dev` or `prod` depending on selected environment
+- name: myapp-{{ .Values.releaseName }} # release name will be one of `dev` or `prod` depending on selected environment
   values:
   - values.yaml.gotmpl
 
-{{ if eq (.Environment.Values.releaseName "prod" ) }}
+{{ if eq (.Values.releaseName "prod" ) }}
 # this release would be installed only if selected environment is `production`
 - name: production-specific-release
   ...
 {{ end }}
 ```
 
+### Note
+
+The `{{ .Values.foo }}` syntax is the recommended way of using environment values.
+
+Prior to this [pull request](https://github.com/roboll/helmfile/pull/647), environment values were made available through the `{{ .Environment.Values.foo }}` syntax.
+This is still working but is **deprecated** and the new `{{ .Values.foo }}` syntax should be used instead.
+
+You can read more infos about the feature proposal [here](https://github.com/roboll/helmfile/issues/640).
+
 ## Environment Secrets
 
 Environment Secrets (not to be confused with Kubernetes Secrets) are encrypted versions of `Environment Values`.
@@ -560,7 +713,7 @@ releases:
 Then the environment secret `foo.bar` can be referenced by the below template expression in your `values.yaml.gotmpl`:
 
 ```yaml
-{{ .Environment.Values.foo.bar }}
+{{ .Values.foo.bar }}
 ```
 
 ## Tillerless
@@ -570,8 +723,49 @@ With the [helm-tiller](https://github.com/rimusz/helm-tiller) plugin installed,
 To enable this mode, you need to define `tillerless: true` and set the `tillerNamespace` in the `helmDefaults` section
 or in the `releases` entries.
 
-Since every commands is run with `helm tiller run ...`, you have to disable concurrency. Otherwise you'll get
-mysterious errors about the tiller daemon.
+## DAG-aware installation/deletion ordering
+
+`needs` controls the order of the installation/deletion of the release:
+
+```yaml
+relesaes:
+- name: somerelease
+  needs:
+  - [TILLER_NAMESPACE/][NAMESPACE/]anotherelease
+```
+
+All the releases listed under `needs` are installed before(or deleted after) the release itself.
+
+For the following example, `helmfile [sync|apply]` installs releases in this order:
+
+1. logging
+2. servicemesh
+3. myapp1 and myapp2
+
+```yaml
+  - name: myapp1
+    chart: charts/myapp
+    needs:
+    - servicemesh
+    - logging
+  - name: myapp2
+    chart: charts/myapp
+    needs:
+    - servicemesh
+    - logging
+  - name: servicemesh
+    chart: charts/istio
+    needs:
+    - logging
+  - name: logging
+    chart: charts/fluentd
+```
+
+Note that all the releases in a same group is installed concurrently. That is, myapp1 and myapp2 are installed concurrently.
+
+On `helmdile [delete|destroy]`, deleations happen in the reverse order.
+
+That is, `myapp1` and `myapp2` are deleted first, then `servicemesh`, and finally `logging`.
 
 ## Separating helmfile.yaml into multiple independent files
 
@@ -644,19 +838,22 @@ Just run `helmfile sync` inside `myteam/`, and you are done.
 All the files are sorted alphabetically per group = array item inside `helmfiles:`, so that you have granular control over ordering, too.
 
 #### selectors
+
 When composing helmfiles you can use selectors from the command line as well as explicit selectors inside the parent helmfile to filter the releases to be used.
+
 ```yaml
 helmfiles:
 - apps/*/helmfile.yaml
 - path: apps/a-helmfile.yaml
-    selectors:          # list of selectors
-    - name=prometheus
-    - tier=frontend
+  selectors:          # list of selectors
+  - name=prometheus
+  - tier=frontend
 - path: apps/b-helmfile.yaml # no selector, so all releases are used
-    selectors: []
+selectors: []
 - path: apps/c-helmfile.yaml # parent selector to be used or cli selector for the initial helmfile
-    selectorsInherited: true
+  selectorsInherited: true
 ```
+
 * When a selector is specified, only this selector applies and the parents or CLI selectors are ignored.
 * When not selector is specified there are 2 modes for the selector inheritance because we would like to change the current inheritance behavior (see [issue #344](https://github.com/roboll/helmfile/issues/344)  ).
   * Legacy mode, sub-helmfiles without selectors inherit selectors from their parent helmfile. The initial helmfiles inherit from the command line selectors.
@@ -692,15 +889,17 @@ A Helmfile hook is a per-release extension point that is composed of:
 - `events`
 - `command`
 - `args`
+- `showlogs`
 
 Helmfile triggers various `events` while it is running.
-Once `events` are triggered, associated `hooks` are executed, by running the `command` with `args`.
+Once `events` are triggered, associated `hooks` are executed, by running the `command` with `args`. The standard output of the `command` will be displayed if `showlogs` is set and it's value is `true`.
 
 Currently supported `events` are:
 
 - `prepare`
-- `cleanup`
 - `presync`
+- `postsync`
+- `cleanup`
 
 Hooks associated to `prepare` events are triggered after each release in your helmfile is loaded from YAML, before execution.
 
@@ -708,6 +907,8 @@ Hooks associated to `cleanup` events are triggered after each release is process
 
 Hooks associated to `presync` events are triggered before each release is applied to the remote cluster. This is the ideal event to execute any commands that may mutate the cluster state as it will not be run for read-only operations like `lint`, `diff` or `template`.
 
+Hooks associated to `postsync` events are triggered after each release is applied to the remote cluster. This is the ideal event to execute any commands that may mutate the cluster state as it will not be run for read-only operations like `lint`, `diff` or `template`.
+
 The following is an example hook that just prints the contextual information provided to hook:
 
 ```
@@ -717,6 +918,7 @@ releases:
   # *snip*
   hooks:
   - events: ["prepare", "cleanup"]
+    showlogs: true
     command: "echo"
     args: ["{{`{{.Environment.Name}}`}}", "{{`{{.Release.Name}}`}}", "{{`{{.HelmfileCommand}}`}}\
 "]
@@ -836,6 +1038,22 @@ Those features are set using the environment variable `HELMFILE_EXPERIMENTAL`. H
 
 If you want to enable all experimental features set the env var to `HELMFILE_EXPERIMENTAL=true`
 
+## Azure ACR integration
+
+Azure offers helm repository [support for Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-helm-repos) as a preview feature.
+
+To use this you must first `az login` and then `az acr helm repo add -n <MyRegistry>`. This will extract a token for the given ACR and configure `helm` to use it, e.g. `helm repo update` should work straight away.
+
+To use `helmfile` with ACR, on the other hand, you must either include a username/password in the repository definition for the ACR in your `helmfile.yaml` or use the `--skip-deps` switch, e.g. `helmfile template --skip-deps`.
+
+An ACR repository definition in `helmfile.yaml` looks like this:
+
+```
+repositories:
+  - name: <MyRegistry>
+    url: https://<MyRegistry>.azurecr.io/helm/v1/repo
+```
+
 ## Examples
 
 For more examples, see the [examples/README.md](https://github.com/roboll/helmfile/blob/master/examples/README.md) or the [`helmfile`](https://github.com/cloudposse/helmfiles/tree/master/releases) distribution by [Cloud Posse](https://github.com/cloudposse/).

USERS.md

@@ -1,18 +1,21 @@
 # Helmfile Users
 
-The following table is compiled from public information or from survey responses. If you are using Helmfile in your
-company or organization, we would like to invite you to [participate in our user survey](https://docs.google.com/forms/d/e/1FAIpQLSfrF5kR9nXsr5yWqNZGgN-KquM1mBTuYNn5lRmEVXEvhq0VIQ/viewform)
-to help us extend this list.
+Who's actually using Helmfile? The following table is compiled from public information, survey response and PRs. It's
+likely this is only a small selection of actual users, but it's a start.
+
+If you are using Helmfile in your company or organization, we would like to invite you to create a PR to add your
+information to this file.
 
 <!-- TABLE_START -->
-| Organization | Workload | More Info | Location |
-|---|---|---|---|
-| [reddit.com](https://www.reddit.com/) | production | [GitHub issue](https://github.com/roboll/helmfile/issues/96#issue-312012249) | San Francisco, CA, US |
-| [ricardo.ch](https://www.ricardo.ch/) | proof-of-concept | | Zug, Switzerland |
+| Organization | Workload | More Info | Location | Added |
+|---|---|---|---|---|
+| [reddit.com](https://www.reddit.com/) | production | [GitHub issue](https://github.com/roboll/helmfile/issues/96#issue-312012249), [Talk](https://www.slideshare.net/GregoryTaylor11/helm-at-reddit-from-local-dev-staging-to-production) | San Francisco, CA | April 2018 |
+| [ricardo.ch](https://www.ricardo.ch/) | production | We're deploying our complete application platform using Helmfile. | Zug, Switzerland | April 2018 |
+| [Sight Machine](https://sightmachine.com/) | production | We don't have anything publicly posted about it, but have been using it for quite a while in production. | San Francisco, CA and Ann Arbor, MI | December 2018 |
+| [VSHN – The DevOps Company](https://vshn.ch) | production	| | Zurich, Switzerland | March 2019 |
+| [Vlocity](https://vlocity.com/) | proof-of-concept | | Melbourne, Australia | March 2019 |
+| [transit](https://transit.app/) | production | [Blog post](https://medium.com/@naseem_60378/helmfile-its-like-a-helm-for-your-helm-74a908581599). | Montreal, Canada | March 2019 |
+| [uniqkey](https://uniqkey.eu/) | production | [Wiki Page](https://ocd-scm.github.io/ocd-meta/) | Copenhagen, Denmark | April 2019 |
+| [bitsofinfo](https://github.com/bitsofinfo/helmfile-deploy) | production | Used with [helmfile-deploy](https://github.com/bitsofinfo/helmfile-deploy) to manage releases for dozens of apps | USA | July 2019 |
+| [kloeckner-i](https://www.kloeckner-i.com/) | production | We are deploying our standard tools via helmfile to all our clusters.  | Berlin, Germany | September 2019 |
 <!-- TABLE_END -->
-
-## Contact
-
-Please fill out [the Google form](https://docs.google.com/forms/d/e/1FAIpQLSfrF5kR9nXsr5yWqNZGgN-KquM1mBTuYNn5lRmEVXEvhq0VIQ/viewform) to add your organization instead of opening a PR.
-
-You can also [reach the maintainer on Twitter](https://twitter.com/DerCed).

cmd/cmd.go

@@ -1,94 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"github.com/roboll/helmfile/helmexec"
-	"github.com/roboll/helmfile/pkg/app"
-	"github.com/roboll/helmfile/state"
-	"github.com/urfave/cli"
-	"go.uber.org/zap"
-	"strings"
-)
-
-func VisitAllDesiredStates(c *cli.Context, converge func(*state.HelmState, helmexec.Interface, app.Context) (bool, []error)) error {
-	a, fileOrDir, err := InitAppEntry(c, false)
-	if err != nil {
-		return err
-	}
-
-	ctx := app.NewContext()
-
-	convergeWithHelmBinary := func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
-		if c.GlobalString("helm-binary") != "" {
-			helm.SetHelmBinary(c.GlobalString("helm-binary"))
-		}
-		return converge(st, helm, ctx)
-	}
-
-	err = a.VisitDesiredStates(fileOrDir, a.Selectors, convergeWithHelmBinary)
-
-	return toCliError(err)
-}
-
-func InitAppEntry(c *cli.Context, reverse bool) (*app.App, string, error) {
-	if c.NArg() > 0 {
-		cli.ShowAppHelp(c)
-		return nil, "", fmt.Errorf("err: extraneous arguments: %s", strings.Join(c.Args(), ", "))
-	}
-
-	fileOrDir := c.GlobalString("file")
-	kubeContext := c.GlobalString("kube-context")
-	namespace := c.GlobalString("namespace")
-	selectors := c.GlobalStringSlice("selector")
-	logger := c.App.Metadata["logger"].(*zap.SugaredLogger)
-
-	env := c.GlobalString("environment")
-	if env == "" {
-		env = state.DefaultEnv
-	}
-
-	app := app.Init(&app.App{
-		KubeContext: kubeContext,
-		Logger:      logger,
-		Reverse:     reverse,
-		Env:         env,
-		Namespace:   namespace,
-		Selectors:   selectors,
-	})
-
-	return app, fileOrDir, nil
-}
-
-func FindAndIterateOverDesiredStatesUsingFlagsWithReverse(c *cli.Context, reverse bool, converge func(*state.HelmState, helmexec.Interface, app.Context) []error) error {
-	a, fileOrDir, err := InitAppEntry(c, reverse)
-	if err != nil {
-		return err
-	}
-
-	ctx := app.NewContext()
-
-	convergeWithHelmBinary := func(st *state.HelmState, helm helmexec.Interface) []error {
-		if c.GlobalString("helm-binary") != "" {
-			helm.SetHelmBinary(c.GlobalString("helm-binary"))
-		}
-		return converge(st, helm, ctx)
-	}
-
-	err = a.VisitDesiredStatesWithReleasesFiltered(fileOrDir, convergeWithHelmBinary)
-
-	return toCliError(err)
-}
-
-func toCliError(err error) error {
-	if err != nil {
-		switch e := err.(type) {
-		case *app.NoMatchingHelmfileError:
-			return cli.NewExitError(e.Error(), 2)
-		case *app.Error:
-			return cli.NewExitError(e.Error(), e.Code())
-		default:
-			panic(fmt.Errorf("BUG: please file an github issue for this unhandled error: %T: %v", e, e))
-		}
-	}
-	return err
-}

cmd/deps.go

@@ -1,37 +0,0 @@
-package cmd
-
-import (
-	"github.com/roboll/helmfile/args"
-	"github.com/roboll/helmfile/helmexec"
-	"github.com/roboll/helmfile/pkg/app"
-	"github.com/roboll/helmfile/state"
-	"github.com/urfave/cli"
-)
-
-func Deps(a *app.App) cli.Command {
-	return cli.Command{
-		Name:  "deps",
-		Usage: "update charts based on the contents of requirements.yaml",
-		Flags: []cli.Flag{
-			cli.StringFlag{
-				Name:  "args",
-				Value: "",
-				Usage: "pass args to helm exec",
-			},
-		},
-		Action: func(c *cli.Context) error {
-			return VisitAllDesiredStates(c, func(state *state.HelmState, helm helmexec.Interface, ctx app.Context) (bool, []error) {
-				args := args.GetArgs(c.String("args"), state)
-				if len(args) > 0 {
-					helm.SetExtraArgs(args...)
-				}
-
-				errs := state.UpdateDeps(helm)
-
-				ok := len(errs) == 0
-
-				return ok, errs
-			})
-		},
-	}
-}

docs/writing-helmfile.md

@@ -11,13 +11,13 @@ One example of how helmfile achieves it is that, `helmfile` fails when you tried
 That is, the following example let `helmfile` fail when you have no `eventApi.replicas` defined in environment values.
 
 ```
-{{ .Environment.Values.eventApi.replicas | default 1 }}
+{{ .Values.eventApi.replicas | default 1 }}
 ```
 
 In case it isn't a mistake and you do want to allow missing keys, use the `getOrNil` template function:
 
 ```
-{{ .Environment.Values | getOrNil "eventApi.replicas" }}
+{{ .Values | getOrNil "eventApi.replicas" }}
 ```
 
 This result in printing `<no value` in your template, that may or may not result in a failure.
@@ -25,7 +25,7 @@ This result in printing `<no value` in your template, that may or may not result
 If you want a kind of default values that is used when a missing key was referenced, use `default` like:
 
 ```
-{{ .Environment.Values | getOrNil "eventApi.replicas" | default 1 }}
+{{ .Values | getOrNil "eventApi.replicas" | default 1 }}
 ```
 
 Now, you get `1` when there is no `eventApi.replicas` defined in environment values.
@@ -57,7 +57,7 @@ releases:
   values:
   - "./config/kubernetes-dashboard/values.yaml"
   - "./config/kubernetes-dashboard/{{ .Environment.Name }}.yaml"
-  values:
+  secrets:
   - "./config/kubernetes-dashboard/secrets.yaml"
   - "./config/kubernetes-dashboard/{{ .Environment.Name }}-secrets.yaml"
 ```
@@ -89,9 +89,51 @@ releases:
   <<: *default
 ```
 
+Release Templating supports the following parts of release definition:
+- basic fields: `name`, `namespace`, `chart`, `version`
+- boolean fields: `installed`, `wait`, `tillerless`, `verify` by the means of additional text
+  fields designed for templating only: `installedTemplate`, `waitTemplate`, `tillerlessTemplate`, `verifyTemplate`
+  ```yaml
+  # ...
+    installedTemplate: '{{`{{ eq .Release.Namespace "kube-system" }}`}}'
+    waitTemplate: '{{`{{ eq .Release.Labels.tag "safe" | not }}`}}'
+  # ...
+  ```
+- `set` block values:
+  ```yaml
+  # ...
+    setTemplate:
+    - name: '{{`{{ .Release.Name }}`}}'
+      values: '{{`{{ .Release.Namespace }}`}}'
+  # ...
+  ```
+- `values` and `secrets` file paths:
+  ```yaml
+  # ...
+    valuesTemplate:
+    - config/{{`{{ .Release.Name }}`}}/values.yaml
+    secrets:
+    - config/{{`{{ .Release.Name }}`}}/secrets.yaml
+  # ...
+  ```
+- inline `values` map:
+  ```yaml
+  # ...
+    valuesTemplate:
+    - image:
+        tag: `{{ .Release.Labels.tag }}`
+  # ...
+  ```
+
 See the [issue 428](https://github.com/roboll/helmfile/issues/428) for more context on how this is supposed to work.
 
-## Layering
+## Layering Release Values
+
+Please note, that it is not possible to layer `values` sections. If `values` is defined in the release and in the release template, only the `values` defined in the release will be considered. The same applies to `secrets` and `set`.
+
+## Layering State Files
+
+> See **Layering State Template Files** if you're layering templates.
 
 You may occasionally end up with many helmfiles that shares common parts like which repositories to use, and whichi release to be bundled by default.
 
@@ -101,23 +143,16 @@ Let's assume that your `helmfile.yaml` looks like:
 
 ```
 bases:
-- commons.yaml
 - environments.yaml
 
 releases:
+- name: metricbaet
+  chart: stable/metricbeat
 - name: myapp
   chart: mychart
 ```
 
-Whereas `commons.yaml` contained a monitoring agent:
-
-```yaml
-releases:
-- name: metricbaet
-  chart: stable/metricbeat
-```
-
-And `environments.yaml` contained well-known environments:
+Whereas `environments.yaml` contained well-known environments:
 
 ```yaml
 environments:
@@ -128,10 +163,6 @@ environments:
 At run time, `bases` in your `helmfile.yaml` are evaluated to produce:
 
 ```yaml
-# commons.yaml
-releases:
-- name: metricbaet
-  chart: stable/metricbeat
 ---
 # environments.yaml
 environments:
@@ -142,6 +173,8 @@ environments:
 releases:
 - name: myapp
   chart: mychart
+- name: metricbaet
+  chart: stable/metricbeat
 ```
 
 Finally the resulting YAML documents are merged in the order of occurrence,
@@ -164,3 +197,162 @@ Great!
 Now, repeat the above steps for each your `helmfile.yaml`, so that all your helmfiles becomes DRY.
 
 Please also see [the discussion in the issue 388](https://github.com/roboll/helmfile/issues/388#issuecomment-491710348) for more advanced layering examples.
+
+## Merging Arrays in Layers
+
+Helmfile doesn't merge arrays across layers. That is, the below example doesn't work as you might have expected:
+
+```yaml
+releases:
+- name: metricbaet
+  chart: stable/metricbeat
+---
+releases:
+- name: myapp
+  chart: mychart
+```
+
+Helmfile overrides the `releases` array with the latest layer so the resulting state file will be:
+
+```yaml
+releases:
+# metricbeat release disappeared! but that's how helmfiel works
+- name: myapp
+  chart: mychart
+```
+
+A work-around is to treat the state file as a go template and use `readFile` template function to import the common part of your state file as a plain text:
+
+`common.yaml`:
+
+```yaml
+templates:
+  metricbeat: &metricbeat
+  - name: metricbeat
+    chart: stable/metricbeat
+```
+
+`helmfile.yaml`:
+
+```yaml
+{{ readFile "common.yaml" }}
+
+releases:
+- <<: *metricbeat
+- name: myapp
+  chart: mychart
+```
+
+## Layering State Template Files
+
+Do you need to make your state file even more DRY?
+
+Turned out layering state files wasn't enough for you?
+
+Helmfile supports an advanced feature that allows you to compose state "template" files to generate the final state to be processed.
+
+In the following example `helmfile.yaml.gotmpl`, each `---` separated part of the file is a go template.
+
+`helmfile.yaml.gotmpl`:
+
+```yaml
+# Part 1: Reused Enviroment Values
+bases:
+  - myenv.yaml
+---
+# Part 2: Reused Defaults
+bases:
+  - mydefaults.yaml
+---
+# Part 3: Dynamic Releases
+releases:
+  - name: test1
+    chart: mychart-{{ .Values.myname }}
+    values:
+      replicaCount: 1
+      image:
+        repository: "nginx"
+        tag: "latest"
+```
+
+Suppose the `myenv.yaml` and `test.env.yaml` loaded in the first part looks like:
+
+`myenv.yaml`:
+
+```yaml
+environments:
+  test:
+    values:
+      - test.env.yaml
+```
+
+`test.env.yaml`:
+
+```yaml
+kubeContext: test
+wait: false
+cvOnly: false
+myname: "dog"
+```
+
+Where the gotmpl file loaded in the second part looks like:
+
+`mydefaults.yaml.gotmpl`:
+
+```yaml
+helmDefaults:
+  tillerNamespace: kube-system
+  kubeContext: {{ .Values.kubeContext }}
+  verify: false
+  {{ if .Values.wait }}
+  wait: true
+  {{ else }}
+  wait: false
+  {{ end }}
+  timeout: 600
+  recreatePods: false
+  force: true
+```
+
+Each go template is rendered in the context where `.Values` is inherited from the previous part. 
+
+So in `mydefaults.yaml.gotmpl`, both `.Values.kubeContext` and `.Values.wait` are valid as they do exist in the environment values inherited from the previous part(=the first part) of your `helmfile.yaml.gotmpl`, and therefore the template is rendered to:
+
+```yaml
+helmDefaults:
+  tillerNamespace: kube-system
+  kubeContext: test
+  verify: false
+  wait: false
+  timeout: 600
+  recreatePods: false
+  force: true
+```
+
+Similarly, the third part of the top-level `helmfile.yaml.gotmpl`, `.Values.myname` is valid as it is included in the environment values inherited from the previous parts:
+
+```yaml
+# Part 3: Dynamic Releases
+releases:
+  - name: test1
+    chart: mychart-{{ .Values.myname }}
+    values:
+      replicaCount: 1
+      image:
+        repository: "nginx"
+        tag: "latest"
+````
+
+hence rendered to:
+
+```yaml
+# Part 3: Dynamic Releases
+releases:
+  - name: test1
+    chart: mychart-dog
+    values:
+      replicaCount: 1
+      image:
+        repository: "nginx"
+        tag: "latest"
+```

environment/environment.go

@@ -1,8 +0,0 @@
-package environment
-
-type Environment struct {
-	Name   string
-	Values map[string]interface{}
-}
-
-var EmptyEnvironment Environment

go.mod

@@ -1,23 +1,47 @@
 module github.com/roboll/helmfile
 
-go 1.12
+go 1.13
 
 require (
-	github.com/Masterminds/semver v1.4.1 // indirect
-	github.com/Masterminds/sprig v2.15.0+incompatible
-	github.com/aokoli/goutils v1.0.1 // indirect
-	github.com/google/go-cmp v0.3.0 // indirect
-	github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c // indirect
-	github.com/huandu/xstrings v1.0.0 // indirect
+	cloud.google.com/go v0.47.0 // indirect
+	github.com/Azure/azure-sdk-for-go v35.0.0+incompatible // indirect
+	github.com/Masterminds/goutils v1.1.0 // indirect
+	github.com/Masterminds/semver v1.4.2
+	github.com/Masterminds/sprig v2.22.0+incompatible
+	github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a
+	github.com/aws/aws-sdk-go v1.25.22 // indirect
+	github.com/go-test/deep v1.0.3
+	github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect
+	github.com/google/go-cmp v0.3.1
+	github.com/gosuri/uitable v0.0.3
+	github.com/hashicorp/go-getter v1.3.0
+	github.com/hashicorp/go-retryablehttp v0.6.3 // indirect
+	github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c // indirect
+	github.com/huandu/xstrings v1.2.0 // indirect
 	github.com/imdario/mergo v0.3.6
-	github.com/mattn/go-runewidth v0.0.4 // indirect
-	github.com/pkg/errors v0.8.1 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/mattn/go-colorable v0.1.4 // indirect
+	github.com/mattn/go-isatty v0.0.10 // indirect
+	github.com/pierrec/lz4 v2.3.0+incompatible // indirect
+	github.com/r3labs/diff v0.0.0-20190801153147-a71de73c46ad
 	github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939
-	github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28
-	go.uber.org/atomic v1.3.2 // indirect
-	go.uber.org/multierr v1.1.0 // indirect
-	go.uber.org/zap v1.8.0
-	golang.org/x/crypto v0.0.0-20180403160946-b2aa35443fbc // indirect
-	gopkg.in/yaml.v2 v2.2.1
+	github.com/urfave/cli v1.20.0
+	github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363
+	github.com/variantdev/vals v0.0.0-20191030045026-1fa6af1b5299
+	go.mozilla.org/sops v0.0.0-20190912205235-14a22d7a7060 // indirect
+	go.opencensus.io v0.22.1 // indirect
+	go.uber.org/multierr v1.1.0
+	go.uber.org/zap v1.9.1
+	golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf // indirect
+	golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271 // indirect
+	golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c // indirect
+	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
+	google.golang.org/api v0.13.0 // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6 // indirect
+	google.golang.org/grpc v1.24.0 // indirect
+	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v2 v2.2.4
+	gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d // indirect
 	gotest.tools v2.2.0+incompatible
 )

main.go

@@ -2,16 +2,16 @@ package main
 
 import (
 	"fmt"
-	"github.com/roboll/helmfile/args"
-	"github.com/roboll/helmfile/cmd"
-	"github.com/roboll/helmfile/helmexec"
+	"os"
+	"strings"
+
 	"github.com/roboll/helmfile/pkg/app"
-	"github.com/roboll/helmfile/state"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/maputil"
+	"github.com/roboll/helmfile/pkg/state"
 	"github.com/urfave/cli"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
-	"os"
-	"strings"
 )
 
 var Version string
@@ -50,6 +50,7 @@ func main() {
 		cli.StringFlag{
 			Name:  "helm-binary, b",
 			Usage: "path to helm binary",
+			Value: "helm",
 		},
 		cli.StringFlag{
 			Name:  "file, f",
@@ -59,6 +60,14 @@ func main() {
 			Name:  "environment, e",
 			Usage: "specify the environment name. defaults to `default`",
 		},
+		cli.StringSliceFlag{
+			Name:  "state-values-set",
+			Usage: "set state values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)",
+		},
+		cli.StringSliceFlag{
+			Name:  "state-values-file",
+			Usage: "specify state values in a YAML file",
+		},
 		cli.BoolFlag{
 			Name:  "quiet, q",
 			Usage: "Silence output. Equivalent to log-level warn",
@@ -67,6 +76,10 @@ func main() {
 			Name:  "kube-context",
 			Usage: "Set kubectl context. Uses current context by default",
 		},
+		cli.BoolFlag{
+			Name:  "no-color",
+			Usage: "Output without color",
+		},
 		cli.StringFlag{
 			Name:  "log-level",
 			Usage: "Set log level, default info",
@@ -82,6 +95,10 @@ func main() {
 	--selector tier=frontend,tier!=proxy --selector tier=backend. Will match all frontend, non-proxy releases AND all backend releases.
 	The name of a release can be used as a label. --selector name=myrelease`,
 		},
+		cli.BoolFlag{
+			Name:  "allow-no-matching-release",
+			Usage: `Do not exit with an error code if the provided selector has no matching releases.`,
+		},
 		cli.BoolFlag{
 			Name:  "interactive, i",
 			Usage: "Request confirmation before attempting to modify clusters",
@@ -90,6 +107,24 @@ func main() {
 
 	cliApp.Before = configureLogging
 	cliApp.Commands = []cli.Command{
+		{
+			Name:  "deps",
+			Usage: "update charts based on the contents of requirements.yaml",
+			Flags: []cli.Flag{
+				cli.StringFlag{
+					Name:  "args",
+					Value: "",
+					Usage: "pass args to helm exec",
+				},
+				cli.BoolFlag{
+					Name:  "skip-repos",
+					Usage: "skip running `helm repo update` before running `helm dependency build`",
+				},
+			},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.Deps(c)
+			}),
+		},
 		{
 			Name:  "repos",
 			Usage: "sync repositories from state file (helm repo add && helm repo update)",
@@ -100,20 +135,9 @@ func main() {
 					Usage: "pass args to helm exec",
 				},
 			},
-			Action: func(c *cli.Context) error {
-				return cmd.VisitAllDesiredStates(c, func(state *state.HelmState, helm helmexec.Interface, ctx app.Context) (bool, []error) {
-					args := args.GetArgs(c.String("args"), state)
-					if len(args) > 0 {
-						helm.SetExtraArgs(args...)
-					}
-
-					errs := ctx.SyncReposOnce(state, helm)
-
-					ok := len(state.Repositories) > 0 && len(errs) == 0
-
-					return ok, errs
-				})
-			},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.Repos(c)
+			}),
 		},
 		{
 			Name:  "charts",
@@ -124,6 +148,10 @@ func main() {
 					Value: "",
 					Usage: "pass args to helm exec",
 				},
+				cli.StringSliceFlag{
+					Name:  "set",
+					Usage: "additional values to be merged into the command",
+				},
 				cli.StringSliceFlag{
 					Name:  "values",
 					Usage: "additional value files to be merged into the command",
@@ -134,14 +162,9 @@ func main() {
 					Usage: "maximum number of concurrent helm processes to run, 0 is unlimited",
 				},
 			},
-			Action: func(c *cli.Context) error {
-				affectedReleases := state.AffectedReleases{}
-				errs := findAndIterateOverDesiredStatesUsingFlags(c, func(st *state.HelmState, helm helmexec.Interface, _ app.Context) []error {
-					return executeSyncCommand(c, &affectedReleases, st, helm)
-				})
-				affectedReleases.DisplayAffectedReleases(c.App.Metadata["logger"].(*zap.SugaredLogger))
-				return errs
-			},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.DeprecatedSyncCharts(c)
+			}),
 		},
 		{
 			Name:  "diff",
@@ -152,6 +175,10 @@ func main() {
 					Value: "",
 					Usage: "pass args to helm exec",
 				},
+				cli.StringSliceFlag{
+					Name:  "set",
+					Usage: "additional values to be merged into the command",
+				},
 				cli.StringSliceFlag{
 					Name:  "values",
 					Usage: "additional value files to be merged into the command",
@@ -173,25 +200,15 @@ func main() {
 					Value: 0,
 					Usage: "maximum number of concurrent helm processes to run, 0 is unlimited",
 				},
+				cli.IntFlag{
+					Name:  "context",
+					Value: 0,
+					Usage: "output NUM lines of context around changes",
+				},
 			},
-			Action: func(c *cli.Context) error {
-				return findAndIterateOverDesiredStatesUsingFlags(c, func(state *state.HelmState, helm helmexec.Interface, ctx app.Context) []error {
-					if !c.Bool("skip-deps") {
-						if errs := ctx.SyncReposOnce(state, helm); errs != nil && len(errs) > 0 {
-							return errs
-						}
-						if errs := state.BuildDeps(helm); errs != nil && len(errs) > 0 {
-							return errs
-						}
-					}
-					if errs := state.PrepareRelease(helm, "diff"); errs != nil && len(errs) > 0 {
-						return errs
-					}
-
-					_, errs := executeDiffCommand(c, state, helm, c.Bool("detailed-exitcode"), c.Bool("suppress-secrets"))
-					return errs
-				})
-			},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.Diff(c)
+			}),
 		},
 		{
 			Name:  "template",
@@ -202,10 +219,18 @@ func main() {
 					Value: "",
 					Usage: "pass args to helm template",
 				},
+				cli.StringSliceFlag{
+					Name:  "set",
+					Usage: "additional values to be merged into the command",
+				},
 				cli.StringSliceFlag{
 					Name:  "values",
 					Usage: "additional value files to be merged into the command",
 				},
+				cli.StringFlag{
+					Name:  "output-dir",
+					Usage: "output directory to pass to helm template (helm template --output-dir)",
+				},
 				cli.IntFlag{
 					Name:  "concurrency",
 					Value: 0,
@@ -216,22 +241,9 @@ func main() {
 					Usage: "skip running `helm repo update` and `helm dependency build`",
 				},
 			},
-			Action: func(c *cli.Context) error {
-				return findAndIterateOverDesiredStatesUsingFlags(c, func(state *state.HelmState, helm helmexec.Interface, ctx app.Context) []error {
-					if !c.Bool("skip-deps") {
-						if errs := ctx.SyncReposOnce(state, helm); errs != nil && len(errs) > 0 {
-							return errs
-						}
-						if errs := state.BuildDeps(helm); errs != nil && len(errs) > 0 {
-							return errs
-						}
-					}
-					if errs := state.PrepareRelease(helm, "template"); errs != nil && len(errs) > 0 {
-						return errs
-					}
-					return executeTemplateCommand(c, state, helm)
-				})
-			},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.Template(c)
+			}),
 		},
 		{
 			Name:  "lint",
@@ -242,6 +254,10 @@ func main() {
 					Value: "",
 					Usage: "pass args to helm exec",
 				},
+				cli.StringSliceFlag{
+					Name:  "set",
+					Usage: "additional values to be merged into the command",
+				},
 				cli.StringSliceFlag{
 					Name:  "values",
 					Usage: "additional value files to be merged into the command",
@@ -251,26 +267,23 @@ func main() {
 					Value: 0,
 					Usage: "maximum number of concurrent downloads of release charts",
 				},
+				cli.BoolFlag{
+					Name:  "skip-deps",
+					Usage: "skip running `helm repo update` and `helm dependency build`",
+				},
 			},
-			Action: func(c *cli.Context) error {
-				return findAndIterateOverDesiredStatesUsingFlags(c, func(state *state.HelmState, helm helmexec.Interface, ctx app.Context) []error {
-					values := c.StringSlice("values")
-					args := args.GetArgs(c.String("args"), state)
-					workers := c.Int("concurrency")
-					if errs := ctx.SyncReposOnce(state, helm); errs != nil && len(errs) > 0 {
-						return errs
-					}
-					if errs := state.PrepareRelease(helm, "lint"); errs != nil && len(errs) > 0 {
-						return errs
-					}
-					return state.LintReleases(helm, values, args, workers)
-				})
-			},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.Lint(c)
+			}),
 		},
 		{
 			Name:  "sync",
 			Usage: "sync all resources from state file (repos, releases and chart deps)",
 			Flags: []cli.Flag{
+				cli.StringSliceFlag{
+					Name:  "set",
+					Usage: "additional values to be merged into the command",
+				},
 				cli.StringSliceFlag{
 					Name:  "values",
 					Usage: "additional value files to be merged into the command",
@@ -290,30 +303,18 @@ func main() {
 					Usage: "skip running `helm repo update` and `helm dependency build`",
 				},
 			},
-			Action: func(c *cli.Context) error {
-				affectedReleases := state.AffectedReleases{}
-				errs := findAndIterateOverDesiredStatesUsingFlags(c, func(st *state.HelmState, helm helmexec.Interface, ctx app.Context) []error {
-					if !c.Bool("skip-deps") {
-						if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
-							return errs
-						}
-						if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
-							return errs
-						}
-					}
-					if errs := st.PrepareRelease(helm, "sync"); errs != nil && len(errs) > 0 {
-						return errs
-					}
-					return executeSyncCommand(c, &affectedReleases, st, helm)
-				})
-				affectedReleases.DisplayAffectedReleases(c.App.Metadata["logger"].(*zap.SugaredLogger))
-				return errs
-			},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.Sync(c)
+			}),
 		},
 		{
 			Name:  "apply",
 			Usage: "apply all resources from state file only when there are changes",
 			Flags: []cli.Flag{
+				cli.StringSliceFlag{
+					Name:  "set",
+					Usage: "additional values to be merged into the command",
+				},
 				cli.StringSliceFlag{
 					Name:  "values",
 					Usage: "additional value files to be merged into the command",
@@ -323,6 +324,11 @@ func main() {
 					Value: 0,
 					Usage: "maximum number of concurrent helm processes to run, 0 is unlimited",
 				},
+				cli.IntFlag{
+					Name:  "context",
+					Value: 0,
+					Usage: "output NUM lines of context around changes",
+				},
 				cli.StringFlag{
 					Name:  "args",
 					Value: "",
@@ -337,88 +343,9 @@ func main() {
 					Usage: "skip running `helm repo update` and `helm dependency build`",
 				},
 			},
-			Action: func(c *cli.Context) error {
-				affectedReleases := state.AffectedReleases{}
-				errs := findAndIterateOverDesiredStatesUsingFlags(c, func(st *state.HelmState, helm helmexec.Interface, ctx app.Context) []error {
-					if !c.Bool("skip-deps") {
-						if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
-							return errs
-						}
-						if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
-							return errs
-						}
-					}
-					if errs := st.PrepareRelease(helm, "apply"); errs != nil && len(errs) > 0 {
-						return errs
-					}
-
-					releases, errs := executeDiffCommand(c, st, helm, true, c.Bool("suppress-secrets"))
-
-					releasesToBeDeleted, err := st.DetectReleasesToBeDeleted(helm)
-					if err != nil {
-						errs = append(errs, err)
-					}
-
-					fatalErrs := []error{}
-
-					noError := true
-					for _, e := range errs {
-						switch err := e.(type) {
-						case *state.ReleaseError:
-							if err.Code != 2 {
-								noError = false
-								fatalErrs = append(fatalErrs, e)
-							}
-						default:
-							noError = false
-							fatalErrs = append(fatalErrs, e)
-						}
-					}
-
-					// sync only when there are changes
-					if noError {
-						if len(releases) == 0 && len(releasesToBeDeleted) == 0 {
-							// TODO better way to get the logger
-							logger := c.App.Metadata["logger"].(*zap.SugaredLogger)
-							logger.Infof("")
-							logger.Infof("No affected releases")
-						} else {
-							names := []string{}
-							for _, r := range releases {
-								names = append(names, fmt.Sprintf("  %s (%s) UPDATED", r.Name, r.Chart))
-							}
-							for _, r := range releasesToBeDeleted {
-								names = append(names, fmt.Sprintf("  %s (%s) DELETED", r.Name, r.Chart))
-							}
-
-							msg := fmt.Sprintf(`Affected releases are:
-%s
-
-Do you really want to apply?
-  Helmfile will apply all your changes, as shown above.
-
-`, strings.Join(names, "\n"))
-							interactive := c.GlobalBool("interactive")
-							if !interactive || interactive && askForConfirmation(msg) {
-								rs := []state.ReleaseSpec{}
-								for _, r := range releases {
-									rs = append(rs, *r)
-								}
-								for _, r := range releasesToBeDeleted {
-									rs = append(rs, *r)
-								}
-
-								st.Releases = rs
-								return executeSyncCommand(c, &affectedReleases, st, helm)
-							}
-						}
-					}
-
-					return fatalErrs
-				})
-				affectedReleases.DisplayAffectedReleases(c.App.Metadata["logger"].(*zap.SugaredLogger))
-				return errs
-			},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.Apply(c)
+			}),
 		},
 		{
 			Name:  "status",
@@ -435,23 +362,19 @@ Do you really want to apply?
 					Usage: "pass args to helm exec",
 				},
 			},
-			Action: func(c *cli.Context) error {
-				return findAndIterateOverDesiredStatesUsingFlags(c, func(state *state.HelmState, helm helmexec.Interface, _ app.Context) []error {
-					workers := c.Int("concurrency")
-
-					args := args.GetArgs(c.String("args"), state)
-					if len(args) > 0 {
-						helm.SetExtraArgs(args...)
-					}
-
-					return state.ReleaseStatuses(helm, workers)
-				})
-			},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.Status(c)
+			}),
 		},
 		{
 			Name:  "delete",
 			Usage: "DEPRECATED: delete releases from state file (helm delete)",
 			Flags: []cli.Flag{
+				cli.IntFlag{
+					Name:  "concurrency",
+					Value: 0,
+					Usage: "maximum number of concurrent helm processes to run, 0 is unlimited",
+				},
 				cli.StringFlag{
 					Name:  "args",
 					Value: "",
@@ -462,77 +385,28 @@ Do you really want to apply?
 					Usage: "purge releases i.e. free release names and histories",
 				},
 			},
-			Action: func(c *cli.Context) error {
-				affectedReleases := state.AffectedReleases{}
-				errs := cmd.FindAndIterateOverDesiredStatesUsingFlagsWithReverse(c, true, func(state *state.HelmState, helm helmexec.Interface, _ app.Context) []error {
-					purge := c.Bool("purge")
-
-					args := args.GetArgs(c.String("args"), state)
-					if len(args) > 0 {
-						helm.SetExtraArgs(args...)
-					}
-
-					names := make([]string, len(state.Releases))
-					for i, r := range state.Releases {
-						names[i] = fmt.Sprintf("  %s (%s)", r.Name, r.Chart)
-					}
-
-					msg := fmt.Sprintf(`Affected releases are:
-%s
-
-Do you really want to delete?
-  Helmfile will delete all your releases, as shown above.
-
-`, strings.Join(names, "\n"))
-					interactive := c.GlobalBool("interactive")
-					if !interactive || interactive && askForConfirmation(msg) {
-						return state.DeleteReleases(&affectedReleases, helm, purge)
-					}
-					return nil
-				})
-				affectedReleases.DisplayAffectedReleases(c.App.Metadata["logger"].(*zap.SugaredLogger))
-				return errs
-			},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.Delete(c)
+			}),
 		},
 		{
 			Name:  "destroy",
 			Usage: "deletes and then purges releases",
 			Flags: []cli.Flag{
+				cli.IntFlag{
+					Name:  "concurrency",
+					Value: 0,
+					Usage: "maximum number of concurrent helm processes to run, 0 is unlimited",
+				},
 				cli.StringFlag{
 					Name:  "args",
 					Value: "",
 					Usage: "pass args to helm exec",
 				},
 			},
-			Action: func(c *cli.Context) error {
-				affectedReleases := state.AffectedReleases{}
-				errs := cmd.FindAndIterateOverDesiredStatesUsingFlagsWithReverse(c, true, func(state *state.HelmState, helm helmexec.Interface, _ app.Context) []error {
-					args := args.GetArgs(c.String("args"), state)
-					if len(args) > 0 {
-						helm.SetExtraArgs(args...)
-					}
-
-					names := make([]string, len(state.Releases))
-					for i, r := range state.Releases {
-						names[i] = fmt.Sprintf("  %s (%s)", r.Name, r.Chart)
-					}
-
-					msg := fmt.Sprintf(`Affected releases are:
-%s
-
-Do you really want to delete?
-  Helmfile will delete all your releases, as shown above.
-
-`, strings.Join(names, "\n"))
-					interactive := c.GlobalBool("interactive")
-					if !interactive || interactive && askForConfirmation(msg) {
-						return state.DeleteReleases(&affectedReleases, helm, true)
-					}
-					return nil
-				})
-				affectedReleases.DisplayAffectedReleases(c.App.Metadata["logger"].(*zap.SugaredLogger))
-				return errs
-			},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.Destroy(c)
+			}),
 		},
 		{
 			Name:  "test",
@@ -558,20 +432,25 @@ Do you really want to delete?
 					Usage: "maximum number of concurrent helm processes to run, 0 is unlimited",
 				},
 			},
-			Action: func(c *cli.Context) error {
-				return findAndIterateOverDesiredStatesUsingFlags(c, func(state *state.HelmState, helm helmexec.Interface, _ app.Context) []error {
-					cleanup := c.Bool("cleanup")
-					timeout := c.Int("timeout")
-					concurrency := c.Int("concurrency")
-
-					args := args.GetArgs(c.String("args"), state)
-					if len(args) > 0 {
-						helm.SetExtraArgs(args...)
-					}
-
-					return state.TestReleases(helm, cleanup, timeout, concurrency)
-				})
-			},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.Test(c)
+			}),
+		},
+		{
+			Name:  "build",
+			Usage: "output compiled helmfile state(s) as YAML",
+			Flags: []cli.Flag{},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.PrintState(c)
+			}),
+		},
+		{
+			Name:  "list",
+			Usage: "list releases defined in state file",
+			Flags: []cli.Flag{},
+			Action: action(func(run *app.App, c configImpl) error {
+				return run.ListReleases(c)
+			}),
 		},
 	}
 
@@ -582,39 +461,184 @@ Do you really want to delete?
 	}
 }
 
-func executeSyncCommand(c *cli.Context, affectedReleases *state.AffectedReleases, state *state.HelmState, helm helmexec.Interface) []error {
-	args := args.GetArgs(c.String("args"), state)
-	if len(args) > 0 {
-		helm.SetExtraArgs(args...)
+type configImpl struct {
+	c *cli.Context
+
+	set map[string]interface{}
+}
+
+func NewUrfaveCliConfigImpl(c *cli.Context) (configImpl, error) {
+	if c.NArg() > 0 {
+		cli.ShowAppHelp(c)
+		return configImpl{}, fmt.Errorf("err: extraneous arguments: %s", strings.Join(c.Args(), ", "))
 	}
 
-	values := c.StringSlice("values")
-	workers := c.Int("concurrency")
-
-	return state.SyncReleases(affectedReleases, helm, values, workers)
-}
-
-func executeTemplateCommand(c *cli.Context, state *state.HelmState, helm helmexec.Interface) []error {
-	args := args.GetArgs(c.String("args"), state)
-	values := c.StringSlice("values")
-	workers := c.Int("concurrency")
-
-	return state.TemplateReleases(helm, values, args, workers)
-}
-
-func executeDiffCommand(c *cli.Context, st *state.HelmState, helm helmexec.Interface, detailedExitCode, suppressSecrets bool) ([]*state.ReleaseSpec, []error) {
-	args := args.GetArgs(c.String("args"), st)
-	if len(args) > 0 {
-		helm.SetExtraArgs(args...)
+	conf := configImpl{
+		c: c,
 	}
 
-	values := c.StringSlice("values")
-	workers := c.Int("concurrency")
-	triggerCleanupEvents := c.Command.HasName("diff")
+	optsSet := c.GlobalStringSlice("state-values-set")
+	if len(optsSet) > 0 {
+		set := map[string]interface{}{}
+		for i := range optsSet {
+			ops := strings.Split(optsSet[i], ",")
+			for j := range ops {
+				op := strings.SplitN(ops[j], "=", 2)
+				k := strings.Split(op[0], ".")
+				v := op[1]
 
-	return st.DiffReleases(helm, values, workers, detailedExitCode, suppressSecrets, triggerCleanupEvents)
+				set = maputil.Set(set, k, v)
+			}
+		}
+		conf.set = set
+	}
+
+	return conf, nil
 }
 
-func findAndIterateOverDesiredStatesUsingFlags(c *cli.Context, converge func(*state.HelmState, helmexec.Interface, app.Context) []error) error {
-	return cmd.FindAndIterateOverDesiredStatesUsingFlagsWithReverse(c, false, converge)
+func (c configImpl) Set() []string {
+	return c.c.StringSlice("set")
+}
+
+func (c configImpl) SkipRepos() bool {
+	return c.c.Bool("skip-repos")
+}
+
+func (c configImpl) Values() []string {
+	return c.c.StringSlice("values")
+}
+
+func (c configImpl) Args() string {
+	return c.c.String("args")
+}
+
+func (c configImpl) OutputDir() string {
+	return c.c.String("output-dir")
+}
+
+func (c configImpl) Concurrency() int {
+	return c.c.Int("concurrency")
+}
+
+func (c configImpl) HasCommandName(name string) bool {
+	return c.c.Command.HasName(name)
+}
+
+// DiffConfig
+
+func (c configImpl) SkipDeps() bool {
+	return c.c.Bool("skip-deps")
+}
+
+func (c configImpl) DetailedExitcode() bool {
+	return c.c.Bool("detailed-exitcode")
+}
+
+func (c configImpl) SuppressSecrets() bool {
+	return c.c.Bool("suppress-secrets")
+}
+
+// DeleteConfig
+
+func (c configImpl) Purge() bool {
+	return c.c.Bool("purge")
+}
+
+// TestConfig
+
+func (c configImpl) Cleanup() bool {
+	return c.c.Bool("cleanup")
+}
+
+func (c configImpl) Timeout() int {
+	return c.c.Int("timeout")
+}
+
+// GlobalConfig
+
+func (c configImpl) HelmBinary() string {
+	return c.c.GlobalString("helm-binary")
+}
+
+func (c configImpl) KubeContext() string {
+	return c.c.GlobalString("kube-context")
+}
+
+func (c configImpl) Namespace() string {
+	return c.c.GlobalString("namespace")
+}
+
+func (c configImpl) FileOrDir() string {
+	return c.c.GlobalString("file")
+}
+
+func (c configImpl) Selectors() []string {
+	return c.c.GlobalStringSlice("selector")
+}
+
+func (c configImpl) StateValuesSet() map[string]interface{} {
+	return c.set
+}
+
+func (c configImpl) StateValuesFiles() []string {
+	return c.c.GlobalStringSlice("state-values-file")
+}
+
+func (c configImpl) Interactive() bool {
+	return c.c.GlobalBool("interactive")
+}
+
+func (c configImpl) NoColor() bool {
+	return c.c.GlobalBool("no-color")
+}
+
+func (c configImpl) Context() int {
+	return c.c.Int("context")
+}
+
+func (c configImpl) Logger() *zap.SugaredLogger {
+	return c.c.App.Metadata["logger"].(*zap.SugaredLogger)
+}
+
+func (c configImpl) Env() string {
+	env := c.c.GlobalString("environment")
+	if env == "" {
+		env = state.DefaultEnv
+	}
+	return env
+}
+
+func action(do func(*app.App, configImpl) error) func(*cli.Context) error {
+	return func(implCtx *cli.Context) error {
+		conf, err := NewUrfaveCliConfigImpl(implCtx)
+		if err != nil {
+			return err
+		}
+
+		a := app.New(conf)
+
+		a.ErrorHandler = func(err error) error {
+			return toCliError(implCtx, err)
+		}
+
+		return do(a, conf)
+	}
+}
+
+func toCliError(c *cli.Context, err error) error {
+	if err != nil {
+		switch e := err.(type) {
+		case *app.NoMatchingHelmfileError:
+			noMatchingExitCode := 3
+			if c.GlobalBool("allow-no-matching-release") {
+				noMatchingExitCode = 0
+			}
+			return cli.NewExitError(e.Error(), noMatchingExitCode)
+		case *app.Error:
+			return cli.NewExitError(e.Error(), e.Code())
+		default:
+			panic(fmt.Errorf("BUG: please file an github issue for this unhandled error: %T: %v", e, e))
+		}
+	}
+	return err
 }

pkg/app/app.go

@@ -1,6 +1,7 @@
 package app
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -8,15 +9,26 @@ import (
 	"os/signal"
 	"strings"
 	"syscall"
+	"text/tabwriter"
+
+	"github.com/gosuri/uitable"
+	"github.com/roboll/helmfile/pkg/argparser"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/remote"
+	"github.com/roboll/helmfile/pkg/state"
+	"github.com/variantdev/vals"
 
-	"github.com/roboll/helmfile/helmexec"
-	"github.com/roboll/helmfile/state"
 	"go.uber.org/zap"
 
 	"path/filepath"
 	"sort"
 )
 
+const (
+	// cache size for improving performance of ref+.* secrets rendering
+	valsCacheSize = 512
+)
+
 type App struct {
 	KubeContext string
 	Logger      *zap.SugaredLogger
@@ -24,8 +36,16 @@ type App struct {
 	Env         string
 	Namespace   string
 	Selectors   []string
+	Args        string
+	ValuesFiles []string
+	Set         map[string]interface{}
+
+	FileOrDir string
+
+	ErrorHandler func(error) error
 
 	readFile          func(string) ([]byte, error)
+	fileExists        func(string) (bool, error)
 	glob              func(string) ([]string, error)
 	abs               func(string) (string, error)
 	fileExistsAt      func(string) bool
@@ -33,6 +53,29 @@ type App struct {
 
 	getwd func() (string, error)
 	chdir func(string) error
+
+	remote *remote.Remote
+
+	helmExecer helmexec.Interface
+
+	valsRuntime vals.Evaluator
+}
+
+func New(conf ConfigProvider) *App {
+	return Init(&App{
+		KubeContext: conf.KubeContext(),
+		Logger:      conf.Logger(),
+		Env:         conf.Env(),
+		Namespace:   conf.Namespace(),
+		Selectors:   conf.Selectors(),
+		Args:        conf.Args(),
+		FileOrDir:   conf.FileOrDir(),
+		ValuesFiles: conf.StateValuesFiles(),
+		Set:         conf.StateValuesSet(),
+		helmExecer: helmexec.New(conf.HelmBinary(), conf.Logger(), conf.KubeContext(), &helmexec.ShellRunner{
+			Logger: conf.Logger(),
+		}),
+	})
 }
 
 func Init(app *App) *App {
@@ -42,10 +85,128 @@ func Init(app *App) *App {
 	app.getwd = os.Getwd
 	app.chdir = os.Chdir
 	app.fileExistsAt = fileExistsAt
+	app.fileExists = fileExists
 	app.directoryExistsAt = directoryExistsAt
+
+	var err error
+	app.valsRuntime, err = vals.New(valsCacheSize)
+	if err != nil {
+		panic(fmt.Sprintf("Failed to initialize vals runtime: %v", err))
+	}
+
 	return app
 }
 
+func (a *App) Deps(c DepsConfigProvider) error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
+		return run.Deps(c)
+	})
+}
+
+func (a *App) Repos(c ReposConfigProvider) error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
+		return run.Repos(c)
+	})
+}
+
+func (a *App) reverse() *App {
+	new := *a
+	new.Reverse = true
+	return &new
+}
+
+func (a *App) DeprecatedSyncCharts(c DeprecatedChartsConfigProvider) error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
+		return run.DeprecatedSyncCharts(c)
+	})
+}
+
+func (a *App) Diff(c DiffConfigProvider) error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
+		return run.Diff(c)
+	})
+}
+
+func (a *App) Template(c TemplateConfigProvider) error {
+	return a.ForEachState(func(run *Run) (bool, []error) {
+		return a.template(run, c)
+	})
+}
+
+func (a *App) Lint(c LintConfigProvider) error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
+		return run.Lint(c)
+	})
+}
+
+func (a *App) Sync(c SyncConfigProvider) error {
+	return a.ForEachState(func(run *Run) (bool, []error) {
+		return a.sync(run, c)
+	})
+}
+
+func (a *App) Apply(c ApplyConfigProvider) error {
+	return a.ForEachState(func(run *Run) (bool, []error) {
+		return a.apply(run, c)
+	})
+}
+
+func (a *App) Status(c StatusesConfigProvider) error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
+		return run.Status(c)
+	})
+}
+
+func (a *App) Delete(c DeleteConfigProvider) error {
+	return a.reverse().ForEachState(func(run *Run) (bool, []error) {
+		return a.delete(run, c.Purge(), c)
+	})
+}
+
+func (a *App) Destroy(c DestroyConfigProvider) error {
+	return a.reverse().ForEachState(func(run *Run) (bool, []error) {
+		return a.delete(run, true, c)
+	})
+}
+
+func (a *App) Test(c TestConfigProvider) error {
+	return a.ForEachStateFiltered(func(run *Run) []error {
+		return run.Test(c)
+	})
+}
+
+func (a *App) PrintState(c StateConfigProvider) error {
+
+	return a.ForEachStateFiltered(func(run *Run) []error {
+		state, err := run.state.ToYaml()
+		if err != nil {
+			return []error{err}
+		}
+		fmt.Printf("---\n#  Source: %s\n\n%+v", run.state.FilePath, state)
+		return []error{}
+	})
+}
+
+func (a *App) ListReleases(c StateConfigProvider) error {
+	table := uitable.New()
+	table.AddRow("NAME", "NAMESPACE", "INSTALLED", "LABELS")
+
+	err := a.ForEachStateFiltered(func(run *Run) []error {
+		//var releases m
+		for _, r := range run.state.Releases {
+			labels := ""
+			for k, v := range r.Labels {
+				labels = fmt.Sprintf("%s,%s:%s", labels, k, v)
+			}
+			installed := r.Installed == nil || *r.Installed
+			table.AddRow(r.Name, r.Namespace, fmt.Sprintf("%t", installed), strings.Trim(labels, ","))
+		}
+		return []error{}
+	})
+	fmt.Println(table.String())
+	return err
+}
+
 func (a *App) within(dir string, do func() error) error {
 	if dir == "." {
 		return do()
@@ -81,7 +242,7 @@ func (a *App) within(dir string, do func() error) error {
 	return appErr
 }
 
-func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error {
+func (a *App) visitStateFiles(fileOrDir string, do func(string, string) error) error {
 	desiredStateFiles, err := a.findDesiredStateFiles(fileOrDir)
 	if err != nil {
 		return appError("", err)
@@ -101,7 +262,12 @@ func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error {
 		a.Logger.Debugf("processing file \"%s\" in directory \"%s\"", file, dir)
 
 		err := a.within(dir, func() error {
-			return do(file)
+			absd, err := a.abs(dir)
+			if err != nil {
+				return err
+			}
+
+			return do(file, absd)
 		})
 		if err != nil {
 			return appError(fmt.Sprintf("in %s/%s", dir, file), err)
@@ -111,26 +277,41 @@ func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error {
 	return nil
 }
 
-func (a *App) loadDesiredStateFromYaml(file string) (*state.HelmState, error) {
+func (a *App) loadDesiredStateFromYaml(file string, opts ...LoadOpts) (*state.HelmState, error) {
 	ld := &desiredStateLoader{
-		readFile:  a.readFile,
-		env:       a.Env,
-		namespace: a.Namespace,
-		logger:    a.Logger,
-		abs:       a.abs,
+		readFile:   a.readFile,
+		fileExists: a.fileExists,
+		env:        a.Env,
+		namespace:  a.Namespace,
+		logger:     a.Logger,
+		abs:        a.abs,
 
 		Reverse:     a.Reverse,
 		KubeContext: a.KubeContext,
 		glob:        a.glob,
+		helm:        a.helmExecer,
+		valsRuntime: a.valsRuntime,
 	}
-	return ld.Load(file)
+
+	var op LoadOpts
+	if len(opts) > 0 {
+		op = opts[0]
+	}
+
+	return ld.Load(file, op)
 }
 
-func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error {
+func (a *App) visitStates(fileOrDir string, defOpts LoadOpts, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error {
 	noMatchInHelmfiles := true
 
-	err := a.visitStateFiles(fileOrDir, func(f string) error {
-		st, err := a.loadDesiredStateFromYaml(f)
+	err := a.visitStateFiles(fileOrDir, func(f, d string) error {
+		opts := defOpts.DeepCopy()
+
+		if opts.CalleePath == "" {
+			opts.CalleePath = f
+		}
+
+		st, err := a.loadDesiredStateFromYaml(f, opts)
 
 		sigs := make(chan os.Signal, 1)
 		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
@@ -150,7 +331,7 @@ func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge f
 
 		ctx := context{a, st}
 
-		helm := helmexec.New(a.Logger, a.KubeContext)
+		helm := a.helmExecer
 
 		if err != nil {
 			switch stateLoadErr := err.(type) {
@@ -166,16 +347,23 @@ func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge f
 				return ctx.wrapErrs(err)
 			}
 		}
-		st.Selectors = selector
+		st.Selectors = opts.Selectors
 
 		if len(st.Helmfiles) > 0 {
 			noMatchInSubHelmfiles := true
 			for i, m := range st.Helmfiles {
+				optsForNestedState := LoadOpts{
+					CalleePath:  filepath.Join(d, f),
+					Environment: m.Environment,
+				}
 				//assign parent selector to sub helm selector in legacy mode or do not inherit in experimental mode
 				if (m.Selectors == nil && !isExplicitSelectorInheritanceEnabled()) || m.SelectorsInherited {
-					m.Selectors = selector
+					optsForNestedState.Selectors = opts.Selectors
+				} else {
+					optsForNestedState.Selectors = m.Selectors
 				}
-				if err := a.VisitDesiredStates(m.Path, m.Selectors, converge); err != nil {
+
+				if err := a.visitStates(m.Path, optsForNestedState, converge); err != nil {
 					switch err.(type) {
 					case *NoMatchingHelmfileError:
 
@@ -193,8 +381,10 @@ func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge f
 		if tmplErr != nil {
 			return appError(fmt.Sprintf("failed executing release templates in \"%s\"", f), tmplErr)
 		}
+
 		processed, errs := converge(templated, helm)
 		noMatchInHelmfiles = noMatchInHelmfiles && !processed
+
 		return context{a, templated}.clean(errs)
 	})
 
@@ -209,9 +399,146 @@ func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge f
 	return nil
 }
 
-func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) []error) error {
+func (a *App) ForEachStateFiltered(do func(*Run) []error) error {
+	ctx := NewContext()
+	err := a.VisitDesiredStatesWithReleasesFiltered(a.FileOrDir, func(st *state.HelmState, helm helmexec.Interface) []error {
+		run := NewRun(st, helm, ctx)
 
-	err := a.VisitDesiredStates(fileOrDir, a.Selectors, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
+		return do(run)
+	})
+
+	if err != nil && a.ErrorHandler != nil {
+		return a.ErrorHandler(err)
+	}
+
+	return err
+}
+
+func (a *App) ForEachState(do func(*Run) (bool, []error)) error {
+	ctx := NewContext()
+	err := a.visitStatesWithSelectorsAndRemoteSupport(a.FileOrDir, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
+		run := NewRun(st, helm, ctx)
+		return do(run)
+	})
+
+	if err != nil && a.ErrorHandler != nil {
+		return a.ErrorHandler(err)
+	}
+
+	return err
+}
+
+func printBatches(batches [][]state.Release) string {
+	buf := &bytes.Buffer{}
+
+	w := new(tabwriter.Writer)
+
+	w.Init(buf, 0, 1, 1, ' ', 0)
+
+	fmt.Fprintln(w, "GROUP\tRELEASES")
+
+	for i, batch := range batches {
+		ids := []string{}
+		for _, r := range batch {
+			ids = append(ids, state.ReleaseToID(&r.ReleaseSpec))
+		}
+		fmt.Fprintf(w, "%d\t%s\n", i+1, strings.Join(ids, ", "))
+	}
+
+	w.Flush()
+
+	return buf.String()
+}
+
+func withDAG(templated *state.HelmState, helm helmexec.Interface, logger *zap.SugaredLogger, reverse bool, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) (bool, []error) {
+	batches, err := templated.PlanReleases(reverse)
+	if err != nil {
+		return false, []error{err}
+	}
+
+	numBatches := len(batches)
+
+	logger.Debugf("processing %d groups of releases in this order:\n%s", numBatches, printBatches(batches))
+
+	any := false
+
+	for i, batch := range batches {
+		var targets []state.ReleaseSpec
+
+		for _, marked := range batch {
+			targets = append(targets, marked.ReleaseSpec)
+		}
+
+		var releaseIds []string
+		for _, r := range targets {
+			releaseIds = append(releaseIds, state.ReleaseToID(&r))
+		}
+
+		logger.Debugf("processing releases in group %d/%d: %s", i+1, numBatches, strings.Join(releaseIds, ", "))
+
+		batchSt := *templated
+		batchSt.Releases = targets
+
+		processed, errs := converge(&batchSt, helm)
+
+		if errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+
+		any = any || processed
+	}
+
+	return any, nil
+}
+
+type Opts struct {
+	DAGEnabled bool
+}
+
+func (a *App) visitStatesWithSelectorsAndRemoteSupport(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error {
+	opts := LoadOpts{
+		Selectors: a.Selectors,
+	}
+
+	envvals := []interface{}{}
+
+	if a.ValuesFiles != nil {
+		for i := range a.ValuesFiles {
+			envvals = append(envvals, a.ValuesFiles[i])
+		}
+	}
+
+	if a.Set != nil {
+		envvals = append(envvals, a.Set)
+	}
+
+	if len(envvals) > 0 {
+		opts.Environment.OverrideValues = envvals
+	}
+
+	dir, err := a.getwd()
+	if err != nil {
+		return err
+	}
+
+	getter := &remote.GoGetter{Logger: a.Logger}
+
+	remote := &remote.Remote{
+		Logger:     a.Logger,
+		Home:       dir,
+		Getter:     getter,
+		ReadFile:   a.readFile,
+		DirExists:  a.directoryExistsAt,
+		FileExists: a.fileExistsAt,
+	}
+
+	a.remote = remote
+
+	return a.visitStates(fileOrDir, opts, converge)
+}
+
+func (a *App) Wrap(converge func(*state.HelmState, helmexec.Interface) []error) func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
+	return func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
 		if len(st.Selectors) > 0 {
 			err := st.FilterReleases()
 			if err != nil {
@@ -222,6 +549,7 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge
 		type Key struct {
 			TillerNamespace, Name string
 		}
+
 		releaseNameCounts := map[Key]int{}
 		for _, r := range st.Releases {
 			tillerNamespace := st.HelmDefaults.TillerNamespace
@@ -241,11 +569,15 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge
 		processed := len(st.Releases) != 0 && len(errs) == 0
 
 		return processed, errs
-	})
-	if err != nil {
-		return err
 	}
-	return nil
+}
+
+func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) []error) error {
+	f := a.Wrap(converge)
+
+	return a.visitStatesWithSelectorsAndRemoteSupport(fileOrDir, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
+		return f(st, helm)
+	})
 }
 
 func (a *App) findStateFilesInAbsPaths(specifiedPath string) ([]string, error) {
@@ -265,6 +597,15 @@ func (a *App) findStateFilesInAbsPaths(specifiedPath string) ([]string, error) {
 }
 
 func (a *App) findDesiredStateFiles(specifiedPath string) ([]string, error) {
+	path, err := a.remote.Locate(specifiedPath)
+	if err != nil {
+		return nil, fmt.Errorf("locate: %v", err)
+	}
+	if specifiedPath != path {
+		a.Logger.Debugf("fetched remote \"%s\" to local cache \"%s\" and loading the latter...", specifiedPath, path)
+	}
+	specifiedPath = path
+
 	var helmfileDir string
 	if specifiedPath != "" {
 		if a.fileExistsAt(specifiedPath) {
@@ -311,11 +652,469 @@ func (a *App) findDesiredStateFiles(specifiedPath string) ([]string, error) {
 	return files, nil
 }
 
+func (a *App) apply(r *Run, c ApplyConfigProvider) (bool, []error) {
+	st := r.state
+	helm := r.helm
+	ctx := r.ctx
+
+	allReleases := st.GetReleasesWithOverrides()
+
+	toApply, err := st.GetSelectedReleasesWithOverrides()
+	if err != nil {
+		return false, []error{err}
+	}
+	if len(toApply) == 0 {
+		return false, nil
+	}
+
+	// Do build deps and prepare only on selected releases so that we won't waste time
+	// on running various helm commands on unnecessary releases
+	st.Releases = toApply
+
+	if !c.SkipDeps() {
+		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+	}
+	if errs := st.PrepareReleases(helm, "apply"); errs != nil && len(errs) > 0 {
+		return false, errs
+	}
+
+	// helm must be 2.11+ and helm-diff should be provided `--detailed-exitcode` in order for `helmfile apply` to work properly
+	detailedExitCode := true
+
+	diffOpts := &state.DiffOpts{
+		NoColor: c.NoColor(),
+		Context: c.Context(),
+		Set:     c.Set(),
+	}
+
+	var changedReleases []state.ReleaseSpec
+	var deletingReleases []state.ReleaseSpec
+	var planningErrs []error
+
+	// TODO Better way to detect diff on only filtered releases
+	{
+		changedReleases, planningErrs = st.DiffReleases(helm, c.Values(), c.Concurrency(), detailedExitCode, c.SuppressSecrets(), false, diffOpts)
+
+		var err error
+		deletingReleases, err = st.DetectReleasesToBeDeletedForSync(helm, st.Releases)
+		if err != nil {
+			planningErrs = append(planningErrs, err)
+		}
+	}
+
+	fatalErrs := []error{}
+
+	for _, e := range planningErrs {
+		switch err := e.(type) {
+		case *state.ReleaseError:
+			if err.Code != 2 {
+				fatalErrs = append(fatalErrs, e)
+			}
+		default:
+			fatalErrs = append(fatalErrs, e)
+		}
+	}
+
+	if len(fatalErrs) > 0 {
+		return false, fatalErrs
+	}
+
+	releasesToBeDeleted := map[string]state.ReleaseSpec{}
+	for _, r := range deletingReleases {
+		id := state.ReleaseToID(&r)
+		releasesToBeDeleted[id] = r
+	}
+
+	releasesToBeUpdated := map[string]state.ReleaseSpec{}
+	for _, r := range changedReleases {
+		id := state.ReleaseToID(&r)
+
+		// If `helm-diff` detected changes but it is not being `helm delete`ed, we should run `helm upgrade`
+		if _, ok := releasesToBeDeleted[id]; !ok {
+			releasesToBeUpdated[id] = r
+		}
+	}
+
+	// sync only when there are changes
+	if len(releasesToBeUpdated) == 0 && len(releasesToBeDeleted) == 0 {
+		// TODO better way to get the logger
+		logger := c.Logger()
+		logger.Infof("")
+		logger.Infof("No affected releases")
+		return true, nil
+	}
+
+	names := []string{}
+	for _, r := range releasesToBeUpdated {
+		names = append(names, fmt.Sprintf("  %s (%s) UPDATED", r.Name, r.Chart))
+	}
+	for _, r := range releasesToBeDeleted {
+		names = append(names, fmt.Sprintf("  %s (%s) DELETED", r.Name, r.Chart))
+	}
+	// Make the output deterministic for testing purpose
+	sort.Strings(names)
+
+	infoMsg := fmt.Sprintf(`Affected releases are:
+%s
+`, strings.Join(names, "\n"))
+	confMsg := fmt.Sprintf(`%s
+Do you really want to apply?
+  Helmfile will apply all your changes, as shown above.
+
+`, infoMsg)
+	interactive := c.Interactive()
+	if !interactive {
+		a.Logger.Debug(infoMsg)
+	}
+
+	syncErrs := []error{}
+
+	affectedReleases := state.AffectedReleases{}
+
+	// Traverse DAG of all the releases so that we don't suffer from false-positive missing dependencies
+	st.Releases = allReleases
+
+	if !interactive || interactive && r.askForConfirmation(confMsg) {
+		r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
+
+		// We deleted releases by traversing the DAG in reverse order
+		if len(releasesToBeDeleted) > 0 {
+			_, deletionErrs := withDAG(st, helm, a.Logger, true, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error {
+				var rs []state.ReleaseSpec
+
+				for _, r := range subst.Releases {
+					if r2, ok := releasesToBeDeleted[state.ReleaseToID(&r)]; ok {
+						rs = append(rs, r2)
+					}
+				}
+
+				subst.Releases = rs
+
+				return subst.DeleteReleasesForSync(&affectedReleases, helm, c.Concurrency())
+			}))
+
+			if deletionErrs != nil && len(deletionErrs) > 0 {
+				syncErrs = append(syncErrs, deletionErrs...)
+			}
+		}
+
+		// We upgrade releases by traversing the DAG
+		if len(releasesToBeUpdated) > 0 {
+			_, updateErrs := withDAG(st, helm, a.Logger, false, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error {
+				var rs []state.ReleaseSpec
+
+				for _, r := range subst.Releases {
+					if r2, ok := releasesToBeUpdated[state.ReleaseToID(&r)]; ok {
+						rs = append(rs, r2)
+					}
+				}
+
+				subst.Releases = rs
+
+				syncOpts := state.SyncOpts{
+					Set: c.Set(),
+				}
+				return subst.SyncReleases(&affectedReleases, helm, c.Values(), c.Concurrency(), &syncOpts)
+			}))
+
+			if updateErrs != nil && len(updateErrs) > 0 {
+				syncErrs = append(syncErrs, updateErrs...)
+			}
+		}
+	}
+
+	affectedReleases.DisplayAffectedReleases(c.Logger())
+	return true, syncErrs
+}
+
+func (a *App) delete(r *Run, purge bool, c DestroyConfigProvider) (bool, []error) {
+	st := r.state
+	helm := r.helm
+
+	affectedReleases := state.AffectedReleases{}
+
+	toSync, err := st.GetSelectedReleasesWithOverrides()
+	if err != nil {
+		return false, []error{err}
+	}
+	if len(toSync) == 0 {
+		return false, nil
+	}
+
+	toDelete, err := st.DetectReleasesToBeDeleted(helm, toSync)
+	if err != nil {
+		return false, []error{err}
+	}
+
+	releasesToDelete := map[string]state.ReleaseSpec{}
+	for _, r := range toDelete {
+		id := state.ReleaseToID(&r)
+		releasesToDelete[id] = r
+	}
+
+	names := make([]string, len(toSync))
+	for i, r := range toSync {
+		names[i] = fmt.Sprintf("  %s (%s)", r.Name, r.Chart)
+	}
+
+	var errs []error
+
+	msg := fmt.Sprintf(`Affected releases are:
+%s
+
+Do you really want to delete?
+  Helmfile will delete all your releases, as shown above.
+
+`, strings.Join(names, "\n"))
+	interactive := c.Interactive()
+	if !interactive || interactive && r.askForConfirmation(msg) {
+		r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
+
+		if len(releasesToDelete) > 0 {
+			_, deletionErrs := withDAG(st, helm, a.Logger, true, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error {
+				var rs []state.ReleaseSpec
+
+				for _, r := range subst.Releases {
+					if _, ok := releasesToDelete[state.ReleaseToID(&r)]; ok {
+						rs = append(rs, r)
+					}
+				}
+
+				subst.Releases = rs
+
+				return subst.DeleteReleases(&affectedReleases, helm, c.Concurrency(), purge)
+			}))
+
+			if deletionErrs != nil && len(deletionErrs) > 0 {
+				errs = append(errs, deletionErrs...)
+			}
+		}
+	}
+	affectedReleases.DisplayAffectedReleases(c.Logger())
+	return true, errs
+}
+
+func (a *App) sync(r *Run, c SyncConfigProvider) (bool, []error) {
+	st := r.state
+	helm := r.helm
+	ctx := r.ctx
+
+	allReleases := st.GetReleasesWithOverrides()
+
+	toSync, err := st.GetSelectedReleasesWithOverrides()
+	if err != nil {
+		return false, []error{err}
+	}
+	if len(toSync) == 0 {
+		return false, nil
+	}
+
+	// Do build deps and prepare only on selected releases so that we won't waste time
+	// on running various helm commands on unnecessary releases
+	st.Releases = toSync
+
+	if !c.SkipDeps() {
+		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+	}
+	if errs := st.PrepareReleases(helm, "sync"); errs != nil && len(errs) > 0 {
+		return false, errs
+	}
+
+	toDelete, err := st.DetectReleasesToBeDeletedForSync(helm, toSync)
+	if err != nil {
+		return false, []error{err}
+	}
+
+	releasesToDelete := map[string]state.ReleaseSpec{}
+	for _, r := range toDelete {
+		id := state.ReleaseToID(&r)
+		releasesToDelete[id] = r
+	}
+
+	var toUpdate []state.ReleaseSpec
+	for _, r := range toSync {
+		if _, deleted := releasesToDelete[state.ReleaseToID(&r)]; !deleted {
+			toUpdate = append(toUpdate, r)
+		}
+	}
+
+	releasesToUpdate := map[string]state.ReleaseSpec{}
+	for _, r := range toUpdate {
+		id := state.ReleaseToID(&r)
+		releasesToUpdate[id] = r
+	}
+
+	names := []string{}
+	for _, r := range releasesToUpdate {
+		names = append(names, fmt.Sprintf("  %s (%s) UPDATED", r.Name, r.Chart))
+	}
+	for _, r := range releasesToDelete {
+		names = append(names, fmt.Sprintf("  %s (%s) DELETED", r.Name, r.Chart))
+	}
+	// Make the output deterministic for testing purpose
+	sort.Strings(names)
+
+	infoMsg := fmt.Sprintf(`Affected releases are:
+%s
+`, strings.Join(names, "\n"))
+
+	a.Logger.Info(infoMsg)
+
+	var errs []error
+
+	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
+
+	// Traverse DAG of all the releases so that we don't suffer from false-positive missing dependencies
+	st.Releases = allReleases
+
+	affectedReleases := state.AffectedReleases{}
+
+	if len(releasesToDelete) > 0 {
+		_, deletionErrs := withDAG(st, helm, a.Logger, true, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error {
+			var rs []state.ReleaseSpec
+
+			for _, r := range subst.Releases {
+				if r2, ok := releasesToDelete[state.ReleaseToID(&r)]; ok {
+					rs = append(rs, r2)
+				}
+			}
+
+			subst.Releases = rs
+
+			return subst.DeleteReleasesForSync(&affectedReleases, helm, c.Concurrency())
+		}))
+
+		if deletionErrs != nil && len(deletionErrs) > 0 {
+			errs = append(errs, deletionErrs...)
+		}
+	}
+
+	if len(releasesToUpdate) > 0 {
+		_, syncErrs := withDAG(st, helm, a.Logger, false, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error {
+			var rs []state.ReleaseSpec
+
+			for _, r := range subst.Releases {
+				if r2, ok := releasesToUpdate[state.ReleaseToID(&r)]; ok {
+					rs = append(rs, r2)
+				}
+			}
+
+			subst.Releases = rs
+
+			opts := &state.SyncOpts{
+				Set: c.Set(),
+			}
+			return subst.SyncReleases(&affectedReleases, helm, c.Values(), c.Concurrency(), opts)
+		}))
+
+		if syncErrs != nil && len(syncErrs) > 0 {
+			errs = append(errs, syncErrs...)
+		}
+	}
+	affectedReleases.DisplayAffectedReleases(c.Logger())
+	return true, errs
+}
+
+func (a *App) template(r *Run, c TemplateConfigProvider) (bool, []error) {
+	st := r.state
+	helm := r.helm
+	ctx := r.ctx
+
+	allReleases := st.GetReleasesWithOverrides()
+
+	toRender, err := st.GetSelectedReleasesWithOverrides()
+	if err != nil {
+		return false, []error{err}
+	}
+	if len(toRender) == 0 {
+		return false, nil
+	}
+
+	// Do build deps and prepare only on selected releases so that we won't waste time
+	// on running various helm commands on unnecessary releases
+	st.Releases = toRender
+
+	if !c.SkipDeps() {
+		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
+			return false, errs
+		}
+	}
+	if errs := st.PrepareReleases(helm, "template"); errs != nil && len(errs) > 0 {
+		return false, errs
+	}
+
+	releasesToRender := map[string]state.ReleaseSpec{}
+	for _, r := range toRender {
+		id := state.ReleaseToID(&r)
+		releasesToRender[id] = r
+	}
+
+	names := make([]string, len(toRender))
+	for i, r := range toRender {
+		names[i] = fmt.Sprintf("  %s (%s)", r.Name, r.Chart)
+	}
+
+	var errs []error
+
+	// Traverse DAG of all the releases so that we don't suffer from false-positive missing dependencies
+	st.Releases = allReleases
+
+	if len(releasesToRender) > 0 {
+		_, templateErrs := withDAG(st, helm, a.Logger, false, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error {
+			var rs []state.ReleaseSpec
+
+			for _, r := range subst.Releases {
+				if r2, ok := releasesToRender[state.ReleaseToID(&r)]; ok {
+					rs = append(rs, r2)
+				}
+			}
+
+			subst.Releases = rs
+
+			args := argparser.GetArgs(c.Args(), st)
+			opts := &state.TemplateOpts{
+				Set: c.Set(),
+			}
+			return subst.TemplateReleases(helm, c.OutputDir(), c.Values(), args, c.Concurrency(), opts)
+		}))
+
+		if templateErrs != nil && len(templateErrs) > 0 {
+			errs = append(errs, templateErrs...)
+		}
+	}
+	return true, errs
+}
+
 func fileExistsAt(path string) bool {
 	fileInfo, err := os.Stat(path)
 	return err == nil && fileInfo.Mode().IsRegular()
 }
 
+func fileExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
 func directoryExistsAt(path string) bool {
 	fileInfo, err := os.Stat(path)
 	return err == nil && fileInfo.Mode().IsDir()

pkg/app/ask.go

@@ -1,4 +1,4 @@
-package main
+package app
 
 import (
 	"bufio"
@@ -11,7 +11,7 @@ import (
 // Copyright (c) 2017 Roland Singer [roland.singer@desertbit.com]
 //
 // Shamelessly borrowed from @r0l1's awesome work that is available at https://gist.github.com/r0l1/3dcbb0c8f6cfe9c66ab8008f55f8f28b
-func askForConfirmation(s string) bool {
+func AskForConfirmation(s string) bool {
 	reader := bufio.NewReader(os.Stdin)
 
 	for {

pkg/app/config.go

@@ -0,0 +1,147 @@
+package app
+
+import "go.uber.org/zap"
+
+type ConfigProvider interface {
+	Args() string
+	HelmBinary() string
+
+	FileOrDir() string
+	KubeContext() string
+	Namespace() string
+	Selectors() []string
+	StateValuesSet() map[string]interface{}
+	StateValuesFiles() []string
+	Env() string
+
+	loggingConfig
+}
+
+type DeprecatedChartsConfigProvider interface {
+	Values() []string
+
+	concurrencyConfig
+	loggingConfig
+}
+
+type DepsConfigProvider interface {
+	Args() string
+	SkipRepos() bool
+}
+
+type ReposConfigProvider interface {
+	Args() string
+}
+
+type ApplyConfigProvider interface {
+	Args() string
+
+	Values() []string
+	Set() []string
+	SkipDeps() bool
+
+	SuppressSecrets() bool
+
+	NoColor() bool
+	Context() int
+
+	concurrencyConfig
+	interactive
+	loggingConfig
+}
+
+type SyncConfigProvider interface {
+	Args() string
+
+	Values() []string
+	Set() []string
+	SkipDeps() bool
+
+	concurrencyConfig
+	loggingConfig
+}
+
+type DiffConfigProvider interface {
+	Args() string
+
+	Values() []string
+	Set() []string
+	SkipDeps() bool
+
+	SuppressSecrets() bool
+
+	DetailedExitcode() bool
+	NoColor() bool
+	Context() int
+
+	concurrencyConfig
+}
+
+type DeleteConfigProvider interface {
+	Args() string
+
+	Purge() bool
+
+	interactive
+	loggingConfig
+	concurrencyConfig
+}
+
+type DestroyConfigProvider interface {
+	Args() string
+
+	interactive
+	loggingConfig
+	concurrencyConfig
+}
+
+type TestConfigProvider interface {
+	Args() string
+
+	Timeout() int
+	Cleanup() bool
+
+	concurrencyConfig
+}
+
+type LintConfigProvider interface {
+	Args() string
+
+	Values() []string
+	Set() []string
+	SkipDeps() bool
+
+	concurrencyConfig
+}
+
+type TemplateConfigProvider interface {
+	Args() string
+
+	Values() []string
+	Set() []string
+	SkipDeps() bool
+	OutputDir() string
+
+	concurrencyConfig
+}
+
+type StatusesConfigProvider interface {
+	Args() string
+
+	concurrencyConfig
+}
+
+type StateConfigProvider interface {
+}
+
+type concurrencyConfig interface {
+	Concurrency() int
+}
+
+type loggingConfig interface {
+	Logger() *zap.SugaredLogger
+}
+
+type interactive interface {
+	Interactive() bool
+}

pkg/app/context.go

@@ -1,6 +1,8 @@
 package app
 
-import "github.com/roboll/helmfile/state"
+import (
+	"github.com/roboll/helmfile/pkg/state"
+)
 
 type Context struct {
 	updatedRepos map[string]struct{}

pkg/app/desired_state_file_loader.go

@@ -4,14 +4,15 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"github.com/imdario/mergo"
-	"github.com/roboll/helmfile/environment"
-	"github.com/roboll/helmfile/state"
-	"go.uber.org/zap"
-	"log"
-	"os"
 	"path/filepath"
 	"sort"
+
+	"github.com/imdario/mergo"
+	"github.com/roboll/helmfile/pkg/environment"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/state"
+	"github.com/variantdev/vals"
+	"go.uber.org/zap"
 )
 
 type desiredStateLoader struct {
@@ -21,18 +22,74 @@ type desiredStateLoader struct {
 	env       string
 	namespace string
 
-	readFile func(string) ([]byte, error)
-	abs      func(string) (string, error)
-	glob     func(string) ([]string, error)
+	readFile   func(string) ([]byte, error)
+	fileExists func(string) (bool, error)
+	abs        func(string) (string, error)
+	glob       func(string) ([]string, error)
 
-	logger *zap.SugaredLogger
+	logger      *zap.SugaredLogger
+	helm        helmexec.Interface
+	valsRuntime vals.Evaluator
 }
 
-func (ld *desiredStateLoader) Load(f string) (*state.HelmState, error) {
-	return ld.loadFile(filepath.Dir(f), filepath.Base(f), true)
+func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, error) {
+	var overrodeEnv *environment.Environment
+
+	args := opts.Environment.OverrideValues
+
+	if len(args) > 0 {
+		if opts.CalleePath == "" {
+			return nil, fmt.Errorf("bug: opts.CalleePath was nil: f=%s, opts=%v", f, opts)
+		}
+		storage := state.NewStorage(opts.CalleePath, ld.logger, ld.glob)
+		envld := state.NewEnvironmentValuesLoader(storage, ld.readFile, ld.logger)
+		handler := state.MissingFileHandlerError
+		vals, err := envld.LoadEnvironmentValues(&handler, args)
+		if err != nil {
+			return nil, err
+		}
+
+		overrodeEnv = &environment.Environment{
+			Name:   ld.env,
+			Values: vals,
+		}
+	}
+
+	st, err := ld.loadFileWithOverrides(nil, overrodeEnv, filepath.Dir(f), filepath.Base(f), true)
+	if err != nil {
+		return nil, err
+	}
+
+	if ld.Reverse {
+		rev := func(i, j int) bool {
+			return j < i
+		}
+		sort.Slice(st.Releases, rev)
+		sort.Slice(st.Helmfiles, rev)
+	}
+
+	if ld.KubeContext != "" {
+		if st.HelmDefaults.KubeContext != "" {
+			return nil, errors.New("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.")
+		}
+		st.HelmDefaults.KubeContext = ld.KubeContext
+	}
+
+	if ld.namespace != "" {
+		if st.OverrideNamespace != "" {
+			return nil, errors.New("err: Cannot use option --namespace and set attribute namespace.")
+		}
+		st.OverrideNamespace = ld.namespace
+	}
+
+	return st, nil
 }
 
-func (ld *desiredStateLoader) loadFile(baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
+func (ld *desiredStateLoader) loadFile(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
+	return ld.loadFileWithOverrides(inheritedEnv, nil, baseDir, file, evaluateBases)
+}
+
+func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
 	var f string
 	if filepath.IsAbs(file) {
 		f = file
@@ -51,6 +108,8 @@ func (ld *desiredStateLoader) loadFile(baseDir, file string, evaluateBases bool)
 
 	if !experimentalModeEnabled() || ext == ".gotmpl" {
 		self, err = ld.renderAndLoad(
+			inheritedEnv,
+			overrodeEnv,
 			baseDir,
 			f,
 			fileBytes,
@@ -62,101 +121,44 @@ func (ld *desiredStateLoader) loadFile(baseDir, file string, evaluateBases bool)
 			baseDir,
 			file,
 			evaluateBases,
+			inheritedEnv,
+			overrodeEnv,
 		)
 	}
 
 	return self, err
 }
 
-func (a *desiredStateLoader) load(yaml []byte, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
-	c := state.NewCreator(a.logger, a.readFile, a.abs)
-	st, err := c.ParseAndLoadEnv(yaml, baseDir, file, a.env)
+func (a *desiredStateLoader) underlying() *state.StateCreator {
+	c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.helm, a.valsRuntime)
+	c.LoadFile = a.loadFile
+	return c
+}
+
+func (a *desiredStateLoader) load(yaml []byte, baseDir, file string, evaluateBases bool, env, overrodeEnv *environment.Environment) (*state.HelmState, error) {
+	merged, err := env.Merge(overrodeEnv)
 	if err != nil {
 		return nil, err
 	}
 
-	helmfiles := []state.SubHelmfileSpec{}
-	for _, hf := range st.Helmfiles {
-		globPattern := hf.Path
-		var absPathPattern string
-		if filepath.IsAbs(globPattern) {
-			absPathPattern = globPattern
-		} else {
-			absPathPattern = st.JoinBase(globPattern)
-		}
-		matches, err := a.glob(absPathPattern)
-		if err != nil {
-			return nil, fmt.Errorf("failed processing %s: %v", globPattern, err)
-		}
-		sort.Strings(matches)
-		for _, match := range matches {
-			newHelmfile := hf
-			newHelmfile.Path = match
-			helmfiles = append(helmfiles, newHelmfile)
-		}
+	st, err := a.underlying().ParseAndLoad(yaml, baseDir, file, a.env, evaluateBases, merged)
+	if err != nil {
+		return nil, err
+	}
 
+	helmfiles, err := st.ExpandedHelmfiles()
+	if err != nil {
+		return nil, err
 	}
 	st.Helmfiles = helmfiles
 
-	if a.Reverse {
-		rev := func(i, j int) bool {
-			return j < i
-		}
-		sort.Slice(st.Releases, rev)
-		sort.Slice(st.Helmfiles, rev)
-	}
-
-	if a.KubeContext != "" {
-		if st.HelmDefaults.KubeContext != "" {
-			log.Printf("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.")
-			os.Exit(1)
-		}
-		st.HelmDefaults.KubeContext = a.KubeContext
-	}
-	if a.namespace != "" {
-		if st.Namespace != "" {
-			log.Printf("err: Cannot use option --namespace and set attribute namespace.")
-			os.Exit(1)
-		}
-		st.Namespace = a.namespace
-	}
-
-	if err != nil {
-		return nil, err
-	}
-
-	if !evaluateBases {
-		if len(st.Bases) > 0 {
-			return nil, errors.New("nested `base` helmfile is unsupported. please submit a feature request if you need this!")
-		}
-
-		return st, nil
-	}
-
-	layers := []*state.HelmState{}
-	for _, b := range st.Bases {
-		base, err := a.loadFile(baseDir, b, false)
-		if err != nil {
-			return nil, err
-		}
-		layers = append(layers, base)
-	}
-	layers = append(layers, st)
-
-	for i := 1; i < len(layers); i++ {
-		if err := mergo.Merge(layers[0], layers[i], mergo.WithAppendSlice); err != nil {
-			return nil, err
-		}
-	}
-
-	return layers[0], nil
+	return st, nil
 }
 
-func (ld *desiredStateLoader) renderAndLoad(baseDir, filename string, content []byte, evaluateBases bool) (*state.HelmState, error) {
+func (ld *desiredStateLoader) renderAndLoad(env, overrodeEnv *environment.Environment, baseDir, filename string, content []byte, evaluateBases bool) (*state.HelmState, error) {
 	parts := bytes.Split(content, []byte("\n---\n"))
 
 	var finalState *state.HelmState
-	var env *environment.Environment
 
 	for i, part := range parts {
 		var yamlBuf *bytes.Buffer
@@ -164,13 +166,13 @@ func (ld *desiredStateLoader) renderAndLoad(baseDir, filename string, content []
 
 		id := fmt.Sprintf("%s.part.%d", filename, i)
 
-		if env == nil {
+		if env == nil && overrodeEnv == nil {
 			yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part)
 			if err != nil {
 				return nil, fmt.Errorf("error during %s parsing: %v", id, err)
 			}
 		} else {
-			yamlBuf, err = ld.renderTemplatesToYaml(baseDir, id, part, *env)
+			yamlBuf, err = ld.renderTemplatesToYamlWithEnv(baseDir, id, part, env, overrodeEnv)
 			if err != nil {
 				return nil, fmt.Errorf("error during %s parsing: %v", id, err)
 			}
@@ -181,6 +183,8 @@ func (ld *desiredStateLoader) renderAndLoad(baseDir, filename string, content []
 			baseDir,
 			filename,
 			evaluateBases,
+			env,
+			overrodeEnv,
 		)
 		if err != nil {
 			return nil, err
@@ -189,7 +193,7 @@ func (ld *desiredStateLoader) renderAndLoad(baseDir, filename string, content []
 		if finalState == nil {
 			finalState = currentState
 		} else {
-			if err := mergo.Merge(finalState, currentState, mergo.WithAppendSlice); err != nil {
+			if err := mergo.Merge(finalState, currentState, mergo.WithOverride); err != nil {
 				return nil, err
 			}
 		}

pkg/app/load_opts.go

@@ -0,0 +1,28 @@
+package app
+
+import (
+	"github.com/roboll/helmfile/pkg/state"
+	"gopkg.in/yaml.v2"
+)
+
+type LoadOpts struct {
+	Selectors   []string
+	Environment state.SubhelmfileEnvironmentSpec
+
+	// CalleePath is the absolute path to the file being loaded
+	CalleePath string
+}
+
+func (o LoadOpts) DeepCopy() LoadOpts {
+	bytes, err := yaml.Marshal(o)
+	if err != nil {
+		panic(err)
+	}
+
+	new := LoadOpts{}
+	if err := yaml.Unmarshal(bytes, &new); err != nil {
+		panic(err)
+	}
+
+	return new
+}

pkg/app/run.go

@@ -0,0 +1,125 @@
+package app
+
+import (
+	"github.com/roboll/helmfile/pkg/argparser"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/state"
+)
+
+type Run struct {
+	state *state.HelmState
+	helm  helmexec.Interface
+	ctx   Context
+
+	Ask func(string) bool
+}
+
+func NewRun(st *state.HelmState, helm helmexec.Interface, ctx Context) *Run {
+	return &Run{state: st, helm: helm, ctx: ctx}
+}
+
+func (r *Run) askForConfirmation(msg string) bool {
+	if r.Ask != nil {
+		return r.Ask(msg)
+	}
+	return AskForConfirmation(msg)
+}
+
+func (r *Run) Deps(c DepsConfigProvider) []error {
+	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
+
+	if !c.SkipRepos() {
+		if errs := r.ctx.SyncReposOnce(r.state, r.helm); errs != nil && len(errs) > 0 {
+			return errs
+		}
+	}
+
+	return r.state.UpdateDeps(r.helm)
+}
+
+func (r *Run) Repos(c ReposConfigProvider) []error {
+	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
+
+	return r.ctx.SyncReposOnce(r.state, r.helm)
+}
+
+func (r *Run) DeprecatedSyncCharts(c DeprecatedChartsConfigProvider) []error {
+	st := r.state
+	helm := r.helm
+
+	affectedReleases := state.AffectedReleases{}
+	errs := st.SyncReleases(&affectedReleases, helm, c.Values(), c.Concurrency())
+	affectedReleases.DisplayAffectedReleases(c.Logger())
+	return errs
+}
+
+func (r *Run) Status(c StatusesConfigProvider) []error {
+	workers := c.Concurrency()
+
+	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
+
+	return r.state.ReleaseStatuses(r.helm, workers)
+}
+
+func (r *Run) Diff(c DiffConfigProvider) []error {
+	st := r.state
+	helm := r.helm
+	ctx := r.ctx
+
+	if !c.SkipDeps() {
+		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
+			return errs
+		}
+		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
+			return errs
+		}
+	}
+	if errs := st.PrepareReleases(helm, "diff"); errs != nil && len(errs) > 0 {
+		return errs
+	}
+
+	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
+
+	opts := &state.DiffOpts{
+		Context: c.Context(),
+		NoColor: c.NoColor(),
+		Set:     c.Set(),
+	}
+	_, errs := st.DiffReleases(helm, c.Values(), c.Concurrency(), c.DetailedExitcode(), c.SuppressSecrets(), true, opts)
+	return errs
+}
+
+func (r *Run) Test(c TestConfigProvider) []error {
+	cleanup := c.Cleanup()
+	timeout := c.Timeout()
+	concurrency := c.Concurrency()
+
+	r.helm.SetExtraArgs(argparser.GetArgs(c.Args(), r.state)...)
+
+	return r.state.TestReleases(r.helm, cleanup, timeout, concurrency)
+}
+
+func (r *Run) Lint(c LintConfigProvider) []error {
+	st := r.state
+	helm := r.helm
+	ctx := r.ctx
+
+	values := c.Values()
+	args := argparser.GetArgs(c.Args(), st)
+	workers := c.Concurrency()
+	if !c.SkipDeps() {
+		if errs := ctx.SyncReposOnce(st, helm); errs != nil && len(errs) > 0 {
+			return errs
+		}
+		if errs := st.BuildDeps(helm); errs != nil && len(errs) > 0 {
+			return errs
+		}
+	}
+	if errs := st.PrepareReleases(helm, "lint"); errs != nil && len(errs) > 0 {
+		return errs
+	}
+	opts := &state.LintOpts{
+		Set: c.Set(),
+	}
+	return st.LintReleases(helm, values, args, workers, opts)
+}

pkg/app/two_pass_renderer.go

@@ -3,10 +3,10 @@ package app
 import (
 	"bytes"
 	"fmt"
-	"github.com/imdario/mergo"
-	"github.com/roboll/helmfile/environment"
-	"github.com/roboll/helmfile/state"
-	"github.com/roboll/helmfile/tmpl"
+	"github.com/google/go-cmp/cmp"
+	"github.com/roboll/helmfile/pkg/environment"
+	"github.com/roboll/helmfile/pkg/state"
+	"github.com/roboll/helmfile/pkg/tmpl"
 	"strings"
 )
 
@@ -19,71 +19,120 @@ func prependLineNumbers(text string) string {
 	return buf.String()
 }
 
-func (r *desiredStateLoader) renderEnvironment(firstPassEnv environment.Environment, baseDir, filename string, content []byte) environment.Environment {
-	tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
+func (r *desiredStateLoader) renderPrestate(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) (*environment.Environment, *state.HelmState) {
+	tmplData := state.EnvironmentTemplateData{
+		Environment: *firstPassEnv,
+		Namespace:   r.namespace,
+		Values:      map[string]interface{}{},
+	}
 	firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData)
 
 	// parse as much as we can, tolerate errors, this is a preparse
 	yamlBuf, err := firstPassRenderer.RenderTemplateContentToBuffer(content)
 	if err != nil && r.logger != nil {
 		r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", filename, prependLineNumbers(string(content)))
+		r.logger.Debugf("template syntax error: %v", err)
 		if yamlBuf == nil { // we have a template syntax error, let the second parse report
-			r.logger.Debugf("template syntax error: %v", err)
-			return firstPassEnv
+			return firstPassEnv, nil
 		}
 	}
-	c := state.NewCreator(r.logger, r.readFile, r.abs)
+	yamlData := yamlBuf.String()
+	if r.logger != nil {
+		r.logger.Debugf("first-pass rendering output of \"%s\":\n%s", filename, prependLineNumbers(yamlData))
+	}
+
+	// Work-around for https://github.com/golang/go/issues/24963
+	sanitized := strings.ReplaceAll(yamlData, "<no value>", "")
+
+	if len(yamlData) != len(sanitized) {
+		msg := "replaced <no value>s to workaround https://github.com/golang/go/issues/24963 to address https://github.com/roboll/helmfile/issues/553:\n%s"
+		r.logger.Debugf(msg, cmp.Diff(yamlData, sanitized))
+	}
+
+	c := r.underlying()
 	c.Strict = false
 	// create preliminary state, as we may have an environment. Tolerate errors.
-	prestate, err := c.ParseAndLoadEnv(yamlBuf.Bytes(), baseDir, filename, r.env)
+	prestate, err := c.ParseAndLoad([]byte(sanitized), baseDir, filename, r.env, false, firstPassEnv)
 	if err != nil && r.logger != nil {
 		switch err.(type) {
 		case *state.StateLoadError:
-			r.logger.Infof("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err)
+			r.logger.Debugf("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err)
 		}
 		r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", filename, prependLineNumbers(yamlBuf.String()))
 	}
 
 	if prestate != nil {
-		intEnv := environment.Environment{Name: firstPassEnv.Name}
-		if err := mergo.Merge(&intEnv, &firstPassEnv, mergo.WithAppendSlice); err != nil {
-			r.logger.Debugf("error in first-pass rendering: result of \"%s\": %v", filename, err)
-			return firstPassEnv
-		}
-		if err := mergo.Merge(&intEnv, &prestate.Env, mergo.WithAppendSlice); err != nil {
-			r.logger.Debugf("error in first-pass rendering: result of \"%s\": %v", filename, err)
-			return firstPassEnv
-		}
-		firstPassEnv = intEnv
-	}
-	return firstPassEnv
-}
-
-func (r *desiredStateLoader) renderTemplatesToYaml(baseDir, filename string, content []byte, context ...environment.Environment) (*bytes.Buffer, error) {
-	var env environment.Environment
-
-	if len(context) > 0 {
-		env = context[0]
-	} else {
-		env = environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
+		firstPassEnv = &prestate.Env
 	}
 
-	return r.twoPassRenderTemplateToYaml(env, baseDir, filename, content)
+	return firstPassEnv, prestate
 }
 
-func (r *desiredStateLoader) twoPassRenderTemplateToYaml(initEnv environment.Environment, baseDir, filename string, content []byte) (*bytes.Buffer, error) {
+type RenderOpts struct {
+}
+
+func (r *desiredStateLoader) renderTemplatesToYaml(baseDir, filename string, content []byte) (*bytes.Buffer, error) {
+	env := &environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
+
+	return r.renderTemplatesToYamlWithEnv(baseDir, filename, content, env, nil)
+}
+
+func (r *desiredStateLoader) renderTemplatesToYamlWithEnv(baseDir, filename string, content []byte, inherited, overrode *environment.Environment) (*bytes.Buffer, error) {
+	return r.twoPassRenderTemplateToYaml(inherited, overrode, baseDir, filename, content)
+}
+
+func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *environment.Environment, baseDir, filename string, content []byte) (*bytes.Buffer, error) {
 	// try a first pass render. This will always succeed, but can produce a limited env
 	if r.logger != nil {
-		r.logger.Debugf("first-pass rendering input of \"%s\": %v", filename, initEnv)
+		r.logger.Debugf("first-pass rendering starting for \"%s\": inherited=%v, overrode=%v", filename, inherited, overrode)
 	}
 
-	firstPassEnv := r.renderEnvironment(initEnv, baseDir, filename, content)
+	initEnv, err := inherited.Merge(overrode)
+	if err != nil {
+		return nil, err
+	}
 
 	if r.logger != nil {
-		r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, firstPassEnv)
+		r.logger.Debugf("first-pass uses: %v", initEnv)
 	}
 
-	tmplData := state.EnvironmentTemplateData{Environment: firstPassEnv, Namespace: r.namespace}
+	renderedEnv, prestate := r.renderPrestate(initEnv, baseDir, filename, content)
+
+	if r.logger != nil {
+		r.logger.Debugf("first-pass produced: %v", renderedEnv)
+	}
+
+	finalEnv, err := inherited.Merge(renderedEnv)
+	if err != nil {
+		return nil, err
+	}
+
+	finalEnv, err = finalEnv.Merge(overrode)
+	if err != nil {
+		return nil, err
+	}
+
+	if r.logger != nil {
+		r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, *finalEnv)
+	}
+
+	vals := map[string]interface{}{}
+	if prestate != nil {
+		prestate.Env = *finalEnv
+		vals, err = prestate.Values()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if prestate != nil {
+		r.logger.Debugf("vals:\n%v\ndefaultVals:%v", vals, prestate.DefaultValues)
+	}
+
+	tmplData := state.EnvironmentTemplateData{
+		Environment: *finalEnv,
+		Namespace:   r.namespace,
+		Values:      vals,
+	}
 	secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData)
 	yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content)
 	if err != nil {

pkg/app/two_pass_renderer_test.go

@@ -1,25 +1,28 @@
 package app
 
 import (
-	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
-	"github.com/roboll/helmfile/helmexec"
-	"github.com/roboll/helmfile/state"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/state"
+	"github.com/roboll/helmfile/pkg/testhelper"
+
 	"gopkg.in/yaml.v2"
 )
 
-func makeLoader(readFile func(string) ([]byte, error), env string) *desiredStateLoader {
+func makeLoader(files map[string]string, env string) (*desiredStateLoader, *testhelper.TestFs) {
+	testfs := testhelper.NewTestFs(files)
 	return &desiredStateLoader{
-		readFile:  readFile,
-		env:       env,
-		namespace: "namespace",
-		logger:    helmexec.NewLogger(os.Stdout, "debug"),
-		abs:       filepath.Abs,
-	}
+		env:        env,
+		namespace:  "namespace",
+		logger:     helmexec.NewLogger(os.Stdout, "debug"),
+		readFile:   testfs.ReadFile,
+		fileExists: testfs.FileExists,
+		abs:        testfs.Abs,
+		glob:       testfs.Glob,
+	}, testfs
 }
 
 func TestReadFromYaml_MakeEnvironmentHasNoSideEffects(t *testing.T) {
@@ -36,30 +39,21 @@ releases:
   chart: mychart1
 `)
 
-	fileReaderCalls := 0
-	// make a reader that returns a simulated context
-	fileReader := func(filename string) ([]byte, error) {
-		expectedFilename := filepath.Clean("default/values.yaml")
-		if !strings.HasSuffix(filename, expectedFilename) {
-			return nil, fmt.Errorf("unexpected filename: expected=%s, actual=%s", expectedFilename, filename)
-		}
-		fileReaderCalls++
-		if fileReaderCalls == 2 {
-			return []byte("SecondPass"), nil
-		}
-		return []byte(""), nil
+	files := map[string]string{
+		"/path/to/default/values.yaml":       ``,
+		"/path/to/other/default/values.yaml": `SecondPass`,
 	}
 
-	r := makeLoader(fileReader, "staging")
+	r, testfs := makeLoader(files, "staging")
 	yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent)
 	if err != nil {
-		t.Errorf("unexpected error: %v", err)
+		t.Fatalf("unexpected error: %v", err)
 	}
 
 	var state state.HelmState
 	err = yaml.Unmarshal(yamlBuf.Bytes(), &state)
 
-	if fileReaderCalls > 2 {
+	if testfs.FileReaderCalls() > 2 {
 		t.Error("reader should be called only twice")
 	}
 
@@ -70,10 +64,10 @@ releases:
 
 func TestReadFromYaml_RenderTemplate(t *testing.T) {
 
-	defaultValuesYaml := []byte(`
+	defaultValuesYaml := `
 releaseName: "hello"
 conditionalReleaseTag: "yes"
-`)
+`
 
 	yamlContent := []byte(`
 environments:
@@ -92,16 +86,11 @@ releases:
 
 `)
 
-	// make a reader that returns a simulated context
-	fileReader := func(filename string) ([]byte, error) {
-		expectedFilename := filepath.Clean("default/values.yaml")
-		if !strings.HasSuffix(filename, expectedFilename) {
-			return nil, fmt.Errorf("unexpected filename: expected=%s, actual=%s", expectedFilename, filename)
-		}
-		return defaultValuesYaml, nil
+	files := map[string]string{
+		"/path/to/default/values.yaml": defaultValuesYaml,
 	}
 
-	r := makeLoader(fileReader, "staging")
+	r, _ := makeLoader(files, "staging")
 	// test the double rendering
 	yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent)
 	if err != nil {
@@ -129,7 +118,7 @@ releases:
 }
 
 func TestReadFromYaml_RenderTemplateWithValuesReferenceError(t *testing.T) {
-	defaultValuesYaml := []byte("")
+	defaultValuesYaml := ``
 
 	yamlContent := []byte(`
 environments:
@@ -145,12 +134,11 @@ releases:
 {{ end }}
 `)
 
-	// make a reader that returns a simulated context
-	fileReader := func(filename string) ([]byte, error) {
-		return defaultValuesYaml, nil
+	files := map[string]string{
+		"/path/to/default/values.yaml": defaultValuesYaml,
 	}
 
-	r := makeLoader(fileReader, "staging")
+	r, _ := makeLoader(files, "staging")
 	// test the double rendering
 	_, err := r.renderTemplatesToYaml("", "", yamlContent)
 
@@ -164,9 +152,9 @@ releases:
 // This does not apply to .gotmpl files, which is a nice side-effect.
 func TestReadFromYaml_RenderTemplateWithGotmpl(t *testing.T) {
 
-	defaultValuesYamlGotmpl := []byte(`
+	defaultValuesYamlGotmpl := `
 releaseName: {{ readFile "nonIgnoredFile" }}
-`)
+`
 
 	yamlContent := []byte(`
 environments:
@@ -182,14 +170,12 @@ releases:
 {{ end }}
 `)
 
-	fileReader := func(filename string) ([]byte, error) {
-		if strings.HasSuffix(filename, "nonIgnoredFile") {
-			return []byte("release-a"), nil
-		}
-		return defaultValuesYamlGotmpl, nil
+	files := map[string]string{
+		"/path/to/nonIgnoredFile":     `release-a`,
+		"/path/to/values.yaml.gotmpl": defaultValuesYamlGotmpl,
 	}
 
-	r := makeLoader(fileReader, "staging")
+	r, _ := makeLoader(files, "staging")
 	rendered, _ := r.renderTemplatesToYaml("", "", yamlContent)
 
 	var state state.HelmState
@@ -205,18 +191,14 @@ releases:
 }
 
 func TestReadFromYaml_RenderTemplateWithNamespace(t *testing.T) {
-	defaultValuesYaml := []byte(``)
 	yamlContent := []byte(`releases:
 - name: {{ .Namespace }}-myrelease
   chart: mychart
 `)
 
-	// make a reader that returns a simulated context
-	fileReader := func(filename string) ([]byte, error) {
-		return defaultValuesYaml, nil
-	}
+	files := map[string]string{}
 
-	r := makeLoader(fileReader, "staging")
+	r, _ := makeLoader(files, "staging")
 	yamlBuf, err := r.renderTemplatesToYaml("", "", yamlContent)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -243,11 +225,8 @@ releases:
 {{ end }}
   chart: mychart
 `)
-	fileReader := func(filename string) ([]byte, error) {
-		return yamlContent, nil
-	}
 
-	r := makeLoader(fileReader, "staging")
+	r, _ := makeLoader(map[string]string{}, "staging")
 	_, err := r.renderTemplatesToYaml("", "", yamlContent)
 	if err == nil {
 		t.Fatalf("wanted error, none returned")

pkg/argparser/args.go

@@ -1,10 +1,9 @@
-package args
+package argparser
 
 import (
 	"fmt"
+	"github.com/roboll/helmfile/pkg/state"
 	"strings"
-
-	"github.com/roboll/helmfile/state"
 )
 
 type keyVal struct {
@@ -83,10 +82,6 @@ func GetArgs(args string, state *state.HelmState) []string {
 		}
 	}
 
-	if state.HelmDefaults.KubeContext != "" {
-		argsMap.SetArg("--kube-context", state.HelmDefaults.KubeContext, false)
-	}
-
 	var argArr []string
 
 	for _, flag := range argsMap.flags {

pkg/argparser/args_test.go

@@ -1,10 +1,9 @@
-package args
+package argparser
 
 import (
+	"github.com/roboll/helmfile/pkg/state"
 	"strings"
 	"testing"
-
-	"github.com/roboll/helmfile/state"
 )
 
 func TestGetArgs(t *testing.T) {
@@ -14,7 +13,7 @@ func TestGetArgs(t *testing.T) {
 	testState := &state.HelmState{HelmDefaults: Helmdefaults}
 	receivedArgs := GetArgs(args, testState)
 
-	expectedOutput := "--timeout=3600 --set app1.bootstrap=true --set app2.bootstrap=false --tiller-namespace ns --recreate-pods --force --kube-context=test"
+	expectedOutput := "--timeout=3600 --set app1.bootstrap=true --set app2.bootstrap=false --tiller-namespace ns --recreate-pods --force"
 
 	if compareArgs(expectedOutput, receivedArgs) == false {
 		t.Errorf("expected %s, got %s", expectedOutput, strings.Join(receivedArgs, " "))
@@ -28,7 +27,7 @@ func Test2(t *testing.T) {
 	testState := &state.HelmState{HelmDefaults: Helmdefaults}
 	receivedArgs := GetArgs(args, testState)
 
-	expectedOutput := "--timeout=3600 --set app1.bootstrap=true --set app2.bootstrap=false,app3.bootstrap=true --tiller-namespace ns --recreate-pods --force --kube-context=test"
+	expectedOutput := "--timeout=3600 --set app1.bootstrap=true --set app2.bootstrap=false,app3.bootstrap=true --tiller-namespace ns --recreate-pods --force"
 
 	if compareArgs(expectedOutput, receivedArgs) == false {
 		t.Errorf("expected %s, got %s", expectedOutput, strings.Join(receivedArgs, " "))

pkg/environment/environment.go

@@ -0,0 +1,66 @@
+package environment
+
+import (
+	"github.com/imdario/mergo"
+	"github.com/roboll/helmfile/pkg/maputil"
+	"gopkg.in/yaml.v2"
+)
+
+type Environment struct {
+	Name     string
+	Values   map[string]interface{}
+	Defaults map[string]interface{}
+}
+
+var EmptyEnvironment Environment
+
+func (e Environment) DeepCopy() Environment {
+	valuesBytes, err := yaml.Marshal(e.Values)
+	if err != nil {
+		panic(err)
+	}
+	var values map[string]interface{}
+	if err := yaml.Unmarshal(valuesBytes, &values); err != nil {
+		panic(err)
+	}
+	values, err = maputil.CastKeysToStrings(values)
+	if err != nil {
+		panic(err)
+	}
+
+	defaultsBytes, err := yaml.Marshal(e.Defaults)
+	if err != nil {
+		panic(err)
+	}
+	var defaults map[string]interface{}
+	if err := yaml.Unmarshal(defaultsBytes, &defaults); err != nil {
+		panic(err)
+	}
+	defaults, err = maputil.CastKeysToStrings(defaults)
+	if err != nil {
+		panic(err)
+	}
+
+	return Environment{
+		Name:     e.Name,
+		Values:   values,
+		Defaults: defaults,
+	}
+}
+
+func (e *Environment) Merge(other *Environment) (*Environment, error) {
+	if e == nil {
+		if other != nil {
+			copy := other.DeepCopy()
+			return &copy, nil
+		}
+		return nil, nil
+	}
+	copy := e.DeepCopy()
+	if other != nil {
+		if err := mergo.Merge(&copy, other, mergo.WithOverride); err != nil {
+			return nil, err
+		}
+	}
+	return &copy, nil
+}

pkg/event/bus.go

@@ -2,22 +2,26 @@ package event
 
 import (
 	"fmt"
-	"github.com/roboll/helmfile/environment"
-	"github.com/roboll/helmfile/helmexec"
-	"github.com/roboll/helmfile/tmpl"
-	"go.uber.org/zap"
 	"os"
+	"strings"
+
+	"github.com/roboll/helmfile/pkg/environment"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/tmpl"
+	"go.uber.org/zap"
 )
 
 type Hook struct {
-	Name    string   `yaml:"name"`
-	Events  []string `yaml:"events"`
-	Command string   `yaml:"command"`
-	Args    []string `yaml:"args"`
+	Name     string   `yaml:"name"`
+	Events   []string `yaml:"events"`
+	Command  string   `yaml:"command"`
+	Args     []string `yaml:"args"`
+	ShowLogs bool     `yaml:"showlogs"`
 }
 
 type event struct {
-	Name string
+	Name  string
+	Error error
 }
 
 type Bus struct {
@@ -34,7 +38,7 @@ type Bus struct {
 	Logger   *zap.SugaredLogger
 }
 
-func (bus *Bus) Trigger(evt string, context map[string]interface{}) (bool, error) {
+func (bus *Bus) Trigger(evt string, evtErr error, context map[string]interface{}) (bool, error) {
 	if bus.Runner == nil {
 		bus.Runner = helmexec.ShellRunner{
 			Dir: bus.BasePath,
@@ -65,7 +69,8 @@ func (bus *Bus) Trigger(evt string, context map[string]interface{}) (bool, error
 			"Environment": bus.Env,
 			"Namespace":   bus.Namespace,
 			"Event": event{
-				Name: evt,
+				Name:  evt,
+				Error: evtErr,
 			},
 		}
 		for k, v := range context {
@@ -90,6 +95,10 @@ func (bus *Bus) Trigger(evt string, context map[string]interface{}) (bool, error
 
 		bytes, err := bus.Runner.Execute(command, args, map[string]string{})
 		bus.Logger.Debugf("hook[%s]: %s\n", name, string(bytes))
+		if hook.ShowLogs {
+			prefix := fmt.Sprintf("\nhook[%s] logs | ", evt)
+			bus.Logger.Infow(prefix + strings.ReplaceAll(string(bytes), "\n", prefix))
+		}
 
 		if err != nil {
 			return false, fmt.Errorf("hook[%s]: command `%s` failed: %v", name, command, err)

pkg/event/bus_test.go

@@ -2,10 +2,13 @@ package event
 
 import (
 	"fmt"
-	"github.com/roboll/helmfile/environment"
-	"github.com/roboll/helmfile/helmexec"
 	"os"
 	"testing"
+
+	"github.com/roboll/helmfile/pkg/environment"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zaptest/observer"
 )
 
 var logger = helmexec.NewLogger(os.Stdout, "warn")
@@ -35,14 +38,21 @@ func TestTrigger(t *testing.T) {
 	}{
 		{
 			"okhook1",
-			&Hook{"okhook1", []string{"foo"}, "ok", []string{}},
+			&Hook{"okhook1", []string{"foo"}, "ok", []string{}, true},
+			"foo",
+			true,
+			"",
+		},
+		{
+			"okhooké",
+			&Hook{"okhook2", []string{"foo"}, "ok", []string{}, false},
 			"foo",
 			true,
 			"",
 		},
 		{
 			"missinghook1",
-			&Hook{"okhook1", []string{"foo"}, "ok", []string{}},
+			&Hook{"okhook1", []string{"foo"}, "ok", []string{}, false},
 			"bar",
 			false,
 			"",
@@ -56,14 +66,14 @@ func TestTrigger(t *testing.T) {
 		},
 		{
 			"nghook1",
-			&Hook{"nghook1", []string{"foo"}, "ng", []string{}},
+			&Hook{"nghook1", []string{"foo"}, "ng", []string{}, false},
 			"foo",
 			false,
 			"hook[nghook1]: command `ng` failed: cmd failed due to invalid cmd: ng",
 		},
 		{
 			"nghook2",
-			&Hook{"nghook2", []string{"foo"}, "ok", []string{"ng"}},
+			&Hook{"nghook2", []string{"foo"}, "ok", []string{"ng"}, false},
 			"foo",
 			false,
 			"hook[nghook2]: command `ok` failed: cmd failed due to invalid arg: ng",
@@ -77,13 +87,15 @@ func TestTrigger(t *testing.T) {
 		if c.hook != nil {
 			hooks = append(hooks, *c.hook)
 		}
+		observer, observedLogs := observer.New(zap.InfoLevel)
+		zeLogger := zap.New(observer).Sugar()
 		bus := &Bus{
 			Hooks:         hooks,
 			StateFilePath: "path/to/helmfile.yaml",
 			BasePath:      "path/to",
 			Namespace:     "myns",
 			Env:           environment.Environment{Name: "prod"},
-			Logger:        logger,
+			Logger:        zeLogger,
 			ReadFile:      readFile,
 		}
 
@@ -92,7 +104,7 @@ func TestTrigger(t *testing.T) {
 			"Release":         "myrel",
 			"HelmfileCommand": "mycmd",
 		}
-		ok, err := bus.Trigger(c.triggeredEvt, data)
+		ok, err := bus.Trigger(c.triggeredEvt, nil, data)
 
 		if ok != c.expectedResult {
 			t.Errorf("unexpected result for case \"%s\": expected=%v, actual=%v", c.name, c.expectedResult, ok)
@@ -109,5 +121,8 @@ func TestTrigger(t *testing.T) {
 				t.Errorf("unexpected error for case \"%s\": %v", c.name, err)
 			}
 		}
+		if observedLogs.Len() != 0 && !hooks[0].ShowLogs {
+			t.Errorf("unexpected error for case \"%s\": Logs should not be created : %v", c.name, observedLogs.All())
+		}
 	}
 }

pkg/exectest/helm.go

@@ -0,0 +1,171 @@
+package exectest
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/roboll/helmfile/pkg/helmexec"
+)
+
+type ListKey struct {
+	Filter string
+	Flags  string
+}
+
+type DiffKey struct {
+	Name  string
+	Chart string
+	Flags string
+}
+
+type Helm struct {
+	Charts               []string
+	Repo                 []string
+	Releases             []Release
+	Deleted              []Release
+	Lists                map[ListKey]string
+	Diffs                map[DiffKey]error
+	Diffed               []Release
+	FailOnUnexpectedDiff bool
+	FailOnUnexpectedList bool
+
+	UpdateDepsCallbacks map[string]func(string) error
+
+	DiffMutex     *sync.Mutex
+	ChartsMutex   *sync.Mutex
+	ReleasesMutex *sync.Mutex
+}
+
+type Release struct {
+	Name  string
+	Flags []string
+}
+
+type Affected struct {
+	Upgraded []*Release
+	Deleted  []*Release
+	Failed   []*Release
+}
+
+func (helm *Helm) UpdateDeps(chart string) error {
+	if strings.Contains(chart, "error") {
+		return fmt.Errorf("simulated UpdateDeps failure for chart: %s", chart)
+	}
+	helm.Charts = append(helm.Charts, chart)
+
+	if helm.UpdateDepsCallbacks != nil {
+		callback, exists := helm.UpdateDepsCallbacks[chart]
+		if exists {
+			if err := callback(chart); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (helm *Helm) BuildDeps(name, chart string) error {
+	if strings.Contains(chart, "error") {
+		return errors.New("error")
+	}
+	helm.Charts = append(helm.Charts, chart)
+	return nil
+}
+
+func (helm *Helm) SetExtraArgs(args ...string) {
+	return
+}
+func (helm *Helm) SetHelmBinary(bin string) {
+	return
+}
+func (helm *Helm) AddRepo(name, repository, cafile, certfile, keyfile, username, password string) error {
+	helm.Repo = []string{name, repository, cafile, certfile, keyfile, username, password}
+	return nil
+}
+func (helm *Helm) UpdateRepo() error {
+	return nil
+}
+func (helm *Helm) SyncRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
+	if strings.Contains(name, "error") {
+		return errors.New("error")
+	}
+	helm.sync(helm.ReleasesMutex, func() {
+		helm.Releases = append(helm.Releases, Release{Name: name, Flags: flags})
+	})
+	helm.sync(helm.ChartsMutex, func() {
+		helm.Charts = append(helm.Charts, chart)
+	})
+
+	return nil
+}
+func (helm *Helm) DiffRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
+	if helm.DiffMutex != nil {
+		helm.DiffMutex.Lock()
+	}
+	helm.Diffed = append(helm.Diffed, Release{Name: name, Flags: flags})
+	if helm.DiffMutex != nil {
+		helm.DiffMutex.Unlock()
+	}
+	key := DiffKey{Name: name, Chart: chart, Flags: strings.Join(flags, "")}
+	err, ok := helm.Diffs[key]
+	if !ok && helm.FailOnUnexpectedDiff {
+		return fmt.Errorf("unexpected diff with key: %v", key)
+	}
+	return err
+}
+func (helm *Helm) ReleaseStatus(context helmexec.HelmContext, release string, flags ...string) error {
+	if strings.Contains(release, "error") {
+		return errors.New("error")
+	}
+	helm.Releases = append(helm.Releases, Release{Name: release, Flags: flags})
+	return nil
+}
+func (helm *Helm) DeleteRelease(context helmexec.HelmContext, name string, flags ...string) error {
+	if strings.Contains(name, "error") {
+		return errors.New("error")
+	}
+	helm.Deleted = append(helm.Deleted, Release{Name: name, Flags: flags})
+	return nil
+}
+func (helm *Helm) List(context helmexec.HelmContext, filter string, flags ...string) (string, error) {
+	key := ListKey{Filter: filter, Flags: strings.Join(flags, "")}
+	res, ok := helm.Lists[key]
+	if !ok && helm.FailOnUnexpectedList {
+		return "", fmt.Errorf("unexpected list key: %v", key)
+	}
+	return res, nil
+}
+func (helm *Helm) DecryptSecret(context helmexec.HelmContext, name string, flags ...string) (string, error) {
+	return "", nil
+}
+func (helm *Helm) TestRelease(context helmexec.HelmContext, name string, flags ...string) error {
+	if strings.Contains(name, "error") {
+		return errors.New("error")
+	}
+	helm.Releases = append(helm.Releases, Release{Name: name, Flags: flags})
+	return nil
+}
+func (helm *Helm) Fetch(chart string, flags ...string) error {
+	return nil
+}
+func (helm *Helm) Lint(name, chart string, flags ...string) error {
+	return nil
+}
+func (helm *Helm) TemplateRelease(name, chart string, flags ...string) error {
+	return nil
+}
+
+func (helm *Helm) IsHelm3() bool {
+	return false
+}
+
+func (helm *Helm) sync(m *sync.Mutex, f func()) {
+	if m != nil {
+		m.Lock()
+		defer m.Unlock()
+	}
+
+	f()
+}

pkg/helmexec/exec.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 
@@ -12,17 +13,20 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
-const (
-	command = "helm"
-)
+type decryptedSecret struct {
+	mutex sync.RWMutex
+	bytes []byte
+}
 
 type execer struct {
-	helmBinary      string
-	runner          Runner
-	logger          *zap.SugaredLogger
-	kubeContext     string
-	extra           []string
-	decryptionMutex sync.Mutex
+	helmBinary           string
+	isHelm3              bool
+	runner               Runner
+	logger               *zap.SugaredLogger
+	kubeContext          string
+	extra                []string
+	decryptedSecretMutex sync.Mutex
+	decryptedSecrets     map[string]*decryptedSecret
 }
 
 func NewLogger(writer io.Writer, logLevel string) *zap.SugaredLogger {
@@ -42,16 +46,31 @@ func NewLogger(writer io.Writer, logLevel string) *zap.SugaredLogger {
 	return zap.New(core).Sugar()
 }
 
-// New for running helm commands
-func New(logger *zap.SugaredLogger, kubeContext string) *execer {
-	return &execer{
-		helmBinary:  command,
-		logger:      logger,
-		kubeContext: kubeContext,
-		runner: &ShellRunner{
-			logger: logger,
-		},
+func detectHelm3(helmBinary string, logger *zap.SugaredLogger, runner Runner) bool {
+	// Support explicit opt-in via environment variable
+	if os.Getenv("HELMFILE_HELM3") != "" {
+		return true
 	}
+
+	// Autodetect from `helm verison`
+	bytes, err := runner.Execute(helmBinary, []string{"version", "--client", "--short"}, nil)
+	if err != nil {
+		panic(err)
+	}
+	return strings.HasPrefix(string(bytes), "v3.")
+}
+
+// New for running helm commands
+func New(helmBinary string, logger *zap.SugaredLogger, kubeContext string, runner Runner) *execer {
+	return &execer{
+		helmBinary:       helmBinary,
+		isHelm3:          detectHelm3(helmBinary, logger, runner),
+		logger:           logger,
+		kubeContext:      kubeContext,
+		runner:           runner,
+		decryptedSecrets: make(map[string]*decryptedSecret),
+	}
+
 }
 
 func (helm *execer) SetExtraArgs(args ...string) {
@@ -62,12 +81,15 @@ func (helm *execer) SetHelmBinary(bin string) {
 	helm.helmBinary = bin
 }
 
-func (helm *execer) AddRepo(name, repository, certfile, keyfile, username, password string) error {
+func (helm *execer) AddRepo(name, repository, cafile, certfile, keyfile, username, password string) error {
 	var args []string
 	args = append(args, "repo", "add", name, repository)
 	if certfile != "" && keyfile != "" {
 		args = append(args, "--cert-file", certfile, "--key-file", keyfile)
 	}
+	if cafile != "" {
+		args = append(args, "--ca-file", cafile)
+	}
 	if username != "" && password != "" {
 		args = append(args, "--username", username, "--password", password)
 	}
@@ -84,6 +106,13 @@ func (helm *execer) UpdateRepo() error {
 	return err
 }
 
+func (helm *execer) BuildDeps(name, chart string) error {
+	helm.logger.Infof("Building dependency release=%v, chart=%v", name, chart)
+	out, err := helm.exec([]string{"dependency", "build", chart}, map[string]string{})
+	helm.info(out)
+	return err
+}
+
 func (helm *execer) UpdateDeps(chart string) error {
 	helm.logger.Infof("Updating dependency %v", chart)
 	out, err := helm.exec([]string{"dependency", "update", chart}, map[string]string{})
@@ -91,15 +120,8 @@ func (helm *execer) UpdateDeps(chart string) error {
 	return err
 }
 
-func (helm *execer) BuildDeps(chart string) error {
-	helm.logger.Infof("Building dependency %v", chart)
-	out, err := helm.exec([]string{"dependency", "build", chart}, map[string]string{})
-	helm.write(out)
-	return err
-}
-
 func (helm *execer) SyncRelease(context HelmContext, name, chart string, flags ...string) error {
-	helm.logger.Infof("Upgrading %v", chart)
+	helm.logger.Infof("Upgrading release=%v, chart=%v", name, chart)
 	preArgs := context.GetTillerlessArgs(helm.helmBinary)
 	env := context.getTillerlessEnv()
 	out, err := helm.exec(append(append(preArgs, "upgrade", "--install", "--reset-values", name, chart), flags...), env)
@@ -120,68 +142,116 @@ func (helm *execer) List(context HelmContext, filter string, flags ...string) (s
 	helm.logger.Infof("Listing releases matching %v", filter)
 	preArgs := context.GetTillerlessArgs(helm.helmBinary)
 	env := context.getTillerlessEnv()
-	out, err := helm.exec(append(append(preArgs, "list", filter), flags...), env)
+	var args []string
+	if helm.IsHelm3() {
+		args = []string{"list", "--filter", filter}
+	} else {
+		args = []string{"list", filter}
+	}
+
+	out, err := helm.exec(append(append(preArgs, args...), flags...), env)
+	// In v2 we have been expecting `helm list FILTER` prints nothing.
+	// In v3 helm still prints the header like `NAME	NAMESPACE	REVISION	UPDATED	STATUS	CHART	APP VERSION`,
+	// which confuses helmfile's existing logic that treats any non-empty output from `helm list` is considered as the indication
+	// of the release to exist.
+	//
+	// This fixes it by removing the header from the v3 output, so that the output is formatted the same as that of v2.
+	if helm.IsHelm3() {
+		lines := strings.Split(string(out), "\n")
+		lines = lines[1:]
+		out = []byte(strings.Join(lines, "\n"))
+	}
 	helm.write(out)
 	return string(out), err
 }
 
 func (helm *execer) DecryptSecret(context HelmContext, name string, flags ...string) (string, error) {
-	// Prevents https://github.com/roboll/helmfile/issues/258
-	helm.decryptionMutex.Lock()
-	defer helm.decryptionMutex.Unlock()
-
-	helm.logger.Infof("Decrypting secret %v", name)
-	preArgs := context.GetTillerlessArgs(helm.helmBinary)
-	env := context.getTillerlessEnv()
-	out, err := helm.exec(append(append(preArgs, "secrets", "dec", name), flags...), env)
-	helm.write(out)
+	absPath, err := filepath.Abs(name)
 	if err != nil {
 		return "", err
 	}
 
+	helm.logger.Debugf("Preparing to decrypt secret %v", absPath)
+	helm.decryptedSecretMutex.Lock()
+
+	secret, ok := helm.decryptedSecrets[absPath]
+
+	// Cache miss
+	if !ok {
+
+		secret = &decryptedSecret{}
+		helm.decryptedSecrets[absPath] = secret
+
+		secret.mutex.Lock()
+		defer secret.mutex.Unlock()
+		helm.decryptedSecretMutex.Unlock()
+
+		helm.logger.Infof("Decrypting secret %v", absPath)
+		preArgs := context.GetTillerlessArgs(helm.helmBinary)
+		env := context.getTillerlessEnv()
+		out, err := helm.exec(append(append(preArgs, "secrets", "dec", absPath), flags...), env)
+		helm.info(out)
+		if err != nil {
+			return "", err
+		}
+
+		// HELM_SECRETS_DEC_SUFFIX is used by the helm-secrets plugin to define the output file
+		decSuffix := os.Getenv("HELM_SECRETS_DEC_SUFFIX")
+		if len(decSuffix) == 0 {
+			decSuffix = ".yaml.dec"
+		}
+		decFilename := strings.Replace(absPath, ".yaml", decSuffix, 1)
+
+		secretBytes, err := ioutil.ReadFile(decFilename)
+		if err != nil {
+			return "", err
+		}
+		secret.bytes = secretBytes
+
+		if err := os.Remove(decFilename); err != nil {
+			return "", err
+		}
+
+	} else {
+		// Cache hit
+		helm.logger.Debugf("Found secret in cache %v", absPath)
+
+		secret.mutex.RLock()
+		helm.decryptedSecretMutex.Unlock()
+		defer secret.mutex.RUnlock()
+	}
+
 	tmpFile, err := ioutil.TempFile("", "secret")
 	if err != nil {
 		return "", err
 	}
-	defer tmpFile.Close()
-
-	// os.Rename seems to results in "cross-device link` errors in some cases
-	// Instead of moving, copy it to the destination temp file as a work-around
-	// See https://github.com/roboll/helmfile/issues/251#issuecomment-417166296f
-	decFilename := name + ".dec"
-	decFile, err := os.Open(decFilename)
+	_, err = tmpFile.Write(secret.bytes)
 	if err != nil {
 		return "", err
 	}
-	defer decFile.Close()
-
-	_, err = io.Copy(tmpFile, decFile)
-	if err != nil {
-		return "", err
-	}
-
-	if err := decFile.Close(); err != nil {
-		return "", err
-	}
-
-	if err := os.Remove(decFilename); err != nil {
-		return "", err
-	}
 
 	return tmpFile.Name(), err
 }
 
-func (helm *execer) TemplateRelease(chart string, flags ...string) error {
-	out, err := helm.exec(append([]string{"template", chart}, flags...), map[string]string{})
+func (helm *execer) TemplateRelease(name string, chart string, flags ...string) error {
+	helm.logger.Infof("Templating release=%v, chart=%v", name, chart)
+	var args []string
+	if helm.IsHelm3() {
+		args = []string{"template", name, chart}
+	} else {
+		args = []string{"template", chart, "--name", name}
+	}
+
+	out, err := helm.exec(append(args, flags...), map[string]string{})
 	helm.write(out)
 	return err
 }
 
 func (helm *execer) DiffRelease(context HelmContext, name, chart string, flags ...string) error {
-	helm.logger.Infof("Comparing %v %v", name, chart)
+	helm.logger.Infof("Comparing release=%v, chart=%v", name, chart)
 	preArgs := context.GetTillerlessArgs(helm.helmBinary)
 	env := context.getTillerlessEnv()
-	out, err := helm.exec(append(append(preArgs, "diff", "upgrade", "--allow-unreleased", name, chart), flags...), env)
+	out, err := helm.exec(append(append(preArgs, "diff", "upgrade", "--reset-values", "--allow-unreleased", name, chart), flags...), env)
 	// Do our best to write STDOUT only when diff existed
 	// Unfortunately, this works only when you run helmfile with `--detailed-exitcode`
 	detailedExitcodeEnabled := false
@@ -205,8 +275,8 @@ func (helm *execer) DiffRelease(context HelmContext, name, chart string, flags .
 	return err
 }
 
-func (helm *execer) Lint(chart string, flags ...string) error {
-	helm.logger.Infof("Linting %v", chart)
+func (helm *execer) Lint(name, chart string, flags ...string) error {
+	helm.logger.Infof("Linting release=%v, chart=%v", name, chart)
 	out, err := helm.exec(append([]string{"lint", chart}, flags...), map[string]string{})
 	helm.write(out)
 	return err
@@ -232,7 +302,13 @@ func (helm *execer) TestRelease(context HelmContext, name string, flags ...strin
 	helm.logger.Infof("Testing %v", name)
 	preArgs := context.GetTillerlessArgs(helm.helmBinary)
 	env := context.getTillerlessEnv()
-	out, err := helm.exec(append(append(preArgs, "test", name), flags...), env)
+	var args []string
+	if helm.IsHelm3() {
+		args = []string{"test", "run", name}
+	} else {
+		args = []string{"test", name}
+	}
+	out, err := helm.exec(append(append(preArgs, args...), flags...), env)
 	helm.write(out)
 	return err
 }
@@ -263,3 +339,7 @@ func (helm *execer) write(out []byte) {
 		fmt.Printf("%s\n", out)
 	}
 }
+
+func (helm *execer) IsHelm3() bool {
+	return helm.isHelm3
+}

pkg/helmexec/exec_test.go

@@ -2,8 +2,10 @@ package helmexec
 
 import (
 	"bytes"
+	"fmt"
 	"os"
 	"path"
+	"path/filepath"
 	"reflect"
 	"testing"
 
@@ -18,12 +20,11 @@ type mockRunner struct {
 }
 
 func (mock *mockRunner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) {
-	return []byte{}, nil
+	return mock.output, mock.err
 }
 
 func MockExecer(logger *zap.SugaredLogger, kubeContext string) *execer {
-	execer := New(logger, kubeContext)
-	execer.runner = &mockRunner{}
+	execer := New("helm", logger, kubeContext, &mockRunner{})
 	return execer
 }
 
@@ -31,8 +32,7 @@ func MockExecer(logger *zap.SugaredLogger, kubeContext string) *execer {
 
 func TestNewHelmExec(t *testing.T) {
 	buffer := bytes.NewBufferString("something")
-	logger := NewLogger(buffer, "debug")
-	helm := New(logger, "dev")
+	helm := MockExecer(NewLogger(buffer, "debug"), "dev")
 	if helm.kubeContext != "dev" {
 		t.Error("helmexec.New() - kubeContext")
 	}
@@ -45,7 +45,7 @@ func TestNewHelmExec(t *testing.T) {
 }
 
 func Test_SetExtraArgs(t *testing.T) {
-	helm := New(NewLogger(os.Stdout, "info"), "dev")
+	helm := MockExecer(NewLogger(os.Stdout, "info"), "dev")
 	helm.SetExtraArgs()
 	if len(helm.extra) != 0 {
 		t.Error("helmexec.SetExtraArgs() - passing no arguments should not change extra field")
@@ -61,7 +61,7 @@ func Test_SetExtraArgs(t *testing.T) {
 }
 
 func Test_SetHelmBinary(t *testing.T) {
-	helm := New(NewLogger(os.Stdout, "info"), "dev")
+	helm := MockExecer(NewLogger(os.Stdout, "info"), "dev")
 	if helm.helmBinary != "helm" {
 		t.Error("helmexec.command - default command is not helm")
 	}
@@ -75,7 +75,7 @@ func Test_AddRepo(t *testing.T) {
 	var buffer bytes.Buffer
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
-	helm.AddRepo("myRepo", "https://repo.example.com/", "cert.pem", "key.pem", "", "")
+	helm.AddRepo("myRepo", "https://repo.example.com/", "", "cert.pem", "key.pem", "", "")
 	expected := `Adding repo myRepo https://repo.example.com/
 exec: helm repo add myRepo https://repo.example.com/ --cert-file cert.pem --key-file key.pem --kube-context dev
 exec: helm repo add myRepo https://repo.example.com/ --cert-file cert.pem --key-file key.pem --kube-context dev: 
@@ -85,7 +85,17 @@ exec: helm repo add myRepo https://repo.example.com/ --cert-file cert.pem --key-
 	}
 
 	buffer.Reset()
-	helm.AddRepo("myRepo", "https://repo.example.com/", "", "", "", "")
+	helm.AddRepo("myRepo", "https://repo.example.com/", "ca.crt", "", "", "", "")
+	expected = `Adding repo myRepo https://repo.example.com/
+exec: helm repo add myRepo https://repo.example.com/ --ca-file ca.crt --kube-context dev
+exec: helm repo add myRepo https://repo.example.com/ --ca-file ca.crt --kube-context dev: 
+`
+	if buffer.String() != expected {
+		t.Errorf("helmexec.AddRepo()\nactual = %v\nexpect = %v", buffer.String(), expected)
+	}
+
+	buffer.Reset()
+	helm.AddRepo("myRepo", "https://repo.example.com/", "", "", "", "", "")
 	expected = `Adding repo myRepo https://repo.example.com/
 exec: helm repo add myRepo https://repo.example.com/ --kube-context dev
 exec: helm repo add myRepo https://repo.example.com/ --kube-context dev: 
@@ -95,7 +105,7 @@ exec: helm repo add myRepo https://repo.example.com/ --kube-context dev:
 	}
 
 	buffer.Reset()
-	helm.AddRepo("myRepo", "https://repo.example.com/", "", "", "example_user", "example_password")
+	helm.AddRepo("myRepo", "https://repo.example.com/", "", "", "", "example_user", "example_password")
 	expected = `Adding repo myRepo https://repo.example.com/
 exec: helm repo add myRepo https://repo.example.com/ --username example_user --password example_password --kube-context dev
 exec: helm repo add myRepo https://repo.example.com/ --username example_user --password example_password --kube-context dev: 
@@ -124,7 +134,7 @@ func Test_SyncRelease(t *testing.T) {
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
 	helm.SyncRelease(HelmContext{}, "release", "chart", "--timeout 10", "--wait")
-	expected := `Upgrading chart
+	expected := `Upgrading release=release, chart=chart
 exec: helm upgrade --install --reset-values release chart --timeout 10 --wait --kube-context dev
 exec: helm upgrade --install --reset-values release chart --timeout 10 --wait --kube-context dev: 
 `
@@ -134,7 +144,7 @@ exec: helm upgrade --install --reset-values release chart --timeout 10 --wait --
 
 	buffer.Reset()
 	helm.SyncRelease(HelmContext{}, "release", "chart")
-	expected = `Upgrading chart
+	expected = `Upgrading release=release, chart=chart
 exec: helm upgrade --install --reset-values release chart --kube-context dev
 exec: helm upgrade --install --reset-values release chart --kube-context dev: 
 `
@@ -149,7 +159,7 @@ func Test_SyncReleaseTillerless(t *testing.T) {
 	helm := MockExecer(logger, "dev")
 	helm.SyncRelease(HelmContext{Tillerless: true, TillerNamespace: "foo"}, "release", "chart",
 		"--timeout 10", "--wait")
-	expected := `Upgrading chart
+	expected := `Upgrading release=release, chart=chart
 exec: helm tiller run foo -- helm upgrade --install --reset-values release chart --timeout 10 --wait --kube-context dev
 exec: helm tiller run foo -- helm upgrade --install --reset-values release chart --timeout 10 --wait --kube-context dev: 
 `
@@ -187,8 +197,8 @@ func Test_BuildDeps(t *testing.T) {
 	var buffer bytes.Buffer
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
-	helm.BuildDeps("./chart/foo")
-	expected := `Building dependency ./chart/foo
+	helm.BuildDeps("foo", "./chart/foo")
+	expected := `Building dependency release=foo, chart=./chart/foo
 exec: helm dependency build ./chart/foo --kube-context dev
 exec: helm dependency build ./chart/foo --kube-context dev: 
 `
@@ -198,8 +208,8 @@ exec: helm dependency build ./chart/foo --kube-context dev:
 
 	buffer.Reset()
 	helm.SetExtraArgs("--verify")
-	helm.BuildDeps("./chart/foo")
-	expected = `Building dependency ./chart/foo
+	helm.BuildDeps("foo", "./chart/foo")
+	expected = `Building dependency release=foo, chart=./chart/foo
 exec: helm dependency build ./chart/foo --verify --kube-context dev
 exec: helm dependency build ./chart/foo --verify --kube-context dev: 
 `
@@ -213,10 +223,20 @@ func Test_DecryptSecret(t *testing.T) {
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
 	helm.DecryptSecret(HelmContext{}, "secretName")
-	expected := `Decrypting secret secretName
-exec: helm secrets dec secretName --kube-context dev
-exec: helm secrets dec secretName --kube-context dev: 
-`
+	cwd, err := filepath.Abs(".")
+	if err != nil {
+		t.Errorf("Error: %v", err)
+	}
+	// Run again for caching
+	helm.DecryptSecret(HelmContext{}, "secretName")
+
+	expected := fmt.Sprintf(`Preparing to decrypt secret %v/secretName
+Decrypting secret %s/secretName
+exec: helm secrets dec %s/secretName --kube-context dev
+exec: helm secrets dec %s/secretName --kube-context dev: 
+Preparing to decrypt secret %s/secretName
+Found secret in cache %s/secretName
+`, cwd, cwd, cwd, cwd, cwd, cwd)
 	if buffer.String() != expected {
 		t.Errorf("helmexec.DecryptSecret()\nactual = %v\nexpect = %v", buffer.String(), expected)
 	}
@@ -227,9 +247,9 @@ func Test_DiffRelease(t *testing.T) {
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
 	helm.DiffRelease(HelmContext{}, "release", "chart", "--timeout 10", "--wait")
-	expected := `Comparing release chart
-exec: helm diff upgrade --allow-unreleased release chart --timeout 10 --wait --kube-context dev
-exec: helm diff upgrade --allow-unreleased release chart --timeout 10 --wait --kube-context dev: 
+	expected := `Comparing release=release, chart=chart
+exec: helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev
+exec: helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev: 
 `
 	if buffer.String() != expected {
 		t.Errorf("helmexec.DiffRelease()\nactual = %v\nexpect = %v", buffer.String(), expected)
@@ -237,9 +257,9 @@ exec: helm diff upgrade --allow-unreleased release chart --timeout 10 --wait --k
 
 	buffer.Reset()
 	helm.DiffRelease(HelmContext{}, "release", "chart")
-	expected = `Comparing release chart
-exec: helm diff upgrade --allow-unreleased release chart --kube-context dev
-exec: helm diff upgrade --allow-unreleased release chart --kube-context dev: 
+	expected = `Comparing release=release, chart=chart
+exec: helm diff upgrade --reset-values --allow-unreleased release chart --kube-context dev
+exec: helm diff upgrade --reset-values --allow-unreleased release chart --kube-context dev: 
 `
 	if buffer.String() != expected {
 		t.Errorf("helmexec.DiffRelease()\nactual = %v\nexpect = %v", buffer.String(), expected)
@@ -251,9 +271,9 @@ func Test_DiffReleaseTillerless(t *testing.T) {
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
 	helm.DiffRelease(HelmContext{Tillerless: true}, "release", "chart", "--timeout 10", "--wait")
-	expected := `Comparing release chart
-exec: helm tiller run -- helm diff upgrade --allow-unreleased release chart --timeout 10 --wait --kube-context dev
-exec: helm tiller run -- helm diff upgrade --allow-unreleased release chart --timeout 10 --wait --kube-context dev: 
+	expected := `Comparing release=release, chart=chart
+exec: helm tiller run -- helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev
+exec: helm tiller run -- helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev: 
 `
 	if buffer.String() != expected {
 		t.Errorf("helmexec.DiffRelease()\nactual = %v\nexpect = %v", buffer.String(), expected)
@@ -392,8 +412,8 @@ func Test_Lint(t *testing.T) {
 	var buffer bytes.Buffer
 	logger := NewLogger(&buffer, "debug")
 	helm := MockExecer(logger, "dev")
-	helm.Lint("path/to/chart", "--values", "file.yml")
-	expected := `Linting path/to/chart
+	helm.Lint("release", "path/to/chart", "--values", "file.yml")
+	expected := `Linting release=release, chart=path/to/chart
 exec: helm lint path/to/chart --values file.yml --kube-context dev
 exec: helm lint path/to/chart --values file.yml --kube-context dev: 
 `
@@ -432,7 +452,7 @@ func Test_LogLevels(t *testing.T) {
 		buffer.Reset()
 		logger := NewLogger(&buffer, logLevel)
 		helm := MockExecer(logger, "")
-		helm.AddRepo("myRepo", "https://repo.example.com/", "", "", "example_user", "example_password")
+		helm.AddRepo("myRepo", "https://repo.example.com/", "", "", "", "example_user", "example_password")
 		if buffer.String() != expected {
 			t.Errorf("helmexec.AddRepo()\nactual = %v\nexpect = %v", buffer.String(), expected)
 		}
@@ -472,3 +492,31 @@ func Test_mergeEnv(t *testing.T) {
 		t.Errorf("mergeEnv()\nactual = %v\nexpect = %v", actual, expected)
 	}
 }
+
+func Test_Template(t *testing.T) {
+	var buffer bytes.Buffer
+	logger := NewLogger(&buffer, "debug")
+	helm := MockExecer(logger, "dev")
+	helm.TemplateRelease("release", "path/to/chart", "--values", "file.yml")
+	expected := `Templating release=release, chart=path/to/chart
+exec: helm template path/to/chart --name release --values file.yml --kube-context dev
+exec: helm template path/to/chart --name release --values file.yml --kube-context dev: 
+`
+	if buffer.String() != expected {
+		t.Errorf("helmexec.Template()\nactual = %v\nexpect = %v", buffer.String(), expected)
+	}
+}
+
+func Test_IsHelm3(t *testing.T) {
+	helm2Runner := mockRunner{output: []byte("Client: v2.16.0+ge13bc94\n")}
+	helm := New("helm", NewLogger(os.Stdout, "info"), "dev", &helm2Runner)
+	if helm.IsHelm3() {
+		t.Error("helmexec.IsHelm3() - Detected Helm 3 with Helm 2 version")
+	}
+
+	helm3Runner := mockRunner{output: []byte("v3.0.0+ge29ce2a\n")}
+	helm = New("helm", NewLogger(os.Stdout, "info"), "dev", &helm3Runner)
+	if !helm.IsHelm3() {
+		t.Error("helmexec.IsHelm3() - Failed to detect Helm 3")
+	}
+}

pkg/helmexec/exit_error.go

@@ -8,8 +8,8 @@ import (
 
 func newExitError(helmCmdPath string, exitStatus int, errorMessage string) ExitError {
 	return ExitError{
-		msg:        fmt.Sprintf("%s exited with status %d:\n%s", filepath.Base(helmCmdPath), exitStatus, indent(strings.TrimSpace(errorMessage))),
-		exitStatus: exitStatus,
+		Message: fmt.Sprintf("%s exited with status %d:\n%s", filepath.Base(helmCmdPath), exitStatus, indent(strings.TrimSpace(errorMessage))),
+		Code:    exitStatus,
 	}
 }
 
@@ -23,14 +23,14 @@ func indent(text string) string {
 
 // ExitError is created whenever your shell command exits with a non-zero exit status
 type ExitError struct {
-	msg        string
-	exitStatus int
+	Message string
+	Code    int
 }
 
 func (e ExitError) Error() string {
-	return e.msg
+	return e.Message
 }
 
 func (e ExitError) ExitStatus() int {
-	return e.exitStatus
+	return e.Code
 }

pkg/helmexec/helmexec.go

@@ -5,18 +5,24 @@ type Interface interface {
 	SetExtraArgs(args ...string)
 	SetHelmBinary(bin string)
 
-	AddRepo(name, repository, certfile, keyfile, username, password string) error
+	AddRepo(name, repository, cafile, certfile, keyfile, username, password string) error
 	UpdateRepo() error
-	BuildDeps(chart string) error
+	BuildDeps(name, chart string) error
 	UpdateDeps(chart string) error
 	SyncRelease(context HelmContext, name, chart string, flags ...string) error
 	DiffRelease(context HelmContext, name, chart string, flags ...string) error
-	TemplateRelease(chart string, flags ...string) error
+	TemplateRelease(name, chart string, flags ...string) error
 	Fetch(chart string, flags ...string) error
-	Lint(chart string, flags ...string) error
+	Lint(name, chart string, flags ...string) error
 	ReleaseStatus(context HelmContext, name string, flags ...string) error
 	DeleteRelease(context HelmContext, name string, flags ...string) error
 	TestRelease(context HelmContext, name string, flags ...string) error
 	List(context HelmContext, filter string, flags ...string) (string, error)
 	DecryptSecret(context HelmContext, name string, flags ...string) (string, error)
+	IsHelm3() bool
+}
+
+type DependencyUpdater interface {
+	UpdateDeps(chart string) error
+	IsHelm3() bool
 }

pkg/helmexec/runner.go

@@ -25,7 +25,7 @@ type Runner interface {
 type ShellRunner struct {
 	Dir string
 
-	logger *zap.SugaredLogger
+	Logger *zap.SugaredLogger
 }
 
 // Execute a shell command
@@ -33,7 +33,7 @@ func (shell ShellRunner) Execute(cmd string, args []string, env map[string]strin
 	preparedCmd := exec.Command(cmd, args...)
 	preparedCmd.Dir = shell.Dir
 	preparedCmd.Env = mergeEnv(os.Environ(), env)
-	return combinedOutput(preparedCmd, shell.logger)
+	return combinedOutput(preparedCmd, shell.Logger)
 }
 
 func combinedOutput(c *exec.Cmd, logger *zap.SugaredLogger) ([]byte, error) {

pkg/maputil/maputil.go

@@ -0,0 +1,91 @@
+package maputil
+
+import "fmt"
+
+func CastKeysToStrings(s interface{}) (map[string]interface{}, error) {
+	new := map[string]interface{}{}
+	switch src := s.(type) {
+	case map[interface{}]interface{}:
+		for k, v := range src {
+			var str_k string
+			switch typed_k := k.(type) {
+			case string:
+				str_k = typed_k
+			default:
+				return nil, fmt.Errorf("unexpected type of key in map: expected string, got %T: value=%v, map=%v", typed_k, typed_k, src)
+			}
+
+			casted_v, err := recursivelyStringifyMapKey(v)
+			if err != nil {
+				return nil, err
+			}
+
+			new[str_k] = casted_v
+		}
+	case map[string]interface{}:
+		for k, v := range src {
+			casted_v, err := recursivelyStringifyMapKey(v)
+			if err != nil {
+				return nil, err
+			}
+
+			new[k] = casted_v
+		}
+	}
+	return new, nil
+}
+
+func recursivelyStringifyMapKey(v interface{}) (interface{}, error) {
+	var casted_v interface{}
+	switch typed_v := v.(type) {
+	case map[interface{}]interface{}, map[string]interface{}:
+		tmp, err := CastKeysToStrings(typed_v)
+		if err != nil {
+			return nil, err
+		}
+		casted_v = tmp
+	case []interface{}:
+		a := []interface{}{}
+		for i := range typed_v {
+			res, err := recursivelyStringifyMapKey(typed_v[i])
+			if err != nil {
+				return nil, err
+			}
+			a = append(a, res)
+		}
+		casted_v = a
+	default:
+		casted_v = typed_v
+	}
+	return casted_v, nil
+}
+
+func Set(m map[string]interface{}, key []string, value string) map[string]interface{} {
+	if len(key) == 0 {
+		panic(fmt.Errorf("bug: unexpected length of key: %d", len(key)))
+	}
+
+	k := key[0]
+
+	if len(key) == 1 {
+		m[k] = value
+		return m
+	}
+
+	remain := key[1:]
+
+	nested, ok := m[k]
+	if !ok {
+		nested = map[string]interface{}{}
+	}
+	switch t := nested.(type) {
+	case map[string]interface{}:
+		nested = Set(t, remain, value)
+	default:
+		panic(fmt.Errorf("unexpected type: %v(%T)", t, t))
+	}
+
+	m[k] = nested
+
+	return m
+}

pkg/maputil/maputil_test.go

@@ -0,0 +1,61 @@
+package maputil
+
+import "testing"
+
+func TestMapUtil_StrKeys(t *testing.T) {
+	m := map[string]interface{}{
+		"a": []interface{}{
+			map[string]interface{}{
+				"b": []interface{}{
+					map[string]interface{}{
+						"c": "C",
+					},
+				},
+			},
+		},
+	}
+
+	r, err := CastKeysToStrings(m)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	a := r["a"].([]interface{})
+	a0 := a[0].(map[string]interface{})
+	b := a0["b"].([]interface{})
+	b0 := b[0].(map[string]interface{})
+	c := b0["c"]
+
+	if c != "C" {
+		t.Errorf("unexpected c: expected=C, got=%s", c)
+	}
+}
+
+func TestMapUtil_IFKeys(t *testing.T) {
+	m := map[interface{}]interface{}{
+		"a": []interface{}{
+			map[interface{}]interface{}{
+				"b": []interface{}{
+					map[interface{}]interface{}{
+						"c": "C",
+					},
+				},
+			},
+		},
+	}
+
+	r, err := CastKeysToStrings(m)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	a := r["a"].([]interface{})
+	a0 := a[0].(map[string]interface{})
+	b := a0["b"].([]interface{})
+	b0 := b[0].(map[string]interface{})
+	c := b0["c"]
+
+	if c != "C" {
+		t.Errorf("unexpected c: expected=C, got=%s", c)
+	}
+}

pkg/remote/remote.go

@@ -0,0 +1,262 @@
+package remote
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/hashicorp/go-getter"
+	"github.com/hashicorp/go-getter/helper/url"
+	"go.uber.org/multierr"
+	"go.uber.org/zap"
+	"gopkg.in/yaml.v2"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const DefaultCacheDir = ".helmfile/cache"
+
+type Remote struct {
+	Logger *zap.SugaredLogger
+
+	// Home is the home directory for helmfile. Usually this points to $HOME of the user running helmfile.
+	// Helmfile saves fetched remote files into .helmfile/cache under home
+	Home string
+
+	// Getter is the underlying implementation of getter used for fetching remote files
+	Getter Getter
+
+	// ReadFile is the implementation of the file reader that reads a local file from the specified path.
+	// Inject any implementation of your choice, like an im-memory impl for testing, ioutil.ReadFile for the real-world use.
+	ReadFile   func(string) ([]byte, error)
+	DirExists  func(string) bool
+	FileExists func(string) bool
+}
+
+func (r *Remote) Unmarshal(src string, dst interface{}) error {
+	bytes, err := r.GetBytes(src)
+	if err != nil {
+		return err
+	}
+
+	strs := strings.Split(src, "/")
+	file := strs[len(strs)-1]
+	ext := filepath.Ext(file)
+
+	{
+		r.Logger.Debugf("unmarshalling %s", string(bytes))
+
+		var err error
+		switch ext {
+		case "json":
+			err = json.Unmarshal(bytes, dst)
+		default:
+			err = yaml.Unmarshal(bytes, dst)
+		}
+
+		r.Logger.Debugf("unmarshalled to %v", dst)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r *Remote) GetBytes(goGetterSrc string) ([]byte, error) {
+	f, err := r.Fetch(goGetterSrc)
+	if err != nil {
+		return nil, err
+	}
+
+	bytes, err := r.ReadFile(f)
+	if err != nil {
+		return nil, fmt.Errorf("read file: %v", err)
+	}
+
+	return bytes, nil
+}
+
+// Locate takes an URL to a remote file or a path to a local file.
+// If the argument was an URL, it fetches the remote directory contained within the URL,
+// and returns the path to the file in the fetched directory
+func (r *Remote) Locate(urlOrPath string) (string, error) {
+	if r.FileExists(urlOrPath) || r.DirExists(urlOrPath) {
+		return urlOrPath, nil
+	}
+	fetched, err := r.Fetch(urlOrPath)
+	if err != nil {
+		switch err.(type) {
+		case InvalidURLError:
+			return urlOrPath, nil
+		}
+		return "", err
+	}
+	return fetched, nil
+}
+
+type InvalidURLError struct {
+	err string
+}
+
+func (e InvalidURLError) Error() string {
+	return e.err
+}
+
+type Source struct {
+	Getter, Scheme, User, Host, Dir, File, RawQuery string
+}
+
+func IsRemote(goGetterSrc string) bool {
+	if _, err := Parse(goGetterSrc); err != nil {
+		return false
+	}
+	return true
+}
+
+func Parse(goGetterSrc string) (*Source, error) {
+	items := strings.Split(goGetterSrc, "::")
+	var getter string
+	switch len(items) {
+	case 2:
+		getter = items[0]
+		goGetterSrc = items[1]
+	}
+
+	u, err := url.Parse(goGetterSrc)
+	if err != nil {
+		return nil, InvalidURLError{err: fmt.Sprintf("parse url: %v", err)}
+	}
+
+	if u.Scheme == "" {
+		return nil, InvalidURLError{err: fmt.Sprintf("parse url: missing scheme - probably this is a local file path? %s", goGetterSrc)}
+	}
+
+	pathComponents := strings.Split(u.Path, "@")
+	if len(pathComponents) != 2 {
+		return nil, fmt.Errorf("invalid src format: it must be `[<getter>::]<scheme>://<host>/<path/to/dir>@<path/to/file>?key1=val1&key2=val2: got %s", goGetterSrc)
+	}
+
+	return &Source{
+		Getter:   getter,
+		User:     u.User.String(),
+		Scheme:   u.Scheme,
+		Host:     u.Host,
+		Dir:      pathComponents[0],
+		File:     pathComponents[1],
+		RawQuery: u.RawQuery,
+	}, nil
+}
+
+func (r *Remote) Fetch(goGetterSrc string) (string, error) {
+	u, err := Parse(goGetterSrc)
+	if err != nil {
+		return "", err
+	}
+
+	srcDir := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, u.Dir)
+	file := u.File
+
+	r.Logger.Debugf("getter: %s", u.Getter)
+	r.Logger.Debugf("scheme: %s", u.Scheme)
+	r.Logger.Debugf("user: %s", u.User)
+	r.Logger.Debugf("host: %s", u.Host)
+	r.Logger.Debugf("dir: %s", u.Dir)
+	r.Logger.Debugf("file: %s", u.File)
+
+	// This should be shared across variant commands, so that they can share cache for the shared imports
+	cacheBaseDir := DefaultCacheDir
+
+	query := u.RawQuery
+
+	var cacheKey string
+	replacer := strings.NewReplacer(":", "", "//", "_", "/", "_", ".", "_")
+	dirKey := replacer.Replace(srcDir)
+	if len(query) > 0 {
+		paramsKey := strings.Replace(query, "&", "_", -1)
+		cacheKey = fmt.Sprintf("%s.%s", dirKey, paramsKey)
+	} else {
+		cacheKey = dirKey
+	}
+
+	cached := false
+
+	// e.g. .helmfile/cache/https_github_com_cloudposse_helmfiles_git.ref=0.xx.0
+	getterDst := filepath.Join(cacheBaseDir, cacheKey)
+
+	// e.g. $PWD/.helmfile/cache/https_github_com_cloudposse_helmfiles_git.ref=0.xx.0
+	cacheDirPath := filepath.Join(r.Home, getterDst)
+
+	r.Logger.Debugf("home: %s", r.Home)
+	r.Logger.Debugf("getter dest: %s", getterDst)
+	r.Logger.Debugf("cached dir: %s", cacheDirPath)
+
+	{
+		if r.FileExists(cacheDirPath) {
+			return "", fmt.Errorf("%s is not directory. please remove it so that variant could use it for dependency caching", getterDst)
+		}
+
+		if r.DirExists(cacheDirPath) {
+			cached = true
+		}
+	}
+
+	if !cached {
+		var getterSrc string
+		if u.User != "" {
+			getterSrc = fmt.Sprintf("%s://%s@%s%s", u.Scheme, u.User, u.Host, u.Dir)
+		} else {
+			getterSrc = fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, u.Dir)
+		}
+
+		if len(query) > 0 {
+			getterSrc = strings.Join([]string{getterSrc, query}, "?")
+		}
+
+		if u.Getter != "" {
+			getterSrc = u.Getter + "::" + getterSrc
+		}
+
+		r.Logger.Debugf("downloading %s to %s", getterSrc, getterDst)
+
+		if err := r.Getter.Get(r.Home, getterSrc, cacheDirPath); err != nil {
+			rmerr := os.RemoveAll(cacheDirPath)
+			if rmerr != nil {
+				return "", multierr.Append(err, rmerr)
+			}
+			return "", err
+		}
+	}
+
+	return filepath.Join(cacheDirPath, file), nil
+}
+
+type Getter interface {
+	Get(wd, src, dst string) error
+}
+
+type GoGetter struct {
+	Logger *zap.SugaredLogger
+}
+
+func (g *GoGetter) Get(wd, src, dst string) error {
+	ctx := context.Background()
+
+	get := &getter.Client{
+		Ctx:     ctx,
+		Src:     src,
+		Dst:     dst,
+		Pwd:     wd,
+		Mode:    getter.ClientModeDir,
+		Options: []getter.ClientOption{},
+	}
+
+	g.Logger.Debugf("client: %+v", *get)
+
+	if err := get.Get(); err != nil {
+		return fmt.Errorf("get: %v", err)
+	}
+
+	return nil
+}

pkg/remote/remote_test.go

@@ -0,0 +1,165 @@
+package remote
+
+import (
+	"fmt"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/testhelper"
+	"os"
+	"testing"
+)
+
+func TestRemote_HttpsGitHub(t *testing.T) {
+	cleanfs := map[string]string{
+		"path/to/home": "",
+	}
+	cachefs := map[string]string{
+		"path/to/home/.helmfile/cache/https_github_com_cloudposse_helmfiles_git.ref=0.40.0/releases/kiam.yaml": "foo: bar",
+	}
+
+	type testcase struct {
+		files          map[string]string
+		expectCacheHit bool
+	}
+
+	testcases := []testcase{
+		{files: cleanfs, expectCacheHit: false},
+		{files: cachefs, expectCacheHit: true},
+	}
+
+	for i := range testcases {
+		testcase := testcases[i]
+
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			testfs := testhelper.NewTestFs(testcase.files)
+
+			hit := true
+
+			get := func(wd, src, dst string) error {
+				if wd != "path/to/home" {
+					return fmt.Errorf("unexpected wd: %s", wd)
+				}
+				if src != "git::https://github.com/cloudposse/helmfiles.git?ref=0.40.0" {
+					return fmt.Errorf("unexpected src: %s", src)
+				}
+
+				hit = false
+
+				return nil
+			}
+
+			getter := &testGetter{
+				get: get,
+			}
+			remote := &Remote{
+				Logger:     helmexec.NewLogger(os.Stderr, "debug"),
+				Home:       "path/to/home",
+				Getter:     getter,
+				ReadFile:   testfs.ReadFile,
+				FileExists: testfs.FileExistsAt,
+				DirExists:  testfs.DirectoryExistsAt,
+			}
+
+			// FYI, go-getter in the `dir` mode accepts URL like the below. So helmfile expects URLs similar to it:
+			//   go-getter -mode dir git::https://github.com/cloudposse/helmfiles.git?ref=0.40.0 gettertest1/b
+
+			// We use `@` to separate dir and the file path. This is a good idea borrowed from helm-git:
+			//   https://github.com/aslafy-z/helm-git
+
+			url := "git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0"
+			file, err := remote.Locate(url)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			if file != "path/to/home/.helmfile/cache/https_github_com_cloudposse_helmfiles_git.ref=0.40.0/releases/kiam.yaml" {
+				t.Errorf("unexpected file located: %s", file)
+			}
+
+			if testcase.expectCacheHit && !hit {
+				t.Errorf("unexpected result: unexpected cache miss")
+			}
+			if !testcase.expectCacheHit && hit {
+				t.Errorf("unexpected result: unexpected cache hit")
+			}
+		})
+	}
+}
+
+func TestRemote_SShGitHub(t *testing.T) {
+	cleanfs := map[string]string{
+		"path/to/home": "",
+	}
+	cachefs := map[string]string{
+		"path/to/home/.helmfile/cache/ssh_github_com_cloudposse_helmfiles_git.ref=0.40.0/releases/kiam.yaml": "foo: bar",
+	}
+
+	type testcase struct {
+		files          map[string]string
+		expectCacheHit bool
+	}
+
+	testcases := []testcase{
+		{files: cleanfs, expectCacheHit: false},
+		{files: cachefs, expectCacheHit: true},
+	}
+
+	for i := range testcases {
+		testcase := testcases[i]
+
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			testfs := testhelper.NewTestFs(testcase.files)
+
+			hit := true
+
+			get := func(wd, src, dst string) error {
+				if wd != "path/to/home" {
+					return fmt.Errorf("unexpected wd: %s", wd)
+				}
+				if src != "git::ssh://git@github.com/cloudposse/helmfiles.git?ref=0.40.0" {
+					return fmt.Errorf("unexpected src: %s", src)
+				}
+
+				hit = false
+
+				return nil
+			}
+
+			getter := &testGetter{
+				get: get,
+			}
+			remote := &Remote{
+				Logger:     helmexec.NewLogger(os.Stderr, "debug"),
+				Home:       "path/to/home",
+				Getter:     getter,
+				ReadFile:   testfs.ReadFile,
+				FileExists: testfs.FileExistsAt,
+				DirExists:  testfs.DirectoryExistsAt,
+			}
+
+			url := "git::ssh://git@github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0"
+			file, err := remote.Locate(url)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			if file != "path/to/home/.helmfile/cache/ssh_github_com_cloudposse_helmfiles_git.ref=0.40.0/releases/kiam.yaml" {
+				t.Errorf("unexpected file located: %s", file)
+			}
+
+			if testcase.expectCacheHit && !hit {
+				t.Errorf("unexpected result: unexpected cache miss")
+			}
+			if !testcase.expectCacheHit && hit {
+				t.Errorf("unexpected result: unexpected cache hit")
+			}
+		})
+	}
+}
+
+type testGetter struct {
+	get func(wd, src, dst string) error
+}
+
+func (t *testGetter) Get(wd, src, dst string) error {
+	return t.get(wd, src, dst)
+}

pkg/state/chart_dependency.go

@@ -0,0 +1,433 @@
+package state
+
+import (
+	"fmt"
+	"github.com/Masterminds/semver"
+	"github.com/r3labs/diff"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"go.uber.org/zap"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+type ChartMeta struct {
+	Name string `yaml:"name"`
+}
+
+type unresolvedChartDependency struct {
+	// ChartName identifies the dependant chart. In Helmfile, ChartName for `chart: stable/envoy` would be just `envoy`.
+	// It can't be collided with other charts referenced in the same helmfile spec.
+	// That is, collocating `chart: incubator/foo` and `chart: stable/foo` isn't allowed. Name them differently for a work-around.
+	ChartName string `yaml:"name"`
+	// Repository contains the URL for the helm chart repository that hosts the chart identified by ChartName
+	Repository string `yaml:"repository"`
+	// VersionConstraint is the version constraint of the dependent chart. "*" means the latest version.
+	VersionConstraint string `yaml:"version"`
+}
+
+type ResolvedChartDependency struct {
+	// ChartName identifies the dependant chart. In Helmfile, ChartName for `chart: stable/envoy` would be just `envoy`.
+	// It can't be collided with other charts referenced in the same helmfile spec.
+	// That is, collocating `chart: incubator/foo` and `chart: stable/foo` isn't allowed. Name them differently for a work-around.
+	ChartName string `yaml:"name"`
+	// Repository contains the URL for the helm chart repository that hosts the chart identified by ChartName
+	Repository string `yaml:"repository"`
+	// Version is the version number of the dependent chart.
+	// In the context of helmfile this can be omitted. When omitted, it is considered `*` which results helm/helmfile fetching the latest version.
+	Version string `yaml:"version"`
+}
+
+type UnresolvedDependencies struct {
+	deps map[string][]unresolvedChartDependency
+}
+
+type ChartRequirements struct {
+	UnresolvedDependencies []unresolvedChartDependency `yaml:"dependencies"`
+}
+
+type ChartLockedRequirements struct {
+	ResolvedDependencies []ResolvedChartDependency `yaml:"dependencies"`
+	Digest               string                    `yaml:"digest"`
+	Generated            string                    `yaml:"generated"`
+}
+
+func (d *UnresolvedDependencies) Add(chart, url, versionConstraint string) error {
+	dep := unresolvedChartDependency{
+		ChartName:         chart,
+		Repository:        url,
+		VersionConstraint: versionConstraint,
+	}
+	return d.add(dep)
+}
+
+func (d *UnresolvedDependencies) add(dep unresolvedChartDependency) error {
+	deps := d.deps[dep.ChartName]
+	if deps == nil {
+		deps = []unresolvedChartDependency{dep}
+	} else {
+		deps = append(deps, dep)
+	}
+	d.deps[dep.ChartName] = deps
+	return nil
+}
+
+func (d *UnresolvedDependencies) ToChartRequirements() *ChartRequirements {
+	deps := []unresolvedChartDependency{}
+
+	for _, ds := range d.deps {
+		for _, d := range ds {
+			if d.VersionConstraint == "" {
+				d.VersionConstraint = "*"
+			}
+			deps = append(deps, d)
+		}
+	}
+
+	return &ChartRequirements{UnresolvedDependencies: deps}
+}
+
+type ResolvedDependencies struct {
+	deps map[string][]ResolvedChartDependency
+}
+
+func (d *ResolvedDependencies) add(dep ResolvedChartDependency) error {
+	deps := d.deps[dep.ChartName]
+	if deps == nil {
+		deps = []ResolvedChartDependency{dep}
+	} else {
+		deps = append(deps, dep)
+	}
+	d.deps[dep.ChartName] = deps
+	return nil
+}
+
+func (d *ResolvedDependencies) Get(chart, versionConstraint string) (string, error) {
+	if versionConstraint == "" {
+		versionConstraint = "*"
+	}
+
+	deps, exists := d.deps[chart]
+	if exists {
+		for _, dep := range deps {
+			constraint, err := semver.NewConstraint(versionConstraint)
+			if err != nil {
+				return "", err
+			}
+			version, err := semver.NewVersion(dep.Version)
+			if err != nil {
+				return "", err
+			}
+			if constraint.Check(version) {
+				return dep.Version, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("no resolved dependency found for \"%s\"", chart)
+}
+
+func (st *HelmState) mergeLockedDependencies() (*HelmState, error) {
+	filename, unresolved, err := getUnresolvedDependenciess(st)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(unresolved.deps) == 0 {
+		return st, nil
+	}
+
+	depMan := NewChartDependencyManager(filename, st.logger)
+
+	if st.readFile != nil {
+		depMan.readFile = st.readFile
+	}
+
+	return resolveDependencies(st, depMan, unresolved)
+}
+
+func resolveDependencies(st *HelmState, depMan *chartDependencyManager, unresolved *UnresolvedDependencies) (*HelmState, error) {
+	resolved, lockfileExists, err := depMan.Resolve(unresolved)
+	if err != nil {
+		return nil, fmt.Errorf("unable to resolve %d deps: %v", len(unresolved.deps), err)
+	}
+	if !lockfileExists {
+		return st, nil
+	}
+
+	repoToURL := map[string]string{}
+
+	for _, r := range st.Repositories {
+		repoToURL[r.Name] = r.URL
+	}
+
+	updated := *st
+	for i, r := range updated.Releases {
+		repo, chart, ok := resolveRemoteChart(r.Chart)
+		if !ok {
+			continue
+		}
+
+		_, ok = repoToURL[repo]
+		// Skip this chart from dependency management, as there's no matching `repository` in the helmfile state,
+		// which may imply that this is a local chart within a directory, like `charts/myapp`
+		if !ok {
+			continue
+		}
+
+		ver, err := resolved.Get(chart, r.Version)
+		if err != nil {
+			return nil, err
+		}
+
+		updated.Releases[i].Version = ver
+	}
+
+	return &updated, nil
+}
+
+func (st *HelmState) updateDependenciesInTempDir(shell helmexec.DependencyUpdater, tempDir func(string, string) (string, error)) (*HelmState, error) {
+	filename, unresolved, err := getUnresolvedDependenciess(st)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(unresolved.deps) == 0 {
+		st.logger.Warnf("There are no repositories defined in your helmfile.yaml.\nThis means helmfile cannot update your dependencies or create a lock file.\nSee https://github.com/roboll/helmfile/issues/878 for more information.")
+		return st, nil
+	}
+
+	d, err := tempDir("", "")
+	if err != nil {
+		return nil, fmt.Errorf("unable to create dir: %v", err)
+	}
+	defer os.RemoveAll(d)
+
+	return updateDependencies(st, shell, unresolved, filename, d)
+}
+
+func getUnresolvedDependenciess(st *HelmState) (string, *UnresolvedDependencies, error) {
+	repoToURL := map[string]string{}
+
+	for _, r := range st.Repositories {
+		repoToURL[r.Name] = r.URL
+	}
+
+	unresolved := &UnresolvedDependencies{deps: map[string][]unresolvedChartDependency{}}
+	//if err := unresolved.Add("stable/envoy", "https://kubernetes-charts.storage.googleapis.com", ""); err != nil {
+	//	panic(err)
+	//}
+
+	for _, r := range st.Releases {
+		repo, chart, ok := resolveRemoteChart(r.Chart)
+		if !ok {
+			continue
+		}
+
+		url, ok := repoToURL[repo]
+		// Skip this chart from dependency management, as there's no matching `repository` in the helmfile state,
+		// which may imply that this is a local chart within a directory, like `charts/myapp`
+		if !ok {
+			continue
+		}
+
+		if err := unresolved.Add(chart, url, r.Version); err != nil {
+			return "", nil, err
+		}
+	}
+
+	filename := filepath.Base(st.FilePath)
+	filename = strings.TrimSuffix(filename, ".gotmpl")
+	filename = strings.TrimSuffix(filename, ".yaml")
+	filename = strings.TrimSuffix(filename, ".yml")
+
+	return filename, unresolved, nil
+}
+
+func updateDependencies(st *HelmState, shell helmexec.DependencyUpdater, unresolved *UnresolvedDependencies, filename, wd string) (*HelmState, error) {
+	depMan := NewChartDependencyManager(filename, st.logger)
+
+	_, err := depMan.Update(shell, wd, unresolved)
+	if err != nil {
+		return nil, fmt.Errorf("unable to resolve %d deps: %v", len(unresolved.deps), err)
+	}
+
+	return resolveDependencies(st, depMan, unresolved)
+}
+
+type chartDependencyManager struct {
+	Name string
+
+	logger *zap.SugaredLogger
+
+	readFile  func(string) ([]byte, error)
+	writeFile func(string, []byte, os.FileMode) error
+}
+
+func NewChartDependencyManager(name string, logger *zap.SugaredLogger) *chartDependencyManager {
+	return &chartDependencyManager{
+		Name:      name,
+		readFile:  ioutil.ReadFile,
+		writeFile: ioutil.WriteFile,
+		logger:    logger,
+	}
+}
+
+func (m *chartDependencyManager) lockFileName() string {
+	return fmt.Sprintf("%s.lock", m.Name)
+}
+
+func (m *chartDependencyManager) Update(shell helmexec.DependencyUpdater, wd string, unresolved *UnresolvedDependencies) (*ResolvedDependencies, error) {
+	if shell.IsHelm3() {
+		return m.updateHelm3(shell, wd, unresolved)
+	}
+	return m.updateHelm2(shell, wd, unresolved)
+}
+
+func (m *chartDependencyManager) updateHelm3(shell helmexec.DependencyUpdater, wd string, unresolved *UnresolvedDependencies) (*ResolvedDependencies, error) {
+	// Generate `Chart.yaml` of the temporary local chart
+	chartMetaContent := fmt.Sprintf("name: %s\nversion: 1.0.0\n", m.Name)
+
+	// Generate `requirements.yaml` of the temporary local chart from the helmfile state
+	reqsContent, err := yaml.Marshal(unresolved.ToChartRequirements())
+	if err != nil {
+		return nil, err
+	}
+	if err := m.writeBytes(filepath.Join(wd, "Chart.yaml"), []byte(chartMetaContent+string(reqsContent))); err != nil {
+		return nil, err
+	}
+
+	return m.doUpdate("Chart.lock", unresolved, shell, wd)
+}
+
+func (m *chartDependencyManager) updateHelm2(shell helmexec.DependencyUpdater, wd string, unresolved *UnresolvedDependencies) (*ResolvedDependencies, error) {
+	// Generate `Chart.yaml` of the temporary local chart
+	if err := m.writeBytes(filepath.Join(wd, "Chart.yaml"), []byte(fmt.Sprintf("name: %s\nversion: 1.0.0\n", m.Name))); err != nil {
+		return nil, err
+	}
+
+	// Generate `requirements.yaml` of the temporary local chart from the helmfile state
+	reqsContent, err := yaml.Marshal(unresolved.ToChartRequirements())
+	if err != nil {
+		return nil, err
+	}
+	if err := m.writeBytes(filepath.Join(wd, "requirements.yaml"), reqsContent); err != nil {
+		return nil, err
+	}
+
+	return m.doUpdate("requirements.lock", unresolved, shell, wd)
+}
+
+func (m *chartDependencyManager) doUpdate(chartLockFile string, unresolved *UnresolvedDependencies, shell helmexec.DependencyUpdater, wd string) (*ResolvedDependencies, error) {
+
+	// Generate `requirements.lock` of the temporary local chart by coping `<basename>.lock`
+	lockFile := m.lockFileName()
+
+	originalLockFileContent, err := m.readBytes(lockFile)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+
+	if shell.IsHelm3() && originalLockFileContent != nil {
+		if err := m.writeBytes(filepath.Join(wd, chartLockFile), originalLockFileContent); err != nil {
+			return nil, err
+		}
+	}
+
+	// Update the lock file by running `helm dependency update`
+	if err := shell.UpdateDeps(wd); err != nil {
+		return nil, err
+	}
+
+	updatedLockFileContent, err := m.readBytes(filepath.Join(wd, chartLockFile))
+	if err != nil {
+		return nil, err
+	}
+
+	// Sort requirements alphabetically by name.
+	lockedReqs := &ChartLockedRequirements{}
+	if err := yaml.Unmarshal(updatedLockFileContent, lockedReqs); err != nil {
+		return nil, err
+	}
+
+	sort.Slice(lockedReqs.ResolvedDependencies, func(i, j int) bool {
+		return lockedReqs.ResolvedDependencies[i].ChartName < lockedReqs.ResolvedDependencies[j].ChartName
+	})
+
+	// Don't update lock file if no dependency updated.
+	if !shell.IsHelm3() && originalLockFileContent != nil {
+		originalLockedReqs := &ChartLockedRequirements{}
+		if err := yaml.Unmarshal(originalLockFileContent, originalLockedReqs); err != nil {
+			return nil, err
+		}
+
+		changes, err := diff.Diff(originalLockedReqs.ResolvedDependencies, lockedReqs.ResolvedDependencies)
+
+		if err != nil {
+			return nil, err
+		}
+
+		if len(changes) == 0 {
+			lockedReqs.Generated = originalLockedReqs.Generated
+		}
+	}
+
+	updatedLockFileContent, err = yaml.Marshal(lockedReqs)
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Commit the lock file if and only if everything looks ok
+	if err := m.writeBytes(lockFile, updatedLockFileContent); err != nil {
+		return nil, err
+	}
+
+	resolved, _, err := m.Resolve(unresolved)
+	return resolved, err
+}
+
+func (m *chartDependencyManager) Resolve(unresolved *UnresolvedDependencies) (*ResolvedDependencies, bool, error) {
+	updatedLockFileContent, err := m.readBytes(m.lockFileName())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, false, nil
+		}
+		return nil, false, err
+	}
+
+	// Load resolved dependencies into memory
+	lockedReqs := &ChartLockedRequirements{}
+	if err := yaml.Unmarshal(updatedLockFileContent, lockedReqs); err != nil {
+		return nil, false, err
+	}
+
+	resolved := &ResolvedDependencies{deps: map[string][]ResolvedChartDependency{}}
+	for _, d := range lockedReqs.ResolvedDependencies {
+		if err := resolved.add(d); err != nil {
+			return nil, false, err
+		}
+	}
+
+	return resolved, true, nil
+}
+
+func (m *chartDependencyManager) readBytes(filename string) ([]byte, error) {
+	bytes, err := m.readFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	m.logger.Debugf("readBytes: read from %s:\n%s", filename, bytes)
+	return bytes, nil
+}
+
+func (m *chartDependencyManager) writeBytes(filename string, data []byte) error {
+	err := m.writeFile(filename, data, 0644)
+	if err != nil {
+		return err
+	}
+	m.logger.Debugf("writeBytes: wrote to %s:\n%s", filename, data)
+	return nil
+}

pkg/state/create.go

@@ -0,0 +1,321 @@
+package state
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/imdario/mergo"
+	"github.com/roboll/helmfile/pkg/environment"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/maputil"
+	"github.com/variantdev/vals"
+	"go.uber.org/zap"
+	"gopkg.in/yaml.v2"
+)
+
+type StateLoadError struct {
+	msg   string
+	Cause error
+}
+
+func (e *StateLoadError) Error() string {
+	return fmt.Sprintf("%s: %v", e.msg, e.Cause)
+}
+
+type UndefinedEnvError struct {
+	msg string
+}
+
+func (e *UndefinedEnvError) Error() string {
+	return e.msg
+}
+
+type StateCreator struct {
+	logger      *zap.SugaredLogger
+	readFile    func(string) ([]byte, error)
+	fileExists  func(string) (bool, error)
+	abs         func(string) (string, error)
+	glob        func(string) ([]string, error)
+	helm        helmexec.Interface
+	valsRuntime vals.Evaluator
+
+	Strict bool
+
+	LoadFile func(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*HelmState, error)
+}
+
+func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), helm helmexec.Interface, valsRuntime vals.Evaluator) *StateCreator {
+	return &StateCreator{
+		logger:      logger,
+		readFile:    readFile,
+		fileExists:  fileExists,
+		abs:         abs,
+		glob:        glob,
+		Strict:      true,
+		helm:        helm,
+		valsRuntime: valsRuntime,
+	}
+}
+
+// Parse parses YAML into HelmState
+func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState, error) {
+	var state HelmState
+
+	state.FilePath = file
+	state.basePath = baseDir
+	state.helm = c.helm
+
+	decoder := yaml.NewDecoder(bytes.NewReader(content))
+	if !c.Strict {
+		decoder.SetStrict(false)
+	} else {
+		decoder.SetStrict(true)
+	}
+	i := 0
+	for {
+		i++
+
+		var intermediate HelmState
+
+		err := decoder.Decode(&intermediate)
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, &StateLoadError{fmt.Sprintf("failed to read %s: reading document at index %d", file, i), err}
+		}
+
+		if err := mergo.Merge(&state, &intermediate, mergo.WithAppendSlice); err != nil {
+			return nil, &StateLoadError{fmt.Sprintf("failed to read %s: merging document at index %d", file, i), err}
+		}
+	}
+
+	if len(state.DeprecatedReleases) > 0 {
+		if len(state.Releases) > 0 {
+			return nil, fmt.Errorf("failed to parse %s: you can't specify both `charts` and `releases` sections", file)
+		}
+		state.Releases = state.DeprecatedReleases
+		state.DeprecatedReleases = []ReleaseSpec{}
+	}
+
+	if state.DeprecatedContext != "" && state.HelmDefaults.KubeContext == "" {
+		state.HelmDefaults.KubeContext = state.DeprecatedContext
+	}
+
+	state.logger = c.logger
+
+	state.readFile = c.readFile
+	state.removeFile = os.Remove
+	state.fileExists = c.fileExists
+	state.glob = c.glob
+	state.valsRuntime = c.valsRuntime
+
+	return &state, nil
+}
+
+// LoadEnvValues loads environment values files relative to the `baseDir`
+func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *environment.Environment) (*HelmState, error) {
+	state := *target
+
+	e, err := state.loadEnvValues(env, ctxEnv, c.readFile, c.glob)
+	if err != nil {
+		return nil, &StateLoadError{fmt.Sprintf("failed to read %s", state.FilePath), err}
+	}
+
+	e.Defaults, err = state.loadValuesEntries(nil, state.DefaultValues)
+	if err != nil {
+		return nil, err
+	}
+
+	state.Env = *e
+
+	return &state, nil
+}
+
+// Parses YAML into HelmState, while loading environment values files relative to the `baseDir`
+func (c *StateCreator) ParseAndLoad(content []byte, baseDir, file string, envName string, evaluateBases bool, envValues *environment.Environment) (*HelmState, error) {
+	state, err := c.Parse(content, baseDir, file)
+	if err != nil {
+		return nil, err
+	}
+
+	if !evaluateBases {
+		if len(state.Bases) > 0 {
+			return nil, errors.New("nested `base` helmfile is unsupported. please submit a feature request if you need this!")
+		}
+	}
+
+	state, err = c.loadBases(envValues, state, baseDir)
+	if err != nil {
+		return nil, err
+	}
+
+	state, err = c.LoadEnvValues(state, envName, envValues)
+	if err != nil {
+		return nil, err
+	}
+
+	state.FilePath = file
+
+	return state, nil
+}
+
+func (c *StateCreator) loadBases(envValues *environment.Environment, st *HelmState, baseDir string) (*HelmState, error) {
+	layers := []*HelmState{}
+	for _, b := range st.Bases {
+		base, err := c.LoadFile(envValues, baseDir, b, false)
+		if err != nil {
+			return nil, err
+		}
+		layers = append(layers, base)
+	}
+	layers = append(layers, st)
+
+	for i := 1; i < len(layers); i++ {
+		if err := mergo.Merge(layers[0], layers[i], mergo.WithAppendSlice); err != nil {
+			return nil, err
+		}
+	}
+
+	return layers[0], nil
+}
+
+func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, readFile func(string) ([]byte, error), glob func(string) ([]string, error)) (*environment.Environment, error) {
+	envVals := map[string]interface{}{}
+	envSpec, ok := st.Environments[name]
+	if ok {
+		var err error
+		envVals, err = st.loadValuesEntries(envSpec.MissingFileHandler, envSpec.Values)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(envSpec.Secrets) > 0 {
+
+			var envSecretFiles []string
+			for _, urlOrPath := range envSpec.Secrets {
+				resolved, skipped, err := st.storage().resolveFile(envSpec.MissingFileHandler, "environment values", urlOrPath)
+				if err != nil {
+					return nil, err
+				}
+				if skipped {
+					continue
+				}
+
+				envSecretFiles = append(envSecretFiles, resolved...)
+			}
+			if err = st.scatterGatherEnvSecretFiles(envSecretFiles, envVals, readFile); err != nil {
+				return nil, err
+			}
+		}
+	} else if ctxEnv == nil && name != DefaultEnv {
+		return nil, &UndefinedEnvError{msg: fmt.Sprintf("environment \"%s\" is not defined", name)}
+	}
+
+	newEnv := &environment.Environment{Name: name, Values: envVals}
+
+	if ctxEnv != nil {
+		intEnv := *ctxEnv
+
+		if err := mergo.Merge(&intEnv, newEnv, mergo.WithOverride); err != nil {
+			return nil, fmt.Errorf("error while merging environment values for \"%s\": %v", name, err)
+		}
+
+		newEnv = &intEnv
+	}
+
+	return newEnv, nil
+}
+
+func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, envVals map[string]interface{}, readFile func(string) ([]byte, error)) error {
+	var errs []error
+
+	inputs := envSecretFiles
+	inputsSize := len(inputs)
+
+	type secretResult struct {
+		result map[string]interface{}
+		err    error
+		path   string
+	}
+
+	secrets := make(chan string, inputsSize)
+	results := make(chan secretResult, inputsSize)
+	helm := st.helm
+
+	st.scatterGather(0, inputsSize,
+		func() {
+			for _, secretFile := range envSecretFiles {
+				secrets <- secretFile
+			}
+			close(secrets)
+		},
+		func(id int) {
+			for path := range secrets {
+				release := &ReleaseSpec{}
+				flags := st.appendConnectionFlags([]string{}, release)
+				decFile, err := helm.DecryptSecret(st.createHelmContext(release, 0), path, flags...)
+				if err != nil {
+					results <- secretResult{nil, err, path}
+					continue
+				}
+				bytes, err := readFile(decFile)
+				if err != nil {
+					results <- secretResult{nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err), path}
+					continue
+				}
+				m := map[string]interface{}{}
+				if err := yaml.Unmarshal(bytes, &m); err != nil {
+					results <- secretResult{nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err), path}
+					continue
+				}
+				// All the nested map key should be string. Otherwise we get strange errors due to that
+				// mergo or reflect is unable to merge map[interface{}]interface{} with map[string]interface{} or vice versa.
+				// See https://github.com/roboll/helmfile/issues/677
+				vals, err := maputil.CastKeysToStrings(m)
+				if err != nil {
+					results <- secretResult{nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err), path}
+					continue
+				}
+				results <- secretResult{vals, nil, path}
+			}
+		},
+		func() {
+			for i := 0; i < inputsSize; i++ {
+				result := <-results
+				if result.err != nil {
+					errs = append(errs, result.err)
+				} else {
+					if err := mergo.Merge(&envVals, &result.result, mergo.WithOverride); err != nil {
+						errs = append(errs, fmt.Errorf("failed to load environment secrets file \"%s\": %v", result.path, err))
+					}
+				}
+			}
+			close(results)
+		},
+	)
+
+	if len(errs) > 1 {
+		for _, err := range errs {
+			st.logger.Error(err)
+		}
+		return fmt.Errorf("Failed loading environment secrets with %d errors", len(errs))
+	}
+	return nil
+}
+
+func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []interface{}) (map[string]interface{}, error) {
+	envVals := map[string]interface{}{}
+
+	valuesEntries := append([]interface{}{}, entries...)
+	ld := NewEnvironmentValuesLoader(st.storage(), st.readFile, st.logger)
+	var err error
+	envVals, err = ld.LoadEnvironmentValues(missingFileHandler, valuesEntries)
+	if err != nil {
+		return nil, err
+	}
+
+	return envVals, nil
+}

pkg/state/create_test.go

@@ -1,25 +1,26 @@
 package state
 
 import (
-	"fmt"
-	"go.uber.org/zap"
 	"io/ioutil"
 	"path/filepath"
 	"reflect"
 	"testing"
 
+	"github.com/roboll/helmfile/pkg/testhelper"
+	"go.uber.org/zap"
+
 	. "gotest.tools/assert"
 	"gotest.tools/assert/cmp"
 )
 
 func createFromYaml(content []byte, file string, env string, logger *zap.SugaredLogger) (*HelmState, error) {
-	c := &creator{
-		logger,
-		ioutil.ReadFile,
-		filepath.Abs,
-		true,
+	c := &StateCreator{
+		logger:   logger,
+		readFile: ioutil.ReadFile,
+		abs:      filepath.Abs,
+		Strict:   true,
 	}
-	return c.ParseAndLoadEnv(content, filepath.Dir(file), file, env)
+	return c.ParseAndLoad(content, filepath.Dir(file), file, env, false, nil)
 }
 
 func TestReadFromYaml(t *testing.T) {
@@ -99,23 +100,17 @@ bar: {{ readFile "bar.txt" }}
 
 	expectedValues := `env: production`
 
-	readFile := func(filename string) ([]byte, error) {
-		switch filename {
-		case fooYamlFile:
-			return fooYamlContent, nil
-		case barYamlFile:
-			return barYamlContent, nil
-		case barTextFile:
-			return barTextContent, nil
-		case valuesFile:
-			return valuesContent, nil
-		}
-		return nil, fmt.Errorf("unexpected filename: %s", filename)
-	}
+	testFs := testhelper.NewTestFs(map[string]string{
+		fooYamlFile: string(fooYamlContent),
+		barYamlFile: string(barYamlContent),
+		barTextFile: string(barTextContent),
+		valuesFile:  string(valuesContent),
+	})
+	testFs.Cwd = "/example/path/to"
 
-	state, err := NewCreator(logger, readFile, filepath.Abs).ParseAndLoadEnv(yamlContent, filepath.Dir(yamlFile), yamlFile, "production")
+	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil, nil).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", false, nil)
 	if err != nil {
-		t.Errorf("unexpected error: %v", err)
+		t.Fatalf("unexpected error: %v", err)
 	}
 
 	actual := state.Env.Values

pkg/state/environment.go

@@ -0,0 +1,15 @@
+package state
+
+type EnvironmentSpec struct {
+	Values  []interface{} `yaml:"values,omitempty"`
+	Secrets []string      `yaml:"secrets,omitempty"`
+
+	// MissingFileHandler instructs helmfile to fail when unable to find a environment values file listed
+	// under `environments.NAME.values`.
+	//
+	// Possible values are  "Error", "Warn", "Info", "Debug". The default is "Error".
+	//
+	// Use "Warn", "Info", or "Debug" if you want helmfile to not fail when a values file is missing, while just leaving
+	// a message about the missing file at the log-level.
+	MissingFileHandler *string `yaml:"missingFileHandler,omitempty"`
+}

pkg/state/envvals_loader.go

@@ -0,0 +1,83 @@
+package state
+
+import (
+	"fmt"
+	"github.com/imdario/mergo"
+	"github.com/roboll/helmfile/pkg/environment"
+	"github.com/roboll/helmfile/pkg/maputil"
+	"github.com/roboll/helmfile/pkg/tmpl"
+	"go.uber.org/zap"
+	"gopkg.in/yaml.v2"
+	"path/filepath"
+)
+
+type EnvironmentValuesLoader struct {
+	storage *Storage
+
+	readFile func(string) ([]byte, error)
+
+	logger *zap.SugaredLogger
+}
+
+func NewEnvironmentValuesLoader(storage *Storage, readFile func(string) ([]byte, error), logger *zap.SugaredLogger) *EnvironmentValuesLoader {
+	return &EnvironmentValuesLoader{
+		storage:  storage,
+		readFile: readFile,
+		logger:   logger,
+	}
+}
+
+func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *string, valuesEntries []interface{}) (map[string]interface{}, error) {
+	result := map[string]interface{}{}
+
+	for _, entry := range valuesEntries {
+		maps := []interface{}{}
+
+		switch strOrMap := entry.(type) {
+		case string:
+			urlOrPath := strOrMap
+			files, skipped, err := ld.storage.resolveFile(missingFileHandler, "environment values", urlOrPath)
+			if err != nil {
+				return nil, err
+			}
+			if skipped {
+				continue
+			}
+
+			for _, f := range files {
+				tmplData := EnvironmentTemplateData{environment.EmptyEnvironment, "", map[string]interface{}{}}
+				r := tmpl.NewFileRenderer(ld.readFile, filepath.Dir(f), tmplData)
+				bytes, err := r.RenderToBytes(f)
+				if err != nil {
+					return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", f, err)
+				}
+				m := map[string]interface{}{}
+				if err := yaml.Unmarshal(bytes, &m); err != nil {
+					return nil, fmt.Errorf("failed to load environment values file \"%s\": %v\n\nOffending YAML:\n%s", f, err, bytes)
+				}
+				maps = append(maps, m)
+				if ld.logger != nil {
+					ld.logger.Debugf("envvals_loader: loaded %s:%v", strOrMap, m)
+				}
+			}
+		case map[interface{}]interface{}:
+			maps = append(maps, strOrMap)
+		default:
+			return nil, fmt.Errorf("unexpected type of value: value=%v, type=%T", strOrMap, strOrMap)
+		}
+		for _, m := range maps {
+			// All the nested map key should be string. Otherwise we get strange errors due to that
+			// mergo or reflect is unable to merge map[interface{}]interface{} with map[string]interface{} or vice versa.
+			// See https://github.com/roboll/helmfile/issues/677
+			vals, err := maputil.CastKeysToStrings(m)
+			if err != nil {
+				return nil, err
+			}
+			if err := mergo.Merge(&result, &vals, mergo.WithOverride); err != nil {
+				return nil, fmt.Errorf("failed to merge %v: %v", m, err)
+			}
+		}
+	}
+
+	return result, nil
+}

pkg/state/helmx.go

@@ -0,0 +1,55 @@
+package state
+
+type Dependency struct {
+	Chart   string `yaml:"chart"`
+	Version string `yaml:"version"`
+	Alias   string `yaml:"alias"`
+}
+
+func (st *HelmState) appendHelmXFlags(flags []string, release *ReleaseSpec) ([]string, error) {
+	for _, d := range release.Dependencies {
+		var dep string
+		if d.Alias != "" {
+			dep += d.Alias + "="
+		}
+		dep += d.Chart
+		if d.Version != "" {
+			dep += ":" + d.Version
+		}
+		flags = append(flags, "--dependency", dep)
+	}
+
+	for _, adopt := range release.Adopt {
+		flags = append(flags, "--adopt", adopt)
+	}
+
+	jsonPatches := release.JSONPatches
+	if len(jsonPatches) > 0 {
+		generatedFiles, err := st.generateTemporaryValuesFiles(jsonPatches, release.MissingFileHandler)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, f := range generatedFiles {
+			flags = append(flags, "--json-patch", f)
+		}
+
+		release.generatedValues = append(release.generatedValues, generatedFiles...)
+	}
+
+	strategicMergePatches := release.StrategicMergePatches
+	if len(strategicMergePatches) > 0 {
+		generatedFiles, err := st.generateTemporaryValuesFiles(strategicMergePatches, release.MissingFileHandler)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, f := range generatedFiles {
+			flags = append(flags, "--strategic-merge-patch", f)
+		}
+
+		release.generatedValues = append(release.generatedValues, generatedFiles...)
+	}
+
+	return flags, nil
+}

pkg/state/release.go

@@ -0,0 +1,208 @@
+package state
+
+import (
+	"fmt"
+
+	"github.com/roboll/helmfile/pkg/tmpl"
+	"gopkg.in/yaml.v2"
+)
+
+func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*ReleaseSpec, error) {
+	var result *ReleaseSpec
+	var err error
+
+	result, err = r.Clone()
+	if err != nil {
+		return nil, fmt.Errorf("failed executing template expressions in release \"%s\": %v", r.Name, err)
+	}
+
+	{
+		ts := result.Name
+		result.Name, err = renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".name = \"%s\": %v", r.Name, ts, err)
+		}
+	}
+
+	{
+		ts := result.Chart
+		result.Chart, err = renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".chart = \"%s\": %v", r.Name, ts, err)
+		}
+	}
+
+	{
+		ts := result.Namespace
+		result.Namespace, err = renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".namespace = \"%s\": %v", r.Name, ts, err)
+		}
+	}
+
+	{
+		ts := result.Version
+		result.Version, err = renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err)
+		}
+	}
+
+	if result.WaitTemplate != nil {
+		ts := *result.WaitTemplate
+		resultTmpl, err := renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err)
+		}
+		result.WaitTemplate = &resultTmpl
+	}
+
+	if result.InstalledTemplate != nil {
+		ts := *result.InstalledTemplate
+		resultTmpl, err := renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err)
+		}
+		result.InstalledTemplate = &resultTmpl
+	}
+
+	if result.TillerlessTemplate != nil {
+		ts := *result.TillerlessTemplate
+		resultTmpl, err := renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err)
+		}
+		result.TillerlessTemplate = &resultTmpl
+	}
+
+	if result.VerifyTemplate != nil {
+		ts := *result.VerifyTemplate
+		resultTmpl, err := renderer.RenderTemplateContentToString([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err)
+		}
+		result.VerifyTemplate = &resultTmpl
+	}
+
+	for key, val := range result.Labels {
+		ts := val
+		s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".labels[%s] = \"%s\": %v", r.Name, key, ts, err)
+		}
+		result.Labels[key] = s.String()
+	}
+
+	if len(result.ValuesTemplate) > 0 {
+		for i, t := range result.ValuesTemplate {
+			switch ts := t.(type) {
+			case map[interface{}]interface{}:
+				serialized, err := yaml.Marshal(ts)
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err)
+				}
+
+				s, err := renderer.RenderTemplateContentToBuffer([]byte(serialized))
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, serialized, err)
+				}
+
+				var deserialized map[interface{}]interface{}
+
+				if err := yaml.Unmarshal(s.Bytes(), &deserialized); err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err)
+				}
+
+				result.ValuesTemplate[i] = deserialized
+			}
+		}
+
+		var newvals []interface{}
+		newvals = append(newvals, result.ValuesTemplate...)
+		result.Values = append(newvals, result.Values...)
+	}
+
+	for i, t := range result.Values {
+		switch ts := t.(type) {
+		case string:
+			s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
+			if err != nil {
+				return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%s\": %v", r.Name, i, ts, err)
+			}
+			result.Values[i] = s.String()
+		}
+	}
+
+	for i, ts := range result.Secrets {
+		s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
+		if err != nil {
+			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".secrets[%d] = \"%s\": %v", r.Name, i, ts, err)
+		}
+		result.Secrets[i] = s.String()
+	}
+
+	if len(result.SetValuesTemplate) > 0 {
+		for i, val := range result.SetValuesTemplate {
+			{
+				// name
+				ts := val.Name
+				s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].name = \"%s\": %v", r.Name, i, ts, err)
+				}
+				result.SetValuesTemplate[i].Name = s.String()
+			}
+			{
+				// value
+				ts := val.Value
+				s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].value = \"%s\": %v", r.Name, i, ts, err)
+				}
+				result.SetValuesTemplate[i].Value = s.String()
+			}
+			{
+				// file
+				ts := val.File
+				s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].file = \"%s\": %v", r.Name, i, ts, err)
+				}
+				result.SetValuesTemplate[i].File = s.String()
+			}
+			for j, ts := range val.Values {
+				// values
+				s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].values[%d] = \"%s\": %v", r.Name, i, j, ts, err)
+				}
+				result.SetValuesTemplate[i].Values[j] = s.String()
+			}
+		}
+
+		var newvals []SetValue
+		newvals = append(newvals, result.SetValuesTemplate...)
+		result.SetValues = append(newvals, result.SetValues...)
+	}
+
+	return result, nil
+}
+
+func (r ReleaseSpec) Clone() (*ReleaseSpec, error) {
+	serialized, err := yaml.Marshal(r)
+	if err != nil {
+		return nil, fmt.Errorf("failed cloning release \"%s\": %v", r.Name, err)
+	}
+
+	var deserialized ReleaseSpec
+
+	if err := yaml.Unmarshal(serialized, &deserialized); err != nil {
+		return nil, fmt.Errorf("failed cloning release \"%s\": %v", r.Name, err)
+	}
+
+	return &deserialized, nil
+}
+
+func (r ReleaseSpec) Desired() bool {
+	return r.Installed == nil || *r.Installed
+}

pkg/state/release_error.go

@@ -1,6 +1,8 @@
 package state
 
-import "fmt"
+import (
+	"fmt"
+)
 
 const ReleaseErrorCodeFailure = 1
 

pkg/state/state_exec_tmpl.go

@@ -0,0 +1,134 @@
+package state
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/imdario/mergo"
+	"github.com/roboll/helmfile/pkg/maputil"
+	"github.com/roboll/helmfile/pkg/tmpl"
+	"gopkg.in/yaml.v2"
+)
+
+func (st *HelmState) Values() (map[string]interface{}, error) {
+	vals := map[string]interface{}{}
+
+	if err := mergo.Merge(&vals, st.Env.Defaults, mergo.WithOverride); err != nil {
+		return nil, err
+	}
+	if err := mergo.Merge(&vals, st.Env.Values, mergo.WithOverride); err != nil {
+		return nil, err
+	}
+
+	vals, err := maputil.CastKeysToStrings(vals)
+	if err != nil {
+		return nil, err
+	}
+
+	return vals, nil
+}
+
+func (st *HelmState) mustLoadVals() map[string]interface{} {
+	vals, err := st.Values()
+	if err != nil {
+		panic(err)
+	}
+	return vals
+}
+
+func (st *HelmState) valuesFileTemplateData() EnvironmentTemplateData {
+	return EnvironmentTemplateData{
+		Environment: st.Env,
+		Namespace:   st.OverrideNamespace,
+		Values:      st.mustLoadVals(),
+	}
+}
+
+func getBoolRefFromStringTemplate(templateRef string) (*bool, error) {
+	var result bool
+	if err := yaml.Unmarshal([]byte(templateRef), &result); err != nil {
+		return nil, fmt.Errorf("failed deserialising string %s: %v", templateRef, err)
+	}
+	return &result, nil
+}
+
+func updateBoolTemplatedValues(r *ReleaseSpec) error {
+
+	if r.InstalledTemplate != nil {
+		if installed, err := getBoolRefFromStringTemplate(*r.InstalledTemplate); err != nil {
+			return fmt.Errorf("installedTemplate: %v", err)
+		} else {
+			r.InstalledTemplate = nil
+			r.Installed = installed
+		}
+	}
+
+	if r.WaitTemplate != nil {
+		if wait, err := getBoolRefFromStringTemplate(*r.WaitTemplate); err != nil {
+			return fmt.Errorf("waitTemplate: %v", err)
+		} else {
+			r.WaitTemplate = nil
+			r.Wait = wait
+		}
+	}
+
+	if r.TillerlessTemplate != nil {
+		if tillerless, err := getBoolRefFromStringTemplate(*r.TillerlessTemplate); err != nil {
+			return fmt.Errorf("tillerlessTemplate: %v", err)
+		} else {
+			r.TillerlessTemplate = nil
+			r.Tillerless = tillerless
+		}
+	}
+
+	if r.VerifyTemplate != nil {
+		if verify, err := getBoolRefFromStringTemplate(*r.VerifyTemplate); err != nil {
+			return fmt.Errorf("verifyTemplate: %v", err)
+		} else {
+			r.VerifyTemplate = nil
+			r.Verify = verify
+		}
+	}
+
+	return nil
+}
+
+func (st *HelmState) ExecuteTemplates() (*HelmState, error) {
+	r := *st
+
+	vals, err := st.Values()
+	if err != nil {
+		return nil, err
+	}
+
+	for i, rt := range st.Releases {
+		successFlag := false
+		for it, prev := 0, &rt; it < 6; it++ {
+			tmplData := releaseTemplateData{
+				Environment: st.Env,
+				Release:     *prev,
+				Values:      vals,
+			}
+			renderer := tmpl.NewFileRenderer(st.readFile, st.basePath, tmplData)
+			r, err := rt.ExecuteTemplateExpressions(renderer)
+			if err != nil {
+				return nil, fmt.Errorf("failed executing templates in release \"%s\".\"%s\": %v", st.FilePath, rt.Name, err)
+			}
+			if reflect.DeepEqual(prev, r) {
+				successFlag = true
+				if err := updateBoolTemplatedValues(r); err != nil {
+					return nil, fmt.Errorf("failed executing templates in release \"%s\".\"%s\": %v", st.FilePath, rt.Name, err)
+				}
+				st.Releases[i] = *r
+				break
+			}
+			prev = r
+		}
+		if !successFlag {
+			return nil, fmt.Errorf("failed executing templates in release \"%s\".\"%s\": %s", st.FilePath, rt.Name,
+				"recursive references can't be resolved")
+		}
+	}
+
+	return &r, nil
+}

pkg/state/state_exec_tmpl_test.go

@@ -0,0 +1,256 @@
+package state
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/go-test/deep"
+	"github.com/roboll/helmfile/pkg/environment"
+)
+
+func boolPtrToString(ptr *bool) string {
+	if ptr == nil {
+		return "<nil>"
+	}
+	return fmt.Sprintf("&%t", *ptr)
+}
+
+func ptr(v interface{}) interface{} {
+	r := v
+	return reflect.ValueOf(r).Addr().Interface()
+}
+
+func TestHelmState_executeTemplates(t *testing.T) {
+	tests := []struct {
+		name  string
+		input ReleaseSpec
+		want  ReleaseSpec
+	}{
+		{
+			name: "Has template expressions in chart, values, secrets, version, labels",
+			input: ReleaseSpec{
+				Chart:          "test-charts/{{ .Release.Name }}",
+				Version:        "{{ .Release.Name }}-0.1",
+				Name:           "test-app",
+				Namespace:      "test-namespace-{{ .Release.Name }}",
+				ValuesTemplate: []interface{}{"config/{{ .Environment.Name }}/{{ .Release.Name }}/values.yaml"},
+				Secrets:        []string{"config/{{ .Environment.Name }}/{{ .Release.Name }}/secrets.yaml"},
+				Labels:         map[string]string{"id": "{{ .Release.Name }}"},
+			},
+			want: ReleaseSpec{
+				Chart:     "test-charts/test-app",
+				Version:   "test-app-0.1",
+				Name:      "test-app",
+				Namespace: "test-namespace-test-app",
+				Values:    []interface{}{"config/test_env/test-app/values.yaml"},
+				Secrets:   []string{"config/test_env/test-app/secrets.yaml"},
+				Labels:    map[string]string{"id": "test-app"},
+			},
+		},
+		{
+			name: "Has template expressions in name with recursive refs",
+			input: ReleaseSpec{
+				Chart:     "test-chart",
+				Name:      "{{ .Release.Labels.id }}-{{ .Release.Namespace }}",
+				Namespace: "dev",
+				Labels:    map[string]string{"id": "{{ .Release.Chart }}"},
+			},
+			want: ReleaseSpec{
+				Chart:     "test-chart",
+				Name:      "test-chart-dev",
+				Namespace: "dev",
+				Labels:    map[string]string{"id": "test-chart"},
+			},
+		},
+		{
+			name: "Has template expressions in boolean values",
+			input: ReleaseSpec{
+				Chart:              "test-chart",
+				Name:               "app-dev",
+				Namespace:          "dev",
+				Labels:             map[string]string{"id": "app"},
+				InstalledTemplate:  func(i string) *string { return &i }(`{{ eq .Release.Labels.id "app" | ternary "yes" "no" }}`),
+				VerifyTemplate:     func(i string) *string { return &i }(`{{ true }}`),
+				Verify:             func(i bool) *bool { return &i }(false),
+				WaitTemplate:       func(i string) *string { return &i }(`{{ false }}`),
+				TillerlessTemplate: func(i string) *string { return &i }(`yes`),
+			},
+			want: ReleaseSpec{
+				Chart:      "test-chart",
+				Name:       "app-dev",
+				Namespace:  "dev",
+				Labels:     map[string]string{"id": "app"},
+				Installed:  func(i bool) *bool { return &i }(true),
+				Verify:     func(i bool) *bool { return &i }(true),
+				Wait:       func(i bool) *bool { return &i }(false),
+				Tillerless: func(i bool) *bool { return &i }(true),
+			},
+		},
+		{
+			name: "Has template in set-values",
+			input: ReleaseSpec{
+				Chart:     "test-charts/chart",
+				Name:      "test-app",
+				Namespace: "dev",
+				SetValuesTemplate: []SetValue{
+					SetValue{Name: "val1", Value: "{{ .Release.Name }}-val1"},
+					SetValue{Name: "val2", File: "{{ .Release.Name }}.yml"},
+					SetValue{Name: "val3", Values: []string{"{{ .Release.Name }}-val2", "{{ .Release.Name }}-val3"}},
+				},
+			},
+			want: ReleaseSpec{
+				Chart:     "test-charts/chart",
+				Name:      "test-app",
+				Namespace: "dev",
+				SetValues: []SetValue{
+					SetValue{Name: "val1", Value: "test-app-val1"},
+					SetValue{Name: "val2", File: "test-app.yml"},
+					SetValue{Name: "val3", Values: []string{"test-app-val2", "test-app-val3"}},
+				},
+			},
+		},
+		{
+			name: "Has template in values (map)",
+			input: ReleaseSpec{
+				Chart:          "test-charts/chart",
+				Verify:         nil,
+				Name:           "app",
+				Namespace:      "dev",
+				ValuesTemplate: []interface{}{map[string]string{"key": "{{ .Release.Name }}-val0"}},
+			},
+			want: ReleaseSpec{
+				Chart:     "test-charts/chart",
+				Verify:    nil,
+				Name:      "app",
+				Namespace: "dev",
+				Values:    []interface{}{map[interface{}]interface{}{"key": "app-val0"}},
+			},
+		},
+	}
+
+	for i := range tests {
+		tt := tests[i]
+		t.Run(tt.name, func(t *testing.T) {
+			state := &HelmState{
+				basePath: ".",
+				HelmDefaults: HelmSpec{
+					KubeContext: "test_context",
+				},
+				Env:               environment.Environment{Name: "test_env"},
+				OverrideNamespace: "test-namespace_",
+				Repositories:      nil,
+				Releases: []ReleaseSpec{
+					tt.input,
+				},
+			}
+
+			r, err := state.ExecuteTemplates()
+			if err != nil {
+				t.Errorf("Unexpected error: %v", err)
+				t.FailNow()
+			}
+
+			actual := r.Releases[0]
+
+			if !reflect.DeepEqual(actual.Name, tt.want.Name) {
+				t.Errorf("expected Name %+v, got %+v", tt.want.Name, actual.Name)
+			}
+			if !reflect.DeepEqual(actual.Chart, tt.want.Chart) {
+				t.Errorf("expected Chart %+v, got %+v", tt.want.Chart, actual.Chart)
+			}
+			if !reflect.DeepEqual(actual.Namespace, tt.want.Namespace) {
+				t.Errorf("expected Namespace %+v, got %+v", tt.want.Namespace, actual.Namespace)
+			}
+			if diff := deep.Equal(actual.Values, tt.want.Values); diff != nil && len(actual.Values) > 0 {
+				t.Errorf("Values differs \n%+v", strings.Join(diff, "\n"))
+			}
+			if diff := deep.Equal(actual.Secrets, tt.want.Secrets); diff != nil && len(actual.Secrets) > 0 {
+				t.Errorf("Secrets differs \n%+v", strings.Join(diff, "\n"))
+			}
+			if diff := deep.Equal(actual.SetValues, tt.want.SetValues); diff != nil && len(actual.SetValues) > 0 {
+				t.Errorf("SetValues differs \n%+v", strings.Join(diff, "\n"))
+			}
+			if diff := deep.Equal(actual.Labels, tt.want.Labels); diff != nil && len(actual.Labels) > 0 {
+				t.Errorf("Labels differs \n%+v", strings.Join(diff, "\n"))
+			}
+			if !reflect.DeepEqual(actual.Version, tt.want.Version) {
+				t.Errorf("expected Version %+v, got %+v", tt.want.Version, actual.Version)
+			}
+			if !reflect.DeepEqual(actual.Installed, tt.want.Installed) {
+				t.Errorf("expected actual.Installed %+v, got %+v",
+					boolPtrToString(tt.want.Installed), boolPtrToString(actual.Installed),
+				)
+			}
+			if !reflect.DeepEqual(actual.Tillerless, tt.want.Tillerless) {
+				t.Errorf("expected actual.Tillerless %+v, got %+v",
+					boolPtrToString(tt.want.Tillerless), boolPtrToString(actual.Tillerless),
+				)
+			}
+			if !reflect.DeepEqual(actual.Verify, tt.want.Verify) {
+				t.Errorf("expected actual.Verify %+v, got %+v",
+					boolPtrToString(tt.want.Verify), boolPtrToString(actual.Verify),
+				)
+			}
+			if !reflect.DeepEqual(actual.Wait, tt.want.Wait) {
+				t.Errorf("expected actual.Wait %+v, got %+v",
+					boolPtrToString(tt.want.Wait), boolPtrToString(actual.Wait),
+				)
+			}
+		})
+	}
+}
+
+func TestHelmState_recursiveRefsTemplates(t *testing.T) {
+
+	tests := []struct {
+		name  string
+		input ReleaseSpec
+	}{
+		{
+			name: "Has reqursive references",
+			input: ReleaseSpec{
+				Chart:     "test-charts/{{ .Release.Name }}",
+				Verify:    nil,
+				Name:      "{{ .Release.Labels.id }}",
+				Namespace: "dev",
+				Labels:    map[string]string{"id": "app-{{ .Release.Name }}"},
+			},
+		},
+		{
+			name: "Has unresolvable boolean templates",
+			input: ReleaseSpec{
+				Name:         "app-dev",
+				Chart:        "test-charts/app",
+				Verify:       nil,
+				Namespace:    "dev",
+				WaitTemplate: func(i string) *string { return &i }("hi"),
+			},
+		},
+	}
+
+	for i := range tests {
+		tt := tests[i]
+		t.Run(tt.name, func(t *testing.T) {
+			state := &HelmState{
+				basePath: ".",
+				HelmDefaults: HelmSpec{
+					KubeContext: "test_context",
+				},
+				Env:               environment.Environment{Name: "test_env"},
+				OverrideNamespace: "test-namespace_",
+				Repositories:      nil,
+				Releases: []ReleaseSpec{
+					tt.input,
+				},
+			}
+
+			r, err := state.ExecuteTemplates()
+			if err == nil {
+				t.Errorf("Expected error, got valid response: %v", r)
+				t.FailNow()
+			}
+		})
+	}
+}

pkg/state/state_run.go

@@ -0,0 +1,199 @@
+package state
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/variantdev/dag/pkg/dag"
+)
+
+type result struct {
+	release ReleaseSpec
+	err     error
+}
+
+func (st *HelmState) scatterGather(concurrency int, items int, produceInputs func(), receiveInputsAndProduceIntermediates func(int), aggregateIntermediates func()) {
+
+	if concurrency < 1 || concurrency > items {
+		concurrency = items
+	}
+
+	for _, r := range st.Releases {
+		if r.Tillerless != nil {
+			if *r.Tillerless {
+				concurrency = 1
+			}
+		} else if st.HelmDefaults.Tillerless {
+			concurrency = 1
+		}
+	}
+
+	// WaitGroup is required to wait until goroutine per job in job queue cleanly stops.
+	var waitGroup sync.WaitGroup
+	waitGroup.Add(concurrency)
+
+	go produceInputs()
+
+	for w := 1; w <= concurrency; w++ {
+		go func(id int) {
+			st.logger.Debugf("worker %d/%d started", id, concurrency)
+			receiveInputsAndProduceIntermediates(id)
+			st.logger.Debugf("worker %d/%d finished", id, concurrency)
+			waitGroup.Done()
+		}(w)
+	}
+
+	aggregateIntermediates()
+
+	// Wait until all the goroutines to gracefully finish
+	waitGroup.Wait()
+}
+
+func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency int,
+	do func(ReleaseSpec, int) error) []error {
+
+	return st.iterateOnReleases(helm, concurrency, st.Releases, do)
+}
+
+func (st *HelmState) iterateOnReleases(helm helmexec.Interface, concurrency int, inputs []ReleaseSpec,
+	do func(ReleaseSpec, int) error) []error {
+	var errs []error
+
+	inputsSize := len(inputs)
+
+	releases := make(chan ReleaseSpec)
+	results := make(chan result)
+
+	st.scatterGather(
+		concurrency,
+		inputsSize,
+		func() {
+			for _, release := range inputs {
+				releases <- release
+			}
+			close(releases)
+		},
+		func(id int) {
+			for release := range releases {
+				err := do(release, id)
+				st.logger.Debugf("sending result for release: %s\n", release.Name)
+				results <- result{release: release, err: err}
+				st.logger.Debugf("sent result for release: %s\n", release.Name)
+			}
+		},
+		func() {
+			for i := range inputs {
+				st.logger.Debugf("receiving result %d", i)
+				r := <-results
+				if r.err != nil {
+					errs = append(errs, fmt.Errorf("release \"%s\" failed: %v", r.release.Name, r.err))
+				} else {
+					st.logger.Debugf("received result for release \"%s\"", r.release.Name)
+				}
+				st.logger.Debugf("received result for %d", i)
+			}
+		},
+	)
+
+	if len(errs) != 0 {
+		return errs
+	}
+
+	return nil
+}
+
+type PlanOpts struct {
+	Reverse bool
+}
+
+func (st *HelmState) PlanReleases(reverse bool) ([][]Release, error) {
+	marked, err := st.SelectReleasesWithOverrides()
+	if err != nil {
+		return nil, err
+	}
+
+	groups, err := SortedReleaseGroups(marked, reverse)
+	if err != nil {
+		return nil, err
+	}
+
+	return groups, nil
+}
+
+func SortedReleaseGroups(releases []Release, reverse bool) ([][]Release, error) {
+	groups, err := GroupReleasesByDependency(releases)
+	if err != nil {
+		return nil, err
+	}
+
+	if reverse {
+		sort.Slice(groups, func(i, j int) bool {
+			return j < i
+		})
+	}
+
+	return groups, nil
+}
+
+func GroupReleasesByDependency(releases []Release) ([][]Release, error) {
+	idToReleases := map[string][]Release{}
+
+	d := dag.New()
+	for _, r := range releases {
+
+		id := ReleaseToID(&r.ReleaseSpec)
+
+		idToReleases[id] = append(idToReleases[id], r)
+
+		// Only compute dependencies from non-filtered releases
+		if !r.Filtered {
+			d.Add(id, dag.Dependencies(r.Needs))
+		}
+	}
+
+	for _, r := range releases {
+		if !r.Filtered {
+			for _, n := range r.Needs {
+				if _, ok := idToReleases[n]; !ok {
+					id := ReleaseToID(&r.ReleaseSpec)
+					return nil, fmt.Errorf("%q has dependency to inexistent release %q", id, n)
+				}
+			}
+		}
+	}
+
+	plan, err := d.Plan()
+	if err != nil {
+		return nil, err
+	}
+
+	var result [][]Release
+
+	for groupIndex := 0; groupIndex < len(plan); groupIndex++ {
+		dagNodesInGroup := plan[groupIndex]
+
+		var idsInGroup []string
+		var releasesInGroup []Release
+
+		for _, node := range dagNodesInGroup {
+			idsInGroup = append(idsInGroup, node.Id)
+		}
+
+		// Make the helmfile behavior deterministic for reproducibility and ease of testing
+		sort.Strings(idsInGroup)
+
+		for _, id := range idsInGroup {
+			releases, ok := idToReleases[id]
+			if !ok {
+				panic(fmt.Errorf("bug: unexpectedly failed to get releases for id %q", id))
+			}
+			releasesInGroup = append(releasesInGroup, releases...)
+		}
+
+		result = append(result, releasesInGroup)
+	}
+
+	return result, nil
+}

pkg/state/storage.go

@@ -0,0 +1,99 @@
+package state
+
+import (
+	"fmt"
+	"go.uber.org/zap"
+	"net/url"
+	"path/filepath"
+	"sort"
+)
+
+type Storage struct {
+	logger *zap.SugaredLogger
+
+	FilePath string
+
+	basePath string
+	glob     func(string) ([]string, error)
+}
+
+func NewStorage(forFile string, logger *zap.SugaredLogger, glob func(string) ([]string, error)) *Storage {
+	return &Storage{
+		FilePath: forFile,
+		basePath: filepath.Dir(forFile),
+		logger:   logger,
+		glob:     glob,
+	}
+}
+
+func (st *Storage) resolveFile(missingFileHandler *string, tpe, path string) ([]string, bool, error) {
+	title := fmt.Sprintf("%s file", tpe)
+
+	files, err := st.ExpandPaths(path)
+	if err != nil {
+		return nil, false, err
+	}
+
+	var handlerId string
+
+	if missingFileHandler != nil {
+		handlerId = *missingFileHandler
+	} else {
+		handlerId = MissingFileHandlerError
+	}
+
+	if len(files) == 0 {
+		switch handlerId {
+		case MissingFileHandlerError:
+			return nil, false, fmt.Errorf("%s matching \"%s\" does not exist in \"%s\"", title, path, st.basePath)
+		case MissingFileHandlerWarn:
+			st.logger.Warnf("skipping missing %s matching \"%s\"", title, path)
+			return nil, true, nil
+		case MissingFileHandlerInfo:
+			st.logger.Infof("skipping missing %s matching \"%s\"", title, path)
+			return nil, true, nil
+		case MissingFileHandlerDebug:
+			st.logger.Debugf("skipping missing %s matching \"%s\"", title, path)
+			return nil, true, nil
+		default:
+			available := []string{
+				MissingFileHandlerError,
+				MissingFileHandlerWarn,
+				MissingFileHandlerInfo,
+				MissingFileHandlerDebug,
+			}
+			return nil, false, fmt.Errorf("invalid missing file handler \"%s\" while processing \"%s\" in \"%s\": it must be one of %s", handlerId, path, st.FilePath, available)
+		}
+	}
+
+	return files, false, nil
+}
+
+func (st *Storage) ExpandPaths(globPattern string) ([]string, error) {
+	result := []string{}
+	absPathPattern := st.normalizePath(globPattern)
+	matches, err := st.glob(absPathPattern)
+	if err != nil {
+		return nil, fmt.Errorf("failed processing %s: %v", globPattern, err)
+	}
+
+	sort.Strings(matches)
+
+	result = append(result, matches...)
+	return result, nil
+}
+
+// normalizes relative path to absolute one
+func (st *Storage) normalizePath(path string) string {
+	u, _ := url.Parse(path)
+	if u.Scheme != "" || filepath.IsAbs(path) {
+		return path
+	} else {
+		return st.JoinBase(path)
+	}
+}
+
+// JoinBase returns an absolute path in the form basePath/relative
+func (st *Storage) JoinBase(relPath string) string {
+	return filepath.Join(st.basePath, relPath)
+}

pkg/state/types.go

@@ -1,6 +1,8 @@
 package state
 
-import "github.com/roboll/helmfile/environment"
+import (
+	"github.com/roboll/helmfile/pkg/environment"
+)
 
 // TemplateSpec defines the structure of a reusable and composable template for helm releases.
 type TemplateSpec struct {
@@ -13,12 +15,16 @@ type EnvironmentTemplateData struct {
 	Environment environment.Environment
 	// Namespace is accessible as `.Namespace` from any non-values template executed by the renderer
 	Namespace string
+	// Values is accessible as `.Values` and it contains default state values overrode by environment values and override values.
+	Values map[string]interface{}
 }
 
-// ReleaseTemplateData provides variables accessible while executing golang text/template expressions in releases of a helmfile YAML file
-type ReleaseTemplateData struct {
+// releaseTemplateData provides variables accessible while executing golang text/template expressions in releases of a helmfile YAML file
+type releaseTemplateData struct {
 	// Environment is accessible as `.Environment` from any template expression executed by the renderer
 	Environment environment.Environment
 	// Release is accessible as `.Release` from any template expression executed by the renderer
 	Release ReleaseSpec
+	// Values is accessible as `.Values` and it contains default state values overrode by environment values and override values.
+	Values map[string]interface{}
 }

pkg/state/util.go

@@ -0,0 +1,52 @@
+package state
+
+import (
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+func isLocalChart(chart string) bool {
+	regex, _ := regexp.Compile("^[.]?./")
+	matched := regex.MatchString(chart)
+	if matched {
+		return true
+	}
+
+	uriLike := strings.Index(chart, "://") > -1
+	if uriLike {
+		return false
+	}
+
+	return chart == "" ||
+		chart[0] == '/' ||
+		strings.Index(chart, "/") == -1 ||
+		len(strings.Split(chart, "/")) != 2
+}
+
+func resolveRemoteChart(repoAndChart string) (string, string, bool) {
+	if isLocalChart(repoAndChart) {
+		return "", "", false
+	}
+
+	parts := strings.Split(repoAndChart, "/")
+	if len(parts) != 2 {
+		return "", "", false
+	}
+
+	repo := parts[0]
+	chart := parts[1]
+
+	return repo, chart, true
+}
+
+// normalizeChart allows for the distinction between a file path reference and repository references.
+// - Any single (or double character) followed by a `/` will be considered a local file reference and
+// 	 be constructed relative to the `base path`.
+// - Everything else is assumed to be an absolute path or an actual <repository>/<chart> reference.
+func normalizeChart(basePath, chart string) string {
+	if !isLocalChart(chart) || chart[0] == '/' {
+		return chart
+	}
+	return filepath.Join(basePath, chart)
+}

pkg/state/util_test.go

@@ -0,0 +1,147 @@
+package state
+
+import "testing"
+
+func TestIsLocalChart(t *testing.T) {
+	testcases := []struct {
+		input    string
+		expected bool
+	}{
+		{
+			input:    "mychart",
+			expected: true,
+		},
+		{
+			input:    "stable/mysql",
+			expected: false,
+		},
+		{
+			input:    "./charts/myapp",
+			expected: true,
+		},
+		{
+			input:    "charts/mysubsystem/myapp",
+			expected: true,
+		},
+		{
+			input:    "./charts/mysubsystem/myapp",
+			expected: true,
+		},
+		{
+			input:    "/charts/mysubsystem/myapp",
+			expected: true,
+		},
+		{
+			// Regression test case for:
+			// * https://github.com/roboll/helmfile/issues/675
+			// * https://github.com/roboll/helmfile/issues/687
+			input:    "https://github.com/arangodb/kube-arangodb/releases/download/0.3.11/kube-arangodb-crd.tgz",
+			expected: false,
+		},
+		{
+			input:    "https://example.com/bar.tgz",
+			expected: false,
+		},
+	}
+
+	for i := range testcases {
+		testcase := testcases[i]
+
+		actual := isLocalChart(testcase.input)
+
+		if testcase.expected != actual {
+			t.Errorf("unexpected result: isLocalChart(\"%s\"): expected=%v, got=%v", testcase.input, testcase.expected, actual)
+		}
+	}
+}
+
+func TestResolveRemortChart(t *testing.T) {
+	testcases := []struct {
+		input  string
+		repo   string
+		chart  string
+		remote bool
+	}{
+		{
+			input:  "mychart",
+			remote: false,
+		},
+		{
+			input:  "stable/mysql",
+			repo:   "stable",
+			chart:  "mysql",
+			remote: true,
+		},
+		{
+			input:  "./charts/myapp",
+			remote: false,
+		},
+		{
+			input:  "charts/mysubsystem/myapp",
+			remote: false,
+		},
+		{
+			input:  "./charts/mysubsystem/myapp",
+			remote: false,
+		},
+		{
+			input:  "/charts/mysubsystem/myapp",
+			remote: false,
+		},
+		{
+			// Regression test case for:
+			// * https://github.com/roboll/helmfile/issues/675
+			// * https://github.com/roboll/helmfile/issues/687
+			input:  "https://github.com/arangodb/kube-arangodb/releases/download/0.3.11/kube-arangodb-crd.tgz",
+			remote: false,
+		},
+		{
+			input:  "https://example.com/bar.tgz",
+			remote: false,
+		},
+	}
+
+	for i := range testcases {
+		testcase := testcases[i]
+
+		repo, chart, actual := resolveRemoteChart(testcase.input)
+
+		if testcase.remote != actual {
+			t.Fatalf("unexpected result: reolveRemoteChart(\"%s\"): expected=%v, got=%v", testcase.input, testcase.remote, actual)
+		}
+
+		if testcase.repo != repo {
+			t.Errorf("unexpected repo: %s: expected=%v, got=%v", testcase.input, testcase.repo, repo)
+		}
+
+		if testcase.chart != chart {
+			t.Errorf("unexpected chart: %s: expected=%v, got=%v", testcase.input, testcase.chart, chart)
+		}
+	}
+}
+
+func TestNormalizeChart(t *testing.T) {
+	testcases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "mychart",
+			expected: "/path/to/mychart",
+		},
+		{
+			input:    "/charts/mychart",
+			expected: "/charts/mychart",
+		},
+	}
+
+	for i := range testcases {
+		testcase := testcases[i]
+
+		actual := normalizeChart("/path/to", testcase.input)
+
+		if testcase.expected != actual {
+			t.Fatalf("unexpected result: normalizeChart(\"/path/to\", \"%s\"): expected=%v, got=%v", testcase.input, testcase.expected, actual)
+		}
+	}
+}

pkg/testhelper/diff.go

@@ -0,0 +1,103 @@
+package testhelper
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math"
+	"strings"
+
+	"github.com/aryann/difflib"
+)
+
+func Diff(want, got string, context int) (string, bool) {
+	records := difflib.Diff(
+		strings.Split(want, "\n"),
+		strings.Split(got, "\n"),
+	)
+
+	w := &bytes.Buffer{}
+
+	changed := checkAndPrintRecords(w, records, context)
+
+	return w.String(), changed
+}
+
+func checkAndPrintRecords(w io.Writer, records []difflib.DiffRecord, context int) bool {
+	var changed bool
+	if context >= 0 {
+		distances := calculateDistances(records)
+		omitting := false
+		for i, diff := range records {
+			if diff.Delta != difflib.Common {
+				changed = true
+			}
+			if distances[i] > context {
+				if !omitting {
+					fmt.Fprintln(w, "...")
+					omitting = true
+				}
+			} else {
+				omitting = false
+				fmt.Fprintln(w, formatRecord(diff))
+			}
+		}
+	} else {
+		for _, diff := range records {
+			if diff.Delta != difflib.Common {
+				changed = true
+			}
+			fmt.Fprintln(w, formatRecord(diff))
+		}
+	}
+	return changed
+}
+
+func formatRecord(diff difflib.DiffRecord) string {
+	var prefix string
+	switch diff.Delta {
+	case difflib.RightOnly:
+		prefix = "+ "
+	case difflib.LeftOnly:
+		prefix = "- "
+	case difflib.Common:
+		prefix = "  "
+	}
+
+	return prefix + diff.Payload
+}
+
+// Shamelessly and thankfully copied from https://github.com/databus23/helm-diff/blob/99b8474af7726ca6f57b37b0b8b8f3cd36c991e8/diff/diff.go#L116
+func calculateDistances(diffs []difflib.DiffRecord) map[int]int {
+	distances := map[int]int{}
+
+	// Iterate forwards through diffs, set 'distance' based on closest 'change' before this line
+	change := -1
+	for i, diff := range diffs {
+		if diff.Delta != difflib.Common {
+			change = i
+		}
+		distance := math.MaxInt32
+		if change != -1 {
+			distance = i - change
+		}
+		distances[i] = distance
+	}
+
+	// Iterate backwards through diffs, reduce 'distance' based on closest 'change' after this line
+	change = -1
+	for i := len(diffs) - 1; i >= 0; i-- {
+		diff := diffs[i]
+		if diff.Delta != difflib.Common {
+			change = i
+		}
+		if change != -1 {
+			distance := change - i
+			if distance < distances[i] {
+				distances[i] = distance
+			}
+		}
+	}
+
+	return distances
+}

pkg/testhelper/testfs.go

@@ -0,0 +1,137 @@
+package testhelper
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type TestFs struct {
+	Cwd   string
+	dirs  map[string]bool
+	files map[string]string
+
+	GlobFixtures map[string][]string
+
+	fileReaderCalls int
+	successfulReads []string
+}
+
+func NewTestFs(files map[string]string) *TestFs {
+	dirs := map[string]bool{}
+	for abs, _ := range files {
+		for d := filepath.Dir(abs); !dirs[d]; d = filepath.Dir(d) {
+			dirs[d] = true
+			fmt.Fprintf(os.Stderr, "testfs: recognized dir: %s\n", d)
+		}
+	}
+	return &TestFs{
+		Cwd:   "/path/to",
+		dirs:  dirs,
+		files: files,
+
+		successfulReads: []string{},
+
+		GlobFixtures: map[string][]string{},
+	}
+}
+
+func (f *TestFs) FileExistsAt(path string) bool {
+	var ok bool
+	if strings.Contains(path, "/") {
+		_, ok = f.files[path]
+	} else {
+		_, ok = f.files[filepath.Join(f.Cwd, path)]
+	}
+	return ok
+}
+
+func (f *TestFs) FileExists(path string) (bool, error) {
+	return f.FileExistsAt(path), nil
+}
+
+func (f *TestFs) DirectoryExistsAt(path string) bool {
+	var ok bool
+	if strings.Contains(path, "/") {
+		_, ok = f.dirs[path]
+	} else {
+		_, ok = f.dirs[filepath.Join(f.Cwd, path)]
+	}
+	return ok
+}
+
+func (f *TestFs) ReadFile(filename string) ([]byte, error) {
+	var str string
+	var ok bool
+	if filename[0] == '/' {
+		str, ok = f.files[filename]
+	} else {
+		str, ok = f.files[filepath.Join(f.Cwd, filename)]
+	}
+	if !ok {
+		return []byte(nil), fmt.Errorf("no registered file found: %s", filename)
+	}
+
+	f.fileReaderCalls += 1
+
+	f.successfulReads = append(f.successfulReads, filename)
+
+	return []byte(str), nil
+}
+
+func (f *TestFs) SuccessfulReads() []string {
+	return f.successfulReads
+}
+
+func (f *TestFs) FileReaderCalls() int {
+	return f.fileReaderCalls
+}
+
+func (f *TestFs) Glob(relPattern string) ([]string, error) {
+	var pattern string
+	if relPattern[0] == '/' {
+		pattern = relPattern
+	} else {
+		pattern = filepath.Join(f.Cwd, relPattern)
+	}
+
+	fixtures, ok := f.GlobFixtures[pattern]
+	if ok {
+		return fixtures, nil
+	}
+
+	matches := []string{}
+	for name, _ := range f.files {
+		matched, err := filepath.Match(pattern, name)
+		if err != nil {
+			return nil, err
+		}
+		if matched {
+			matches = append(matches, name)
+		}
+	}
+	return matches, nil
+}
+
+func (f *TestFs) Abs(path string) (string, error) {
+	var p string
+	if path[0] == '/' {
+		p = path
+	} else {
+		p = filepath.Join(f.Cwd, path)
+	}
+	return filepath.Clean(p), nil
+}
+
+func (f *TestFs) Getwd() (string, error) {
+	return f.Cwd, nil
+}
+
+func (f *TestFs) Chdir(dir string) error {
+	if _, ok := f.dirs[dir]; ok {
+		f.Cwd = dir
+		return nil
+	}
+	return fmt.Errorf("unexpected chdir \"%s\"", dir)
+}

pkg/tmpl/context_funcs.go

@@ -149,7 +149,7 @@ func FromYaml(str string) (Values, error) {
 	m := Values{}
 
 	if err := yaml.Unmarshal([]byte(str), &m); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%s, offending yaml: %s", err, str)
 	}
 	return m, nil
 }

pkg/tmpl/context_tmpl.go

@@ -13,9 +13,9 @@ func (c *Context) stringTemplate() *template.Template {
 	}
 	tmpl := template.New("stringTemplate").Funcs(funcMap)
 	if c.preRender {
-		tmpl.Option("missingkey=zero")
+		tmpl = tmpl.Option("missingkey=zero")
 	} else {
-		tmpl.Option("missingkey=error")
+		tmpl = tmpl.Option("missingkey=error")
 	}
 	return tmpl
 }

pkg/tmpl/context_tmpl_test.go

@@ -230,7 +230,8 @@ func Test_renderTemplateToString(t *testing.T) {
 			wantErr: true,
 		},
 	}
-	for _, tt := range tests {
+	for i := range tests {
+		tt := tests[i]
 		t.Run(tt.name, func(t *testing.T) {
 			for k, v := range tt.args.envs {
 				err := os.Setenv(k, v)

pkg/tmpl/file_renderer.go

@@ -46,6 +46,8 @@ func (r *FileRenderer) RenderTemplateFileToBuffer(file string) (*bytes.Buffer, e
 	return r.RenderTemplateContentToBuffer(content)
 }
 
+// RenderToBytes loads the content of the file.
+// If its extension is `gotmpl` it treats the content as a go template and renders it.
 func (r *FileRenderer) RenderToBytes(path string) ([]byte, error) {
 	var yamlBytes []byte
 	splits := strings.Split(path, ".")

pkg/tmpl/file_renderer_test.go

@@ -2,7 +2,7 @@ package tmpl
 
 import (
 	"fmt"
-	"github.com/roboll/helmfile/environment"
+	"github.com/roboll/helmfile/pkg/environment"
 	"reflect"
 	"testing"
 )

state/create.go

@@ -1,180 +0,0 @@
-package state
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-
-	"github.com/imdario/mergo"
-	"github.com/roboll/helmfile/environment"
-	"github.com/roboll/helmfile/helmexec"
-	"github.com/roboll/helmfile/tmpl"
-	"go.uber.org/zap"
-	"gopkg.in/yaml.v2"
-)
-
-type StateLoadError struct {
-	msg   string
-	Cause error
-}
-
-func (e *StateLoadError) Error() string {
-	return fmt.Sprintf("%s: %v", e.msg, e.Cause)
-}
-
-type UndefinedEnvError struct {
-	msg string
-}
-
-func (e *UndefinedEnvError) Error() string {
-	return e.msg
-}
-
-type creator struct {
-	logger   *zap.SugaredLogger
-	readFile func(string) ([]byte, error)
-	abs      func(string) (string, error)
-
-	Strict bool
-}
-
-func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), abs func(string) (string, error)) *creator {
-	return &creator{
-		logger:   logger,
-		readFile: readFile,
-		abs:      abs,
-		Strict:   true,
-	}
-}
-
-// Parses YAML into HelmState, while loading environment values files relative to the `cwd`
-func (c *creator) ParseAndLoadEnv(content []byte, baseDir, file string, env string) (*HelmState, error) {
-	var state HelmState
-
-	state.FilePath = file
-	state.basePath = baseDir
-
-	decoder := yaml.NewDecoder(bytes.NewReader(content))
-	if !c.Strict {
-		decoder.SetStrict(false)
-	} else {
-		decoder.SetStrict(true)
-	}
-	i := 0
-	for {
-		i++
-
-		var intermediate HelmState
-
-		err := decoder.Decode(&intermediate)
-		if err == io.EOF {
-			break
-		} else if err != nil {
-			return nil, &StateLoadError{fmt.Sprintf("failed to read %s: reading document at index %d", file, i), err}
-		}
-
-		if err := mergo.Merge(&state, &intermediate, mergo.WithAppendSlice); err != nil {
-			return nil, &StateLoadError{fmt.Sprintf("failed to read %s: merging document at index %d", file, i), err}
-		}
-	}
-
-	if len(state.DeprecatedReleases) > 0 {
-		if len(state.Releases) > 0 {
-			return nil, fmt.Errorf("failed to parse %s: you can't specify both `charts` and `releases` sections", file)
-		}
-		state.Releases = state.DeprecatedReleases
-		state.DeprecatedReleases = []ReleaseSpec{}
-	}
-
-	if state.DeprecatedContext != "" && state.HelmDefaults.KubeContext == "" {
-		state.HelmDefaults.KubeContext = state.DeprecatedContext
-	}
-
-	state.logger = c.logger
-
-	e, err := state.loadEnv(env, c.readFile)
-	if err != nil {
-		return nil, &StateLoadError{fmt.Sprintf("failed to read %s", file), err}
-	}
-	state.Env = *e
-
-	state.readFile = c.readFile
-	state.removeFile = os.Remove
-	state.fileExists = func(path string) (bool, error) {
-		_, err := os.Stat(path)
-
-		if err != nil {
-			if os.IsNotExist(err) {
-				return false, nil
-			}
-			return false, err
-		}
-		return true, nil
-	}
-
-	return &state, nil
-}
-
-func (st *HelmState) loadEnv(name string, readFile func(string) ([]byte, error)) (*environment.Environment, error) {
-	envVals := map[string]interface{}{}
-	envSpec, ok := st.Environments[name]
-	if ok {
-		for _, envvalFile := range envSpec.Values {
-			envvalFullPath := filepath.Join(st.basePath, envvalFile)
-			tmplData := EnvironmentTemplateData{Environment: environment.EmptyEnvironment, Namespace: ""}
-			r := tmpl.NewFileRenderer(readFile, filepath.Dir(envvalFullPath), tmplData)
-			bytes, err := r.RenderToBytes(envvalFullPath)
-			if err != nil {
-				return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", envvalFile, err)
-			}
-			m := map[string]interface{}{}
-			if err := yaml.Unmarshal(bytes, &m); err != nil {
-				return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", envvalFile, err)
-			}
-			if err := mergo.Merge(&envVals, &m, mergo.WithOverride); err != nil {
-				return nil, fmt.Errorf("failed to load \"%s\": %v", envvalFile, err)
-			}
-		}
-
-		if len(envSpec.Secrets) > 0 {
-			helm := helmexec.New(st.logger, "")
-			for _, secFile := range envSpec.Secrets {
-				path := filepath.Join(st.basePath, secFile)
-				if _, err := os.Stat(path); os.IsNotExist(err) {
-					return nil, err
-				}
-				// Work-around to allow decrypting environment secrets
-				//
-				// We don't have releases loaded yet and therefore unable to decide whether
-				// helmfile should use helm-tiller to call helm-secrets or not.
-				//
-				// This means that, when you use environment secrets + tillerless setup, you still need a tiller
-				// installed on the cluster, just for decrypting secrets!
-				// Related: https://github.com/futuresimple/helm-secrets/issues/83
-				release := &ReleaseSpec{}
-				flags := st.appendTillerFlags([]string{}, release)
-				decFile, err := helm.DecryptSecret(st.createHelmContext(release, 0), path, flags...)
-				if err != nil {
-					return nil, err
-				}
-				bytes, err := readFile(decFile)
-				if err != nil {
-					return nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", secFile, err)
-				}
-				m := map[string]interface{}{}
-				if err := yaml.Unmarshal(bytes, &m); err != nil {
-					return nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", secFile, err)
-				}
-				if err := mergo.Merge(&envVals, &m, mergo.WithOverride); err != nil {
-					return nil, fmt.Errorf("failed to load \"%s\": %v", secFile, err)
-				}
-			}
-		}
-	} else if name != DefaultEnv {
-		return nil, &UndefinedEnvError{msg: fmt.Sprintf("environment \"%s\" is not defined", name)}
-	}
-
-	return &environment.Environment{Name: name, Values: envVals}, nil
-}

state/environment.go

@@ -1,6 +0,0 @@
-package state
-
-type EnvironmentSpec struct {
-	Values  []string `yaml:"values"`
-	Secrets []string `yaml:"secrets"`
-}

state/release.go

@@ -1,73 +0,0 @@
-package state
-
-import (
-	"fmt"
-	"github.com/roboll/helmfile/tmpl"
-	"gopkg.in/yaml.v2"
-)
-
-func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*ReleaseSpec, error) {
-	var result *ReleaseSpec
-	var err error
-
-	result, err = r.Clone()
-	if err != nil {
-		return nil, fmt.Errorf("failed executing template expressions in release \"%s\": %v", r.Name, err)
-	}
-
-	{
-		ts := result.Chart
-		result.Chart, err = renderer.RenderTemplateContentToString([]byte(ts))
-		if err != nil {
-			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".chart = \"%s\": %v", r.Name, ts, err)
-		}
-	}
-
-	{
-		ts := result.Namespace
-		result.Namespace, err = renderer.RenderTemplateContentToString([]byte(ts))
-		if err != nil {
-			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".namespace = \"%s\": %v", r.Name, ts, err)
-		}
-	}
-
-	for i, t := range result.Values {
-		switch ts := t.(type) {
-		case string:
-			s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
-			if err != nil {
-				return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%s\": %v", r.Name, i, ts, err)
-			}
-			result.Values[i] = s.String()
-		}
-	}
-
-	for i, ts := range result.Secrets {
-		s, err := renderer.RenderTemplateContentToBuffer([]byte(ts))
-		if err != nil {
-			return nil, fmt.Errorf("failed executing template expressions in release \"%s\".secrets[%d] = \"%s\": %v", r.Name, i, ts, err)
-		}
-		result.Secrets[i] = s.String()
-	}
-
-	return result, nil
-}
-
-func (r ReleaseSpec) Clone() (*ReleaseSpec, error) {
-	serialized, err := yaml.Marshal(r)
-	if err != nil {
-		return nil, fmt.Errorf("failed cloning release \"%s\": %v", r.Name, err)
-	}
-
-	var deserialized ReleaseSpec
-
-	if err := yaml.Unmarshal(serialized, &deserialized); err != nil {
-		return nil, fmt.Errorf("failed cloning release \"%s\": %v", r.Name, err)
-	}
-
-	return &deserialized, nil
-}
-
-func (r ReleaseSpec) Desired() bool {
-	return r.Installed == nil || *r.Installed
-}

state/state_exec_tmpl.go

@@ -1,32 +0,0 @@
-package state
-
-import (
-	"fmt"
-	"github.com/roboll/helmfile/tmpl"
-)
-
-func (st *HelmState) envTemplateData() EnvironmentTemplateData {
-	return EnvironmentTemplateData{
-		st.Env,
-		st.Namespace,
-	}
-}
-
-func (st *HelmState) ExecuteTemplates() (*HelmState, error) {
-	r := *st
-
-	for i, rt := range st.Releases {
-		tmplData := ReleaseTemplateData{
-			Environment: st.Env,
-			Release:     rt,
-		}
-		renderer := tmpl.NewFileRenderer(st.readFile, st.basePath, tmplData)
-		r, err := rt.ExecuteTemplateExpressions(renderer)
-		if err != nil {
-			return nil, fmt.Errorf("failed executing templates in release \"%s\".\"%s\": %v", st.FilePath, rt.Name, err)
-		}
-		st.Releases[i] = *r
-	}
-
-	return &r, nil
-}

state/state_exec_tmpl_test.go

@@ -1,76 +0,0 @@
-package state
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/roboll/helmfile/environment"
-)
-
-func TestHelmState_executeTemplates(t *testing.T) {
-	tests := []struct {
-		name  string
-		input ReleaseSpec
-		want  ReleaseSpec
-	}{
-		{
-			name: "Has template expressions in chart, values, and secrets",
-			input: ReleaseSpec{
-				Chart:     "test-charts/{{ .Release.Name }}",
-				Version:   "0.1",
-				Verify:    nil,
-				Name:      "test-app",
-				Namespace: "test-namespace-{{ .Release.Name }}",
-				Values:    []interface{}{"config/{{ .Environment.Name }}/{{ .Release.Name }}/values.yaml"},
-				Secrets:   []string{"config/{{ .Environment.Name }}/{{ .Release.Name }}/secrets.yaml"},
-			},
-			want: ReleaseSpec{
-				Chart:     "test-charts/test-app",
-				Version:   "0.1",
-				Verify:    nil,
-				Name:      "test-app",
-				Namespace: "test-namespace-test-app",
-				Values:    []interface{}{"config/test_env/test-app/values.yaml"},
-				Secrets:   []string{"config/test_env/test-app/secrets.yaml"},
-			},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			state := &HelmState{
-				basePath: ".",
-				HelmDefaults: HelmSpec{
-					KubeContext: "test_context",
-				},
-				Env:          environment.Environment{Name: "test_env"},
-				Namespace:    "test-namespace_",
-				Repositories: nil,
-				Releases: []ReleaseSpec{
-					tt.input,
-				},
-			}
-
-			r, err := state.ExecuteTemplates()
-			if err != nil {
-				t.Errorf("Unexpected error: %v", err)
-				t.FailNow()
-			}
-
-			actual := r.Releases[0]
-
-			if !reflect.DeepEqual(actual.Chart, tt.want.Chart) {
-				t.Errorf("expected %+v, got %+v", tt.want.Chart, actual.Chart)
-			}
-			if !reflect.DeepEqual(actual.Namespace, tt.want.Namespace) {
-				t.Errorf("expected %+v, got %+v", tt.want.Namespace, actual.Namespace)
-			}
-			if !reflect.DeepEqual(actual.Values, tt.want.Values) {
-				t.Errorf("expected %+v, got %+v", tt.want.Values, actual.Values)
-			}
-			if !reflect.DeepEqual(actual.Secrets, tt.want.Secrets) {
-				t.Errorf("expected %+v, got %+v", tt.want.Secrets, actual.Secrets)
-			}
-		})
-	}
-}

state/state_run.go

@@ -1,89 +0,0 @@
-package state
-
-import (
-	"fmt"
-	"github.com/roboll/helmfile/helmexec"
-	"sync"
-)
-
-type result struct {
-	release ReleaseSpec
-	err     error
-}
-
-func (st *HelmState) scatterGather(concurrency int, items int, produceInputs func(), receiveInputsAndProduceIntermediates func(int), aggregateIntermediates func()) {
-	numReleases := len(st.Releases)
-	if concurrency < 1 {
-		concurrency = numReleases
-	} else if concurrency > numReleases {
-		concurrency = numReleases
-	}
-
-	// WaitGroup is required to wait until goroutine per job in job queue cleanly stops.
-	var waitGroup sync.WaitGroup
-	waitGroup.Add(concurrency)
-
-	go produceInputs()
-
-	for w := 1; w <= concurrency; w++ {
-		go func(id int) {
-			st.logger.Debugf("worker %d/%d started", id, concurrency)
-			receiveInputsAndProduceIntermediates(id)
-			st.logger.Debugf("worker %d/%d finished", id, concurrency)
-			waitGroup.Done()
-		}(w)
-	}
-
-	aggregateIntermediates()
-
-	// Wait until all the goroutines to gracefully finish
-	waitGroup.Wait()
-}
-
-func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency int,
-	do func(ReleaseSpec, int) error) []error {
-	var errs []error
-
-	inputs := st.Releases
-	inputsSize := len(inputs)
-
-	releases := make(chan ReleaseSpec)
-	results := make(chan result)
-
-	st.scatterGather(
-		concurrency,
-		inputsSize,
-		func() {
-			for _, release := range inputs {
-				releases <- release
-			}
-			close(releases)
-		},
-		func(id int) {
-			for release := range releases {
-				err := do(release, id)
-				st.logger.Debugf("sending result for release: %s\n", release.Name)
-				results <- result{release: release, err: err}
-				st.logger.Debugf("sent result for release: %s\n", release.Name)
-			}
-		},
-		func() {
-			for i := range inputs {
-				st.logger.Debugf("receiving result %d", i)
-				r := <-results
-				if r.err != nil {
-					errs = append(errs, fmt.Errorf("release \"%s\" failed: %v", r.release.Name, r.err))
-				} else {
-					st.logger.Debugf("received result for release \"%s\"", r.release.Name)
-				}
-				st.logger.Debugf("received result for %d", i)
-			}
-		},
-	)
-
-	if len(errs) != 0 {
-		return errs
-	}
-
-	return nil
-}

test/integration/environment.values.yaml

@@ -0,0 +1 @@
+mysecret: MYSECRET

test/integration/happypath.yaml

@@ -1,6 +1,13 @@
+environments:
+  default:
+    values:
+      - environment.values.yaml
+---
 repositories:
   - name: stable
     url: https://kubernetes-charts.storage.googleapis.com/
+  - name: incubator
+    url: https://kubernetes-charts-incubator.storage.googleapis.com
 
 helmDefaults:
   kubeContext: minikube
@@ -12,3 +19,9 @@ releases:
     set:
       - name: ingress.enabled
         value: false
+
+  - name: raw
+    chart: incubator/raw
+    values:
+      - mysecret: {{ .Environment.Values.mysecret }}
+      - values.yaml

test/integration/run.sh

@@ -45,9 +45,15 @@ function retry() {
 
 set -e
 info "Using namespace: ${test_ns}"
-info "Using Helm version: $(helm version --short --client | grep -o v.*$)"
-${helm} init --wait --override spec.template.spec.automountServiceAccountToken=true
-${helm} plugin install https://github.com/databus23/helm-diff --version master
+# helm v2
+if helm version --client 2>/dev/null | grep '"v2\.'; then
+  info "Using Helm version: $(helm version --short --client | grep -o v.*$)"
+  ${helm} init --wait --override spec.template.spec.automountServiceAccountToken=true
+# helm v3
+else
+  info "Using Helm version: $(helm version --short | grep -o v.*$)"
+fi
+${helm} plugin install https://github.com/databus23/helm-diff --version v3.0.0-rc.7
 ${kubectl} get namespace ${test_ns} &> /dev/null && warn "Namespace ${test_ns} exists, from a previous test run?"
 $kubectl create namespace ${test_ns} || fail "Could not create namespace ${test_ns}"
 trap "{ $kubectl delete namespace ${test_ns}; }" EXIT # remove namespace whenever we exit this script
@@ -60,6 +66,12 @@ test_start "happypath - simple rollout of httpbin chart"
 info "Diffing ${dir}/happypath.yaml"
 bash -c "${helmfile} -f ${dir}/happypath.yaml diff --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
 
+info "Diffing ${dir}/happypath.yaml without color"
+bash -c "${helmfile} -f ${dir}/happypath.yaml --no-color diff --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
+
+info "Diffing ${dir}/happypath.yaml with limited context"
+bash -c "${helmfile} -f ${dir}/happypath.yaml diff --context 3 --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
+
 info "Templating ${dir}/happypath.yaml"
 ${helmfile} -f ${dir}/happypath.yaml template
 code=$?
@@ -76,6 +88,17 @@ ${helmfile} -f ${dir}/happypath.yaml apply
 code=$?
 [ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile apply: ${code}"
 
+info "Locking dependencies"
+${helmfile} -f ${dir}/happypath.yaml deps
+code=$?
+[ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile deps: ${code}"
+
+info "Applying ${dir}/happypath.yaml with locked dependencies"
+${helmfile} -f ${dir}/happypath.yaml apply
+code=$?
+[ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile apply: ${code}"
+${helm} list --namespace=${test_ns} || fail "unable to list releases"
+
 info "Deleting release"
 ${helmfile} -f ${dir}/happypath.yaml delete
 ${helm} status --namespace=${test_ns} httpbin &> /dev/null && fail "release should not exist anymore after a delete"
@@ -83,6 +106,9 @@ ${helm} status --namespace=${test_ns} httpbin &> /dev/null && fail "release shou
 info "Ensuring \"helmfile delete\" doesn't fail when no releases installed"
 ${helmfile} -f ${dir}/happypath.yaml delete || fail "\"helmfile delete\" shouldn't fail when there are no installed releases"
 
+info "Ensuring \"helmfile template\" output does contain only YAML docs"
+(${helmfile} -f ${dir}/happypath.yaml template | kubectl apply -f -) || fail "\"helmfile template | kubectl apply -f -\" shouldn't fail"
+
 test_pass "happypath"
 
 

test/integration/values.yaml

@@ -0,0 +1,9 @@
+resources: []
+templates:
+  - |
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: common-secret
+    stringData:
+      mykey: {{ .Values.mysecret }}