Add helm-secrets-encrypted values template file (#1701)

Secret files ending with .gotmpl are now also rendered as a gotemplate.

```
releases:
- name: myapp
  secrets:
  - secrets.yaml.gotmpl
```

Note that currently, .gotmpl files must be valid YAML files as well.

The expected use-case of this feature is to compose a YAML array from values and encrypted secrets.

Without this feature, you would have tried to do something like the below, which didn't work.

**Example (doesn't work!)**

`values.yaml.gotmpl`:

```
environment:
  -   name: MY_EXTERNAL_IP
      value: |
          {{ exec "./get-external-ip.sh" (list "") }}
```

`secrets.yaml`:
```
_sops:
  #...
environment:
  - name: MY_SECRET_VALUE
    value: (encrypted by sops)
```

`helmfile.yaml`:

```
releases:
- name: foo
  values:
  - values.yaml
  secrets:
  - secrets.yaml
```

This doesn't work because `values.yaml` and the decrypted `secrets.yaml` are passed to `helm` to be merged, and helm overrides the array instead of merging or concatenating the arrays.

**Example (works!)**

Instead of `values.yaml` and `secrets.yaml`, you provide a single `secrets.yaml.gotmpl` that is a valid YAML and encrypted by sops:

```
_sops:
  #...
environment:
  -   name: MY_EXTERNAL_IP
      value: |
          {{ exec "./get-external-ip.sh" (list "") }}
  - name: MY_SECRET_VALUE
    value: (encrypted by sops)
```

`helmfile.yaml`:

```
releases:
- name: foo
  secrets:
  - secrets.yaml.gotmpl
```

Helmfile decrypts the gotmpl by handing it over to helm-secrets and then renders the result as a gotmpl file. The end result is that you have a two-element array `environments` that can be just passed to helm.

Resolves #1700

Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>

.circleci/config.yml

@@ -1,4 +1,4 @@
-version: 2
+version: 2.1
 
 jobs:
 
@@ -78,52 +78,43 @@ jobs:
   # thanks to https://raw.githubusercontent.com/weaveworks/launcher/master/.circleci/config.yml
   integration_tests:
     machine:
-      image: circleci/classic:201808-01
+      image: ubuntu-2004:202010-01
+    parameters:
+      helm-version:
+        type: string
     steps:
       - checkout
       - run: mkdir ~/build
       - attach_workspace:
           at: ~/build
       - run:
+          name: Install test dependencies
           command: |
             cp ~/build/helmfile ~/project/helmfile
             cp ~/build/diff-yamls ~/project/diff-yamls
             cp ~/build/yamldiff ~/project/yamldiff
-      - run: make -C .circleci helm2
+            if [[ "<< parameters.helm-version >>" == v3* ]]
-      - run: make -C .circleci kustomize
+            then
-      - run: make -C .circleci minikube
+              make -C .circleci helm
+            else
+              make -C .circleci helm2
+            fi            
+            make -C .circleci vault
+            make -C .circleci sops
+            make -C .circleci kustomize
+            make -C .circleci minikube
       - run:
           name: Execute integration tests
           environment:
             TERM: "xterm"
           command: |
-            make integration
+            export TERM=xterm
-
+            if [[ "<< parameters.helm-version >>" == v3* ]]
-  integration_tests_helm3:
+            then
-    machine:
+              HELMFILE_HELM3=1 make integration
-      image: circleci/classic:201808-01
+            else
-    steps:
+              make integration
-    - checkout
+            fi
-    - run: mkdir ~/build
-    - attach_workspace:
-        at: ~/build
-    - run:
-        command: |
-          cp ~/build/helmfile ~/project/helmfile
-          cp ~/build/diff-yamls ~/project/diff-yamls
-          cp ~/build/yamldiff ~/project/yamldiff
-    - run: make -C .circleci helm
-    - run: make -C .circleci vault
-    - run: make -C .circleci sops
-    - run: make -C .circleci kustomize
-    - run: make -C .circleci minikube
-    - run:
-        name: Execute integration tests
-        environment:
-          HELMFILE_HELM3: "1"
-          TERM: "xterm"
-        command: |
-          make integration
 
 # GITHUB_TOKEN env var must be setup in circleci console
 
@@ -156,9 +147,9 @@ workflows:
       - integration_tests:
           requires:
             - build
-      - integration_tests_helm3:
+          matrix:
-          requires:
+            parameters:
-            - build
+              helm-version: ["v2.17.0", "v3.4.2"]
       - release:
           filters:
             branches:

pkg/helmexec/exec.go

@@ -278,7 +278,14 @@ func (helm *execer) DecryptSecret(context HelmContext, name string, flags ...str
 		if len(decSuffix) == 0 {
 			decSuffix = ".yaml.dec"
 		}
-		decFilename := strings.Replace(absPath, ".yaml", decSuffix, 1)
+
+		// helm secrets replaces the extension with its suffix ONLY when the extension is ".yaml"
+		var decFilename string
+		if strings.HasSuffix(absPath, ".yaml") {
+			decFilename = strings.Replace(absPath, ".yaml", decSuffix, 1)
+		} else {
+			decFilename = absPath + decSuffix
+		}
 
 		secretBytes, err := ioutil.ReadFile(decFilename)
 		if err != nil {
@@ -308,7 +315,9 @@ func (helm *execer) DecryptSecret(context HelmContext, name string, flags ...str
 
 	if tempFile == nil {
 		tempFile = func(content []byte) (string, error) {
-			tmpFile, err := ioutil.TempFile("", "secret")
+			dir := filepath.Dir(name)
+			extension := filepath.Ext(name)
+			tmpFile, err := ioutil.TempFile(dir, "secret*"+extension)
 			if err != nil {
 				return "", err
 			}

pkg/helmexec/exec_test.go

@@ -289,6 +289,29 @@ Found secret in cache %s/secretName
 	}
 }
 
+func Test_DecryptSecretWithGotmpl(t *testing.T) {
+	var buffer bytes.Buffer
+	logger := NewLogger(&buffer, "debug")
+	helm := MockExecer(logger, "dev")
+
+	tmpFilePath := "path/to/temp/file"
+	helm.writeTempFile = func(content []byte) (string, error) {
+		return tmpFilePath, nil
+	}
+
+	secretName := "secretName.yaml.gotmpl"
+	_, decryptErr := helm.DecryptSecret(HelmContext{}, secretName)
+	cwd, err := filepath.Abs(".")
+	if err != nil {
+		t.Errorf("Error: %v", err)
+	}
+
+	expected := fmt.Sprintf(`%s/%s.yaml.dec`, cwd, secretName)
+	if d := cmp.Diff(expected, decryptErr.(*os.PathError).Path); d != "" {
+		t.Errorf("helmexec.DecryptSecret(): want (-), got (+):\n%s", d)
+	}
+}
+
 func Test_DiffRelease(t *testing.T) {
 	var buffer bytes.Buffer
 	logger := NewLogger(&buffer, "debug")

pkg/state/state.go

@@ -2601,7 +2601,7 @@ func (st *HelmState) generateVanillaValuesFiles(release *ReleaseSpec) ([]string,
 }
 
 func (st *HelmState) generateSecretValuesFiles(helm helmexec.Interface, release *ReleaseSpec, workerIndex int) ([]string, error) {
-	var generatedFiles []string
+	var generatedDecryptedFiles []interface{}
 
 	for _, v := range release.Secrets {
 		var (
@@ -2652,8 +2652,16 @@ func (st *HelmState) generateSecretValuesFiles(helm helmexec.Interface, release
 		if err != nil {
 			return nil, err
 		}
+		defer func() {
+			_ = os.Remove(valfile)
+		}()
 
-		generatedFiles = append(generatedFiles, valfile)
+		generatedDecryptedFiles = append(generatedDecryptedFiles, valfile)
+	}
+
+	generatedFiles, err := st.generateTemporaryReleaseValuesFiles(release, generatedDecryptedFiles, release.MissingFileHandler)
+	if err != nil {
+		return nil, err
 	}
 
 	return generatedFiles, nil

test/integration/.gnupg/openpgp-revocs.d/B2D6D7BBEC03B2E66571C8C00AD18E16CFDEF700.rev

@@ -0,0 +1,15 @@
+:-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: This is a revocation certificate
+
+iQG2BCABCgAgFiEEstbXu+wDsuZlccjACtGOFs/e9wAFAmA4+f8CHQAACgkQCtGO
+Fs/e9wBN4wv/S0D4RCu5+BLt8y0vpAI9o1iRXj+yzOvPJUCm/QH1kN39saeI3rbr
+8pyytpmzO5wIr+G+6BHh56p3NzjxWxfIt/RQ1vPMYf/dxzvtRuvYY00Fu683A65i
+UqU9hiA6q1310OrMyEds4GmIteM++5xtKhV2s3A4bXJp6QD83MKV3m1IFYJ5cWfM
+GmUSVlXBMiZ5Vfe8a04KaPKU3EkVSSIxLsvOW5+tPDZzkdUAHMCoHRAIBeOd7Aqo
+uNytWylZtk1SeSeglbpm22NrXdvQbxV2oZplEazegqkydyUy8XmCh/57t1lHekAZ
+Dcw0XgXu1s5TWj0vXa2g9sW+CKLGsDNg5XYp2cR4UGkGXCctp7aZb6k1+wv33zjB
+hEL03GRq4AQ7yRzMsP0ue1YveolUfHRy+9OlFVdN25rL2VOvIkAxknnpe1c0gPLk
+NOer0NAVPzE8QzHFQIDICriJ2urfzMm/nfa2UHW8O81opib5LNRbcPz+GOmKEgLf
+aKRmcXNIZkPx
+=ohmg
+-----END PGP PUBLIC KEY BLOCK-----

test/integration/charts/httpbin/templates/deployment.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
 metadata:
@@ -10,6 +10,9 @@ metadata:
     heritage: {{ .Release.Service }}  
 spec:
   replicas: 1
+  selector:
+    matchLabels:
+      app: {{ template "httpbin.name" . }}
   strategy: {}
   template:
     metadata:

test/integration/run.sh

@@ -20,6 +20,15 @@ test_ns="helmfile-tests-$(date +"%Y%m%d-%H%M%S")"
 helmfile="./helmfile --namespace=${test_ns}"
 helm="helm --kube-context=minikube"
 kubectl="kubectl --context=minikube --namespace=${test_ns}"
+helm_dir="${PWD}/${dir}/.helm"
+export HELM_DATA_HOME="${helm_dir}/data"
+export HELM_HOME="${HELM_DATA_HOME}"
+export HELM_PLUGINS="${HELM_DATA_HOME}/plugins"
+export HELM_CONFIG_HOME="${helm_dir}/config"
+HELM_SECRETS_VERSION=3.5.0
+HELM_DIFF_VERSION=3.0.0-rc.7
+export GNUPGHOME="${PWD}/${dir}/.gnupg"
+export SOPS_PGP_FP="B2D6D7BBEC03B2E66571C8C00AD18E16CFDEF700"
 
 # FUNCTIONS ----------------------------------------------------------------------------------------------------------
 
@@ -45,26 +54,32 @@ function retry() {
     done
 }
 
+function cleanup() {
+    set +e
+    info "Deleting ${helm_dir}"
+    rm -rf ${helm_dir} # remove helm data so reinstalling plugins does not fail
+    info "Deleting minikube namespace ${test_ns}"
+    $kubectl delete namespace ${test_ns} # remove namespace whenever we exit this script
+}
+
 # SETUP --------------------------------------------------------------------------------------------------------------
 
 set -e
+trap cleanup EXIT
 info "Using namespace: ${test_ns}"
 # helm v2
 if helm version --client 2>/dev/null | grep '"v2\.'; then
     helm_major_version=2
     info "Using Helm version: $(helm version --short --client | grep -o v.*$)"
     ${helm} init --stable-repo-url https://charts.helm.sh/stable --wait --override spec.template.spec.automountServiceAccountToken=true
-    ${helm} plugin ls | grep diff || ${helm} plugin install https://github.com/databus23/helm-diff --version v2.11.0+5
 else # helm v3
     helm_major_version=3
     info "Using Helm version: $(helm version --short | grep -o v.*$)"
-    ${helm} plugin ls | grep diff || ${helm} plugin install https://github.com/databus23/helm-diff --version v3.1.3
-    # ${helm} plugin ls | grep secrets || ${helm} plugin install https://github.com/jkroepke/helm-secrets --version v3.5.0
 fi
-info "Using Kustomize version: $(kustomize version --short | grep -o 'v[^ ]+')"
+${helm} plugin ls | grep diff || ${helm} plugin install https://github.com/databus23/helm-diff --version v${HELM_DIFF_VERSION}
+info "Using Kustomize version: $(kustomize version --short | grep -o 'v[0-9.]\+')"
 ${kubectl} get namespace ${test_ns} &> /dev/null && warn "Namespace ${test_ns} exists, from a previous test run?"
 $kubectl create namespace ${test_ns} || fail "Could not create namespace ${test_ns}"
-trap "{ $kubectl delete namespace ${test_ns}; }" EXIT # remove namespace whenever we exit this script
 
 
 # TEST CASES----------------------------------------------------------------------------------------------------------

test/integration/scripts/echo.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+echo $1

test/integration/secrets-golden/direct.build.yaml

@@ -13,3 +13,5 @@ stringData:
   key_1: value_1
   key_2: value_2
   key_shared: value_2
+  my_other_key: MY_OTHER_SECRET
+  my_templated_key: MY_TEMPLATED_SECRET

test/integration/secrets-golden/reverse.build.yaml

@@ -13,3 +13,5 @@ stringData:
   key_1: value_1
   key_2: value_2
   key_shared: value_1
+  my_other_key: MY_OTHER_SECRET
+  my_templated_key: MY_TEMPLATED_SECRET

test/integration/secrets.yaml

@@ -0,0 +1,29 @@
+my_other_secret: ENC[AES256_GCM,data:1a4CrBDijRelwRkFea5M,iv:NOLrOJwXq5ldgh7dfd80G2XKC/JaHw8ozT4DX1O6x1U=,tag:jJZJSSicLNFU8/m5uGeznw==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    lastmodified: '2021-02-26T15:11:53Z'
+    mac: ENC[AES256_GCM,data:Dl5X27oM8Ze4L6LX8zdJdMFiLvPDKl3ZrWSnR8Yn6zFRyUvd6Hy5uCW1JCZHpTtxPvvFCzmsJ4J9NTc2esbWg1YzPT2o3npbq8JXUajy2Dy624KqX+sq4dRgdKXyCnLEy4SL6VcuyjmbBP9qjZpRx2sbuFWdnQjt1qLJ7z/I+BE=,iv:tFhPgOjS/pP6pQatWiUlsS0+X77J6E0DOY48F0xdorM=,tag:wP1PjMeavF0U7GF/Mvgt5Q==,type:str]
+    pgp:
+    -   created_at: '2021-02-26T15:11:52Z'
+        enc: |
+            -----BEGIN PGP MESSAGE-----
+
+            hQGMA7Px5yX5jd+nAQv+KHcMR2P/4ywivwYMEIchWTxeSnA7foBVwxqpO3bcqOU8
+            2e4O2N6vI74u3rff2UzDaBENXOzdlcl8aBAkNaKsjv1vTLFiGkt28ZSYA4mMbLQi
+            JRbr1Ld376L8TXfP/roGJL3RsXVVXZQQHTw4DV+Wbb2P8bgROqN9edcuDB9JMyLK
+            GQPHC0QUCGVY+EtU6cTJ2ghZ3BxpBdwgm0uxxMT0W899ivP/2qmdMF+wGOVCO4dk
+            t0IiqZdck+AybdXMT+h2B949VvNQtotpSQEARuv89BPH71ynjayFnp28KQU3PhGN
+            KdJSiBZMmb9RlkSU56wGmglKGFqpSKDr8+jcUExN3QZhjZuC/k6qneg9xaKRmlno
+            B0a0TSDVEwd1qKVIsN29ALSAxWomAcs0H3gtZUd+8TEEYYm8F9dVCIf4npot9S1I
+            yASFvySKgWDgRttxbkwfAq0uxDGqOBRL67BVqqMk06jaEVX5x+TAwME1t7FYnOfk
+            +Z9wmAYcihL7iZg3HclU0l4BZSsDugB+YGubBfN/hkORQkMa2anyfzkshC181eJ1
+            mjXFgaPMj03vlBegGUUW+V5KUzmKT9czbWxLWq9KZtAOQTkI0kEz0jsJ8W/up3bU
+            eXR6z6sM5pHR7rn9liOj
+            =Ko7j
+            -----END PGP MESSAGE-----
+        fp: B2D6D7BBEC03B2E66571C8C00AD18E16CFDEF700
+    unencrypted_suffix: _unencrypted
+    version: 3.6.1

test/integration/secrets_templated.yaml.gotmpl

@@ -0,0 +1,29 @@
+my_templated_secret: ENC[AES256_GCM,data:tQeqvLecCi2sJkoZbbA5Vj0pyBqMqz7YIpmOWUTViEQi/UARNYHk4zZMW+IegElwW83yC9NsCHAuYg==,iv:RR75PQ4f6nteg+6ciizQxcTdiEAzfW0CpTg+3VtExBw=,tag:iO5SflR/Ny4oUs9LhxkN8w==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    lastmodified: '2021-02-26T15:44:26Z'
+    mac: ENC[AES256_GCM,data:9rpFipXjZdMxEnxDTKwmQ9NwLsoZ3PLtIeWIrOEMLkVCBiOIoYHdEcKt362o59+pyPLGu2Cg5XWcuIcy7HK5vYj7r97U7aI5OJ1yEzdNjl/D+lG9mCIMQ8CoHM4HlwIncBiDu6JTk4ufAGwAbzBagOWF6lFfgMC48s0f6E8AgDk=,iv:O49gGSGNsjtEBvOxoQGqh4g4WkrN+uDaWVrvXCmIRS8=,tag:Wnbojg645yuakxOHuE2v6w==,type:str]
+    pgp:
+    -   created_at: '2021-02-26T15:44:26Z'
+        enc: |
+            -----BEGIN PGP MESSAGE-----
+
+            hQGMA7Px5yX5jd+nAQv/desIyeJfXKCYB4LGBNp0N5InJnWUM5MBiGd97s1dDIP+
+            Iq7aFZL7K9YjVL9+dJuKZcInRxaLk6EwhHBlxohjoL3jAbNOM/5BcROdGbAeU+el
+            O9k/RfoQBwmuZj9VezGNEggS4dRn2n5/hvnj6n1dusvpkTAt/DaIbcI6KkrQ1PJt
+            0Tb5kSqkpobge9Hefm7v1RPz+ywDlfBfpRrhikQobIgvNaBoDSwb2JPkdui4gC5r
+            jj8QZn8V3wB6ulgGJCBstG4iI2xKeuMH9ejk6mfQDHPCEAgKgN+DatqQNPKiFezC
+            pUe7vg1LnyyZD6vE88wT15p2sB69MeBE2SNaYdNrhLNq4hOQxRmNCV5Ra1MuxA2d
+            3ZdTwCvEe9n7YAYU3+Cht42urPqJyRlO/8acEhQ4STS3uelM/GWqbcjV5JNN70Jv
+            44E7V+1Xx4MWhpFUA1VJKwYFkl+n/aXla6vr2WxG4AHk0dMe8658fLIwJn4AONW8
+            wfnJd1x25u0sV22PZ3N70l4BN+4EFjE2hXjlWykJnpKcnfdx8qdsiQ9Ww3W1QO9W
+            NpkLqY96e4gL47c5KHaRblYyB9opCL2h6HkztATGiy1iHshFAbJBLUghHmENG1mA
+            xrccqozKVloWcgsSrLa0
+            =y5Fb
+            -----END PGP MESSAGE-----
+        fp: B2D6D7BBEC03B2E66571C8C00AD18E16CFDEF700
+    unencrypted_suffix: _unencrypted
+    version: 3.6.1

test/integration/secretssops.yaml

@@ -24,6 +24,9 @@ releases:
   - name: raw
     chart: center/incubator/raw
     version: 0.2.3
+    secrets:
+    - secrets.yaml
+    - secrets_templated.yaml.gotmpl
     values:
       - templates:
         - |
@@ -35,3 +38,5 @@ releases:
             key_1: {{ .Environment.Values.key_1 }}
             key_2: {{ .Environment.Values.key_2 }}
             key_shared: {{ .Environment.Values.key_shared }}
+            my_other_key: {{` {{ .Values.my_other_secret }} `}}
+            my_templated_key: {{` {{ .Values.my_templated_secret }} `}}

test/integration/templates-golden/v2/httpbin/deployment.yaml

@@ -1,6 +1,6 @@
 ---
 # Source: httpbin/templates/deployment.yaml
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
 metadata:
@@ -12,6 +12,9 @@ metadata:
     heritage: Tiller
 spec:
   replicas: 1
+  selector:
+    matchLabels:
+      app: httpbin
   strategy: {}
   template:
     metadata:

test/integration/templates-golden/v3/httpbin/deployment.yaml

@@ -1,6 +1,6 @@
 ---
 # Source: httpbin/templates/deployment.yaml
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
 metadata:
@@ -12,6 +12,9 @@ metadata:
     heritage: Helm  
 spec:
   replicas: 1
+  selector:
+    matchLabels:
+      app: httpbin
   strategy: {}
   template:
     metadata: