Use go-getter with secrets as well (#560)

* Use go-getter with secrets as well

Signed-off-by: Quan TRAN <account@itscaro.me>

docs/index.md

@@ -914,9 +914,9 @@ This is still working but is **deprecated** and the new `{{ .Values.foo }}` synt
 
 You can read more infos about the feature proposal [here](https://github.com/roboll/helmfile/issues/640).
 
-### Loading remote environment values files
+### Loading remote Environment values files
 
-Since #1296 and Helmfile v0.118.8, you can use `go-getter`-style URLs to refer to remote values files:
+Since Helmfile v0.118.8, you can use `go-getter`-style URLs to refer to remote values files:
 
 ```yaml
 environments:
@@ -946,13 +946,13 @@ This is particularly useful when you co-locate helmfiles within your project rep
 
 ## Environment Secrets
 
-Environment Secrets (not to be confused with Kubernetes Secrets) are encrypted versions of `Environment Values`.
+Environment Secrets *(not to be confused with Kubernetes Secrets)* are encrypted versions of `Environment Values`.
 You can list any number of `secrets.yaml` files created using `helm secrets` or `sops`, so that
 Helmfile could automatically decrypt and merge the secrets into the environment values.
 
 First you must have the [helm-secrets](https://github.com/jkroepke/helm-secrets) plugin installed along with a
 `.sops.yaml` file to configure the method of encryption (this can be in the same directory as your helmfile or
-in the sub-directory containing your secrets files).
+in the subdirectory containing your secrets files).
 
 Then suppose you have a secret `foo.bar` defined in `environments/production/secrets.yaml`:
 
@@ -985,6 +985,21 @@ Then the environment secret `foo.bar` can be referenced by the below template ex
 {{ .Values.foo.bar }}
 ```
 
+### Loading remote Environment secrets files
+
+Since Helmfile v0.149.0, you can use `go-getter`-style URLs to refer to remote secrets files, the same way as in values files:
+```yaml
+environments:
+  staging:
+    secrets:
+      - git::https://{{ env "GITHUB_PAT" }}@github.com/org/repo.git@/environments/staging.secret.yaml?ref=main
+      - http://$HOSTNAME/artifactory/example-repo-local/test.tgz@environments/staging.secret.yaml
+  production:
+    secrets:
+      - git::https://{{ env "GITHUB_PAT" }}@github.com/org/repo.git@/environments/production.secret.yaml?ref=main
+      - http://$HOSTNAME/artifactory/example-repo-local/test.tgz@environments/production.secret.yaml
+```
+
 ## Tillerless
 
 With the [helm-tiller](https://github.com/rimusz/helm-tiller) plugin installed, you can work without tiller installed.

pkg/envvar/const.go

@@ -1,9 +1,12 @@
 package envvar
 
-const DisableInsecureFeatures = "HELMFILE_DISABLE_INSECURE_FEATURES"
+const (
-const SkipInsecureTemplateFunctions = "HELMFILE_SKIP_INSECURE_TEMPLATE_FUNCTIONS"
+	DisableInsecureFeatures       = "HELMFILE_DISABLE_INSECURE_FEATURES"
-const Experimental = "HELMFILE_EXPERIMENTAL" // environment variable for experimental features, expecting "true" lower case
+	DisableRunnerUniqueID         = "HELMFILE_DISABLE_RUNNER_UNIQUE_ID"
-const Environment = "HELMFILE_ENVIRONMENT"
+	SkipInsecureTemplateFunctions = "HELMFILE_SKIP_INSECURE_TEMPLATE_FUNCTIONS"
-const TempDir = "HELMFILE_TEMPDIR"
+	Experimental                  = "HELMFILE_EXPERIMENTAL" // environment variable for experimental features, expecting "true" lower case
-const Helm3 = "HELMFILE_HELM3"
+	Environment                   = "HELMFILE_ENVIRONMENT"
-const UpgradeNoticeDisabled = "HELMFILE_UPGRADE_NOTICE_DISABLED"
+	TempDir                       = "HELMFILE_TEMPDIR"
+	Helm3                         = "HELMFILE_HELM3"
+	UpgradeNoticeDisabled         = "HELMFILE_UPGRADE_NOTICE_DISABLED"
+)

pkg/helmexec/runner.go

@@ -13,6 +13,8 @@ import (
 	"syscall"
 
 	"go.uber.org/zap"
+
+	"github.com/helmfile/helmfile/pkg/envvar"
 )
 
 // Runner interface for shell commands
@@ -68,9 +70,18 @@ func Output(c *exec.Cmd, logWriterGenerators ...*logWriterGenerator) ([]byte, er
 
 	var logWriters []io.Writer
 
-	id := newExecutionID()
+	var id string
+	if os.Getenv(envvar.DisableRunnerUniqueID) == "" {
+		id = newExecutionID()
+	}
+	path := filepath.Base(c.Path)
 	for _, g := range logWriterGenerators {
-		logPrefix := fmt.Sprintf("%s:%s> ", filepath.Base(c.Path), id)
+		var logPrefix string
+		if id == "" {
+			logPrefix = fmt.Sprintf("%s> ", path)
+		} else {
+			logPrefix = fmt.Sprintf("%s:%s> ", path, id)
+		}
 		logWriters = append(logWriters, g.Writer(logPrefix))
 	}
 

pkg/remote/remote.go

@@ -60,7 +60,7 @@ func (r *Remote) Unmarshal(src string, dst interface{}) error {
 	ext := filepath.Ext(file)
 
 	{
-		r.Logger.Debugf("unmarshalling %s", string(bytes))
+		r.Logger.Debugf("remote> unmarshalling %s", string(bytes))
 
 		var err error
 		switch ext {
@@ -70,7 +70,7 @@ func (r *Remote) Unmarshal(src string, dst interface{}) error {
 			err = yaml.Unmarshal(bytes, dst)
 		}
 
-		r.Logger.Debugf("unmarshalled to %v", dst)
+		r.Logger.Debugf("remote> unmarshalled to %v", dst)
 
 		if err != nil {
 			return err
@@ -172,12 +172,12 @@ func (r *Remote) Fetch(goGetterSrc string, cacheDirOpt ...string) (string, error
 	srcDir := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, u.Dir)
 	file := u.File
 
-	r.Logger.Debugf("getter: %s", u.Getter)
+	r.Logger.Debugf("remote> getter: %s", u.Getter)
-	r.Logger.Debugf("scheme: %s", u.Scheme)
+	r.Logger.Debugf("remote> scheme: %s", u.Scheme)
-	r.Logger.Debugf("user: %s", u.User)
+	r.Logger.Debugf("remote> user: %s", u.User)
-	r.Logger.Debugf("host: %s", u.Host)
+	r.Logger.Debugf("remote> host: %s", u.Host)
-	r.Logger.Debugf("dir: %s", u.Dir)
+	r.Logger.Debugf("remote> dir: %s", u.Dir)
-	r.Logger.Debugf("file: %s", u.File)
+	r.Logger.Debugf("remote> file: %s", u.File)
 
 	// This should be shared across variant commands, so that they can share cache for the shared imports
 	cacheBaseDir := ""
@@ -211,9 +211,9 @@ func (r *Remote) Fetch(goGetterSrc string, cacheDirOpt ...string) (string, error
 	// e.g. os.CacheDir()/helmfile/https_github_com_cloudposse_helmfiles_git.ref=0.xx.0
 	cacheDirPath := filepath.Join(r.Home, getterDst)
 
-	r.Logger.Debugf("home: %s", r.Home)
+	r.Logger.Debugf("remote> home: %s", r.Home)
-	r.Logger.Debugf("getter dest: %s", getterDst)
+	r.Logger.Debugf("remote> getter dest: %s", getterDst)
-	r.Logger.Debugf("cached dir: %s", cacheDirPath)
+	r.Logger.Debugf("remote> cached dir: %s", cacheDirPath)
 
 	{
 		if r.fs.FileExistsAt(cacheDirPath) {
@@ -241,7 +241,7 @@ func (r *Remote) Fetch(goGetterSrc string, cacheDirOpt ...string) (string, error
 			getterSrc = u.Getter + "::" + getterSrc
 		}
 
-		r.Logger.Debugf("downloading %s to %s", getterSrc, getterDst)
+		r.Logger.Debugf("remote> downloading %s to %s", getterSrc, getterDst)
 
 		if err := r.Getter.Get(r.Home, getterSrc, cacheDirPath); err != nil {
 			rmerr := os.RemoveAll(cacheDirPath)

pkg/state/create.go

@@ -293,9 +293,15 @@ func (c *StateCreator) scatterGatherEnvSecretFiles(st *HelmState, envSecretFiles
 		},
 		func(id int) {
 			for secret := range secrets {
+				urlOrPath := secret.path
+				localPath, err := c.remote.Locate(urlOrPath)
+				if err == nil {
+					urlOrPath = localPath
+				}
+
 				release := &ReleaseSpec{}
 				flags := st.appendConnectionFlags([]string{}, helm, release)
-				decFile, err := helm.DecryptSecret(st.createHelmContext(release, 0), secret.path, flags...)
+				decFile, err := helm.DecryptSecret(st.createHelmContext(release, 0), urlOrPath, flags...)
 				if err != nil {
 					results <- secretResult{secret.id, nil, err, secret.path}
 					continue

test/e2e/template/helmfile/snapshot_test.go

@@ -16,6 +16,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"github.com/variantdev/chartify/helmtesting"
 	"gopkg.in/yaml.v3"
+
+	"github.com/helmfile/helmfile/pkg/envvar"
 )
 
 var (
@@ -46,6 +48,9 @@ func TestHelmfileTemplateWithBuildCommand(t *testing.T) {
 	_, filename, _, _ := runtime.Caller(0)
 	projectRoot := filepath.Join(filepath.Dir(filename), "..", "..", "..", "..")
 	helmfileBin := filepath.Join(projectRoot, "helmfile")
+	if runtime.GOOS == "windows" {
+		helmfileBin = helmfileBin + ".exe"
+	}
 	testdataDir := "testdata/snapshot"
 	chartsDir := "testdata/charts"
 
@@ -169,6 +174,12 @@ func TestHelmfileTemplateWithBuildCommand(t *testing.T) {
 			args := []string{"-f", inputFile}
 			args = append(args, helmfileArgs...)
 			cmd := exec.CommandContext(ctx, helmfileBin, args...)
+			cmd.Env = os.Environ()
+			cmd.Env = append(
+				cmd.Env,
+				envvar.TempDir+"=/tmp/helmfile",
+				envvar.DisableRunnerUniqueID+"=1",
+			)
 			got, err := cmd.CombinedOutput()
 			if err != nil {
 				t.Logf("Output from %v: %s", args, string(got))

test/e2e/template/helmfile/testdata/snapshot/pr_560/config.yaml

@@ -0,0 +1,4 @@
+helmfileArgs:
+- template
+- --debug
+- --concurrency=1

test/e2e/template/helmfile/testdata/snapshot/pr_560/input.yaml

@@ -0,0 +1,17 @@
+releases:
+- name: foo
+  chart: ../../charts/raw-0.1.0
+  values:
+  - templates:
+      - |
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: {{`{{ .Release.Name }}`}}-1
+          namespace: {{`{{ .Release.Namespace }}`}}
+        data:
+          foo: FOO
+  - git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/values.yaml?ref=main
+  secrets:
+  - git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/secrets.yaml?ref=main
+  missingFileHandler: Debug

test/e2e/template/helmfile/testdata/snapshot/pr_560/output.yaml

@@ -0,0 +1,107 @@
+processing file "input.yaml" in directory "testdata/snapshot/pr_560"
+changing working directory to "/home/runner/work/helmfile/helmfile/test/e2e/template/helmfile/testdata/snapshot/pr_560"
+first-pass rendering starting for "input.yaml.part.0": inherited=&{default map[] map[]}, overrode=<nil>
+first-pass uses: &{default map[] map[]}
+first-pass rendering output of "input.yaml.part.0":
+ 0: releases:
+ 1: - name: foo
+ 2:   chart: ../../charts/raw-0.1.0
+ 3:   values:
+ 4:   - templates:
+ 5:       - |
+ 6:         apiVersion: v1
+ 7:         kind: ConfigMap
+ 8:         metadata:
+ 9:           name: {{ .Release.Name }}-1
+10:           namespace: {{ .Release.Namespace }}
+11:         data:
+12:           foo: FOO
+13:   - git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/values.yaml?ref=main
+14:   secrets:
+15:   - git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/secrets.yaml?ref=main
+16:   missingFileHandler: Debug
+17: 
+
+first-pass produced: &{default map[] map[]}
+first-pass rendering result of "input.yaml.part.0": {default map[] map[]}
+vals:
+map[]
+defaultVals:[]
+second-pass rendering result of "input.yaml.part.0":
+ 0: releases:
+ 1: - name: foo
+ 2:   chart: ../../charts/raw-0.1.0
+ 3:   values:
+ 4:   - templates:
+ 5:       - |
+ 6:         apiVersion: v1
+ 7:         kind: ConfigMap
+ 8:         metadata:
+ 9:           name: {{ .Release.Name }}-1
+10:           namespace: {{ .Release.Namespace }}
+11:         data:
+12:           foo: FOO
+13:   - git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/values.yaml?ref=main
+14:   secrets:
+15:   - git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/secrets.yaml?ref=main
+16:   missingFileHandler: Debug
+17: 
+
+merged environment: &{default map[] map[]}
+helm> v3.10.2+g50f003e
+Building dependency release=foo, chart=../../charts/raw-0.1.0
+exec: helm dependency build ../../charts/raw-0.1.0 --skip-refresh
+1 release(s) found in input.yaml
+
+processing 1 groups of releases in this order:
+GROUP RELEASES
+1     foo
+
+processing releases in group 1/1: foo
+remote> getter: git
+remote> scheme: https
+remote> user: 
+remote> host: github.com
+remote> dir: /helmfile/helmfile.git
+remote> file: test/e2e/template/helmfile/testdata/snapshot/pr_560/values.yaml
+remote> home: /home/runner/.cache/helmfile
+remote> getter dest: values/https_github_com_helmfile_helmfile_git.ref=main
+remote> cached dir: /home/runner/.cache/helmfile/values/https_github_com_helmfile_helmfile_git.ref=main
+remote> downloading git::https://github.com/helmfile/helmfile.git?ref=main to values/https_github_com_helmfile_helmfile_git.ref=main
+client: {Ctx:context.Background Src:git::https://github.com/helmfile/helmfile.git?ref=main Dst:/home/runner/.cache/helmfile/values/https_github_com_helmfile_helmfile_git.ref=main Pwd:/home/runner/.cache/helmfile Mode:3 Umask:---------- Detectors:[] Decompressors:map[] Getters:map[] Dir:false ProgressListener:<nil> Insecure:false DisableSymlinks:false Options:[]}
+skipping missing values file matching "git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/values.yaml?ref=main"
+remote> getter: git
+remote> scheme: https
+remote> user: 
+remote> host: github.com
+remote> dir: /helmfile/helmfile.git
+remote> file: test/e2e/template/helmfile/testdata/snapshot/pr_560/secrets.yaml
+remote> home: /home/runner/.cache/helmfile
+remote> getter dest: values/https_github_com_helmfile_helmfile_git.ref=main
+remote> cached dir: /home/runner/.cache/helmfile/values/https_github_com_helmfile_helmfile_git.ref=main
+skipping missing secrets file matching "git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/secrets.yaml?ref=main"
+Templating release=foo, chart=../../charts/raw-0.1.0
+exec: helm template foo ../../charts/raw-0.1.0 --values /tmp/helmfile/foo-values-79c7c784c9 --debug
+helm> install.go:192: [debug] Original chart version: ""
+helm> install.go:209: [debug] CHART PATH: /home/runner/work/helmfile/helmfile/test/e2e/template/helmfile/testdata/charts/raw-0.1.0
+helm> ---
+# Source: raw/templates/resources.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: foo-1
+  namespace: default
+data:
+  foo: FOO
+---
+# Source: raw/templates/resources.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: foo-1
+  namespace: default
+data:
+  foo: FOO
+
+Removed /tmp/helmfile/foo-values-79c7c784c9
+changing working directory back to "/home/runner/work/helmfile/helmfile/test/e2e/template/helmfile"