Use gopkg.in/yaml.v2 for Helmfile v0.x (#609)

This should fix #435 for Helmfile v0.x releases since the next v0.150.0.
We introduce a new envvar to opt-in to the new YAML library, so that you can give it a shot before upgrading your Helmfile to v1. The same envvar can be used to opt-out of the new YAML library after you upgrade to Helmfile v1, giving you a more flexible migration story.

Signed-off-by: Yusuke Kuoka <ykuoka@gmail.com>

test/e2e/template/helmfile/snapshot_test.go

@@ -8,7 +8,8 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
-	"runtime"
+	goruntime "runtime"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -57,6 +58,18 @@ func (f fakeInit) Force() bool {
 }
 
 func TestHelmfileTemplateWithBuildCommand(t *testing.T) {
+	t.Run("with goccy/go-yaml", func(t *testing.T) {
+		testHelmfileTemplateWithBuildCommand(t, true)
+	})
+
+	t.Run("with gopkg.in/yaml.v2", func(t *testing.T) {
+		testHelmfileTemplateWithBuildCommand(t, false)
+	})
+}
+
+func testHelmfileTemplateWithBuildCommand(t *testing.T, goccyGoYaml bool) {
+	t.Setenv(envvar.GoccyGoYaml, strconv.FormatBool(goccyGoYaml))
+
 	localChartPortSets := make(map[int]struct{})
 
 	logger := helmexec.NewLogger(os.Stderr, "info")
@@ -69,10 +82,10 @@ func TestHelmfileTemplateWithBuildCommand(t *testing.T) {
 	err := helmfileInit.CheckHelmPlugins()
 	require.NoError(t, err)
 
-	_, filename, _, _ := runtime.Caller(0)
+	_, filename, _, _ := goruntime.Caller(0)
 	projectRoot := filepath.Join(filepath.Dir(filename), "..", "..", "..", "..")
 	helmfileBin := filepath.Join(projectRoot, "helmfile")
-	if runtime.GOOS == "windows" {
+	if goruntime.GOOS == "windows" {
 		helmfileBin = helmfileBin + ".exe"
 	}
 	testdataDir := "testdata/snapshot"
@@ -200,6 +213,15 @@ func TestHelmfileTemplateWithBuildCommand(t *testing.T) {
 				}
 			}
 
+			tmpDir := t.TempDir()
+			// HELM_CACHE_HOME contains downloaded chart archives
+			helmCacheHome := filepath.Join(tmpDir, "helm_cache")
+			// HELMFILE_CACHE_HOME contains remote charts and manifests downloaded by Helmfile using the go-getter integration
+			helmfileCacheHome := filepath.Join(tmpDir, "helmfile_cache")
+			// HELM_CONFIG_HOME contains the registry auth file (registry.json) and the index of all the repos added via helm-repo-add (repositories.yaml).
+			helmConfigHome := filepath.Join(tmpDir, "helm_config")
+			t.Logf("Using HELM_CACHE_HOME=%s, HELMFILE_CACHE_HOME=%s, HELM_CONFIG_HOME=%s", helmCacheHome, helmfileCacheHome, helmConfigHome)
+
 			inputFile := filepath.Join(testdataDir, name, "input.yaml")
 			outputFile := filepath.Join(testdataDir, name, "output.yaml")
 
@@ -214,6 +236,9 @@ func TestHelmfileTemplateWithBuildCommand(t *testing.T) {
 				cmd.Env,
 				envvar.TempDir+"=/tmp/helmfile",
 				envvar.DisableRunnerUniqueID+"=1",
+				"HELM_CACHE_HOME="+helmCacheHome,
+				"HELM_CONFIG_HOME="+helmConfigHome,
+				"HELMFILE_CACHE_HOME="+helmfileCacheHome,
 			)
 			got, err := cmd.CombinedOutput()
 			if err != nil {
@@ -231,6 +256,9 @@ func TestHelmfileTemplateWithBuildCommand(t *testing.T) {
 			gotStr = chartGitFullPathRegex.ReplaceAllString(gotStr, `chart=$$GoGetterPath`)
 			// Replace helm version with $HelmVersion
 			gotStr = helmShortVersionRegex.ReplaceAllString(gotStr, `$$HelmVersion`)
+			// Replace all occurrences of HELMFILE_CACHE_HOME with /home/runner/.cache/helmfile
+			// for stable test result
+			gotStr = strings.ReplaceAll(gotStr, helmfileCacheHome, "/home/runner/.cache/helmfile")
 
 			// OCI based helm charts are pulled and exported under temporary directory.
 			// We are not sure the exact name of the temporary directory generated by helmfile,

test/e2e/template/helmfile/testdata/snapshot/issue_2098_release_template_needs/output.yaml

@@ -6,23 +6,23 @@ helmBinary: helm
 environments:
   default: {}
 repositories:
-  - name: aservo
-    url: https://aservo.github.io/charts
+- name: aservo
+  url: https://aservo.github.io/charts
 releases:
-  - chart: aservo/util
-    version: 0.0.1
-    name: default-shared-resources
-    namespace: default
-    labels:
-      service: shared-resources
-  - chart: aservo/util
-    version: 0.0.1
-    needs:
-      - default/default-shared-resources
-    name: default-release-resources
-    namespace: default
-    labels:
-      service: release-resources
+- chart: aservo/util
+  version: 0.0.1
+  name: default-shared-resources
+  namespace: default
+  labels:
+    service: shared-resources
+- chart: aservo/util
+  version: 0.0.1
+  needs:
+  - default/default-shared-resources
+  name: default-release-resources
+  namespace: default
+  labels:
+    service: release-resources
 templates:
   defaults:
     name: default-{{ .Release.Labels.service }}

test/e2e/template/helmfile/testdata/snapshot/oci_need/output.yaml

@@ -1,8 +1,4 @@
 Building dependency release=foo, chart=$WD/temp1/foo
-Hang tight while we grab the latest from your chart repositories...
-...Successfully got an update from the "myrepo" chart repository
-...Successfully got an update from the "istio" chart repository
-Update Complete. ⎈Happy Helming!⎈
 Saving 1 charts
 Downloading raw from repo oci://localhost:5000/myrepo
 Deleting outdated charts

test/e2e/template/helmfile/testdata/snapshot/postrenderer/output.yaml

@@ -1,9 +1,5 @@
 Building dependency release=foo, chart=../../charts/raw-0.1.0
 Building dependency release=baz, chart=$WD/temp1/baz
-Hang tight while we grab the latest from your chart repositories...
-...Successfully got an update from the "myrepo" chart repository
-...Successfully got an update from the "istio" chart repository
-Update Complete. ⎈Happy Helming!⎈
 Saving 1 charts
 Downloading raw from repo oci://localhost:5001/myrepo
 Deleting outdated charts

test/e2e/template/helmfile/testdata/snapshot/pr_560/config.yaml

@@ -1,4 +1,3 @@
 helmfileArgs:
 - template
-- --debug
 - --concurrency=1

test/e2e/template/helmfile/testdata/snapshot/pr_560/output.yaml

@@ -1,103 +1,5 @@
-processing file "input.yaml" in directory "testdata/snapshot/pr_560"
-changing working directory to "/home/runner/work/helmfile/helmfile/test/e2e/template/helmfile/testdata/snapshot/pr_560"
-first-pass rendering starting for "input.yaml.part.0": inherited=&{default map[] map[]}, overrode=<nil>
-first-pass uses: &{default map[] map[]}
-first-pass rendering output of "input.yaml.part.0":
- 0: releases:
- 1: - name: foo
- 2:   chart: ../../charts/raw-0.1.0
- 3:   values:
- 4:   - templates:
- 5:       - |
- 6:         apiVersion: v1
- 7:         kind: ConfigMap
- 8:         metadata:
- 9:           name: {{ .Release.Name }}-1
-10:           namespace: {{ .Release.Namespace }}
-11:         data:
-12:           foo: FOO
-13:   - git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/values.yaml?ref=main
-14:   secrets:
-15:   - git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/secrets.yaml?ref=main
-16:   missingFileHandler: Debug
-17: 
-
-first-pass produced: &{default map[] map[]}
-first-pass rendering result of "input.yaml.part.0": {default map[] map[]}
-vals:
-map[]
-defaultVals:[]
-second-pass rendering result of "input.yaml.part.0":
- 0: releases:
- 1: - name: foo
- 2:   chart: ../../charts/raw-0.1.0
- 3:   values:
- 4:   - templates:
- 5:       - |
- 6:         apiVersion: v1
- 7:         kind: ConfigMap
- 8:         metadata:
- 9:           name: {{ .Release.Name }}-1
-10:           namespace: {{ .Release.Namespace }}
-11:         data:
-12:           foo: FOO
-13:   - git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/values.yaml?ref=main
-14:   secrets:
-15:   - git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/secrets.yaml?ref=main
-16:   missingFileHandler: Debug
-17: 
-
-merged environment: &{default map[] map[]}
-helm> $HelmVersion
-helm> 
 Building dependency release=foo, chart=../../charts/raw-0.1.0
-exec: helm dependency build ../../charts/raw-0.1.0
-1 release(s) found in input.yaml
-
-processing 1 groups of releases in this order:
-GROUP RELEASES
-1     foo
-
-processing releases in group 1/1: foo
-remote> getter: git
-remote> scheme: https
-remote> user: 
-remote> host: github.com
-remote> dir: /helmfile/helmfile.git
-remote> file: test/e2e/template/helmfile/testdata/snapshot/pr_560/values.yaml
-remote> home: /home/runner/.cache/helmfile
-remote> getter dest: values/https_github_com_helmfile_helmfile_git.ref=main
-remote> cached dir: /home/runner/.cache/helmfile/values/https_github_com_helmfile_helmfile_git.ref=main
-remote> downloading git::https://github.com/helmfile/helmfile.git?ref=main to values/https_github_com_helmfile_helmfile_git.ref=main
-client: {Ctx:context.Background Src:git::https://github.com/helmfile/helmfile.git?ref=main Dst:/home/runner/.cache/helmfile/values/https_github_com_helmfile_helmfile_git.ref=main Pwd:/home/runner/.cache/helmfile Mode:3 Umask:---------- Detectors:[] Decompressors:map[] Getters:map[] Dir:false ProgressListener:<nil> Insecure:false DisableSymlinks:false Options:[]}
-skipping missing values file matching "git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/values.yaml?ref=main"
-remote> getter: git
-remote> scheme: https
-remote> user: 
-remote> host: github.com
-remote> dir: /helmfile/helmfile.git
-remote> file: test/e2e/template/helmfile/testdata/snapshot/pr_560/secrets.yaml
-remote> home: /home/runner/.cache/helmfile
-remote> getter dest: values/https_github_com_helmfile_helmfile_git.ref=main
-remote> cached dir: /home/runner/.cache/helmfile/values/https_github_com_helmfile_helmfile_git.ref=main
-skipping missing secrets file matching "git::https://github.com/helmfile/helmfile.git@test/e2e/template/helmfile/testdata/snapshot/pr_560/secrets.yaml?ref=main"
 Templating release=foo, chart=../../charts/raw-0.1.0
-exec: helm template foo ../../charts/raw-0.1.0 --values /tmp/helmfile/foo-values-d459bc67c --debug
-helm> install.go:192: [debug] Original chart version: ""
-helm> 
-helm> install.go:209: [debug] CHART PATH: /home/runner/work/helmfile/helmfile/test/e2e/template/helmfile/testdata/charts/raw-0.1.0
-helm> 
-helm> 
-helm> ---
-helm> # Source: raw/templates/resources.yaml
-helm> apiVersion: v1
-helm> kind: ConfigMap
-helm> metadata:
-helm> name: foo-1
-helm> namespace: default
-helm> data:
-helm> foo: FOO
-helm> 
 ---
 # Source: raw/templates/resources.yaml
 apiVersion: v1
@@ -108,6 +10,3 @@ metadata:
 data:
   foo: FOO
 
-Removed /tmp/helmfile/foo-values-d459bc67c
-Removed /tmp/helmfile
-changing working directory back to "/home/runner/work/helmfile/helmfile/test/e2e/template/helmfile"

test/e2e/template/helmfile/testdata/snapshot/release_template_inheritance/config.yaml

@@ -3,3 +3,5 @@ localChartRepoServer:
   port: 18084
 helmfileArgs:
 - template
+# Prevent two releases foo and bar from racing and randomizing the log
+- --concurrency=1

test/e2e/template/helmfile/testdata/snapshot/release_template_inheritance/input.yaml

@@ -12,7 +12,7 @@ templates:
     values:
     - template1: template1
     valuesTemplate:
-    - template1Label: "{{` '{{ .Release.Labels.template1 }}' `}}"
+    - template1Label: "{{`{{ .Release.Labels.template1 }}`}}"
     labels:
       template1: template1
     inherit:
@@ -23,8 +23,8 @@ templates:
     values:
     - template2: template2
     valuesTemplate:
-    - inheritedBaseLabel: "{{` '{{ .Release.Labels.base }}' `}}"
-      template2Label: "{{` '{{ .Release.Labels.template2 }}' `}}"
+    - inheritedBaseLabel: "{{`{{ .Release.Labels.base }}`}}"
+      template2Label: "{{`{{ .Release.Labels.template2 }}`}}"
     labels:
       template2: template2
     inherit:
@@ -46,7 +46,7 @@ releases:
         name: {{`{{ .Release.Name }}`}}-1
         namespace: {{`{{ .Release.Namespace }}`}}
       data:
-        {{` {{ .Values | toYaml | nindent 2 }} `}}
+        {{` {{ (unset .Values "templates") | toYaml | nindent 2 }} `}}
 - name: foo2
   chart: ../../charts/raw-0.1.0
   inherit:
@@ -60,4 +60,4 @@ releases:
         name: {{`{{ .Release.Name }}`}}-1
         namespace: {{`{{ .Release.Namespace }}`}}
       data:
-        {{` {{ .Values | toYaml | nindent 2 }} `}}
+        {{` {{ (unset .Values "templates") | toYaml | nindent 2 }} `}}

test/e2e/template/helmfile/testdata/snapshot/release_template_inheritance/output.yaml

@@ -16,9 +16,6 @@ data:
   base: base
   template1: template1
   template1Label: template1
-  templates:
-  - "apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: {{ .Release.Name }}-1\n  namespace:
-    {{ .Release.Namespace }}\ndata:\n   {{ .Values | toYaml | nindent 2 }} \n"
 
 Templating release=foo2, chart=../../charts/raw-0.1.0
 ---
@@ -33,13 +30,4 @@ data:
   inheritedBaseLabel: base
   template2: template2
   template2Label: template2
-  templates:
-  - |-
-    apiVersion: v1
-    kind: ConfigMap
-    metadata:
-      name: {{ .Release.Name }}-1
-      namespace: {{ .Release.Namespace }}
-    data:
-       {{ .Values | toYaml | nindent 2 }}