Use gopkg.in/yaml.v2 for Helmfile v0.x (#609)

This should fix #435 for Helmfile v0.x releases since the next v0.150.0.
We introduce a new envvar to opt-in to the new YAML library, so that you can give it a shot before upgrading your Helmfile to v1. The same envvar can be used to opt-out of the new YAML library after you upgrade to Helmfile v1, giving you a more flexible migration story.

Signed-off-by: Yusuke Kuoka <ykuoka@gmail.com>

pkg/app/app_template_test.go

@@ -15,8 +15,8 @@ import (
 	"github.com/helmfile/helmfile/pkg/exectest"
 	ffs "github.com/helmfile/helmfile/pkg/filesystem"
 	"github.com/helmfile/helmfile/pkg/helmexec"
+	"github.com/helmfile/helmfile/pkg/runtime"
 	"github.com/helmfile/helmfile/pkg/testhelper"
-	"github.com/helmfile/helmfile/pkg/yaml"
 )
 
 func TestTemplate(t *testing.T) {
@@ -309,20 +309,21 @@ releases:
 }
 
 func TestTemplate_StrictParsing(t *testing.T) {
-	v := yaml.GoccyGoYaml
-	yaml.GoccyGoYaml = true
-	t.Cleanup(func() {
-		yaml.GoccyGoYaml = v
-	})
-
 	type testcase struct {
-		ns    string
-		error string
+		goccyGoYaml bool
+		ns          string
+		error       string
 	}
 
 	check := func(t *testing.T, tc testcase) {
 		t.Helper()
 
+		v := runtime.GoccyGoYaml
+		runtime.GoccyGoYaml = tc.goccyGoYaml
+		t.Cleanup(func() {
+			runtime.GoccyGoYaml = v
+		})
+
 		var helm = &exectest.Helm{
 			FailOnUnexpectedList: true,
 			FailOnUnexpectedDiff: true,
@@ -381,8 +382,9 @@ releases:
 		})
 	}
 
-	t.Run("fail due to known field", func(t *testing.T) {
+	t.Run("fail due to unknown field with goccy/go-yaml", func(t *testing.T) {
 		check(t, testcase{
+			goccyGoYaml: true,
 			error: `in ./helmfile.yaml: failed to read helmfile.yaml: reading document at index 1: [4:3] unknown field "foobar"
        2 | releases:
        3 | - name: app1
@@ -391,6 +393,14 @@ releases:
        5 |   chart: incubator/raw`,
 		})
 	})
+
+	t.Run("fail due to unknown field with gopkg.in/yaml.v2", func(t *testing.T) {
+		check(t, testcase{
+			goccyGoYaml: false,
+			error: `in ./helmfile.yaml: failed to read helmfile.yaml: reading document at index 1: yaml: unmarshal errors:
+  line 4: field foobar not found in type state.ReleaseSpec`,
+		})
+	})
 }
 
 func TestTemplate_CyclicInheritance(t *testing.T) {

pkg/app/app_test.go

@@ -9,7 +9,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"regexp"
-	"runtime"
+	goruntime "runtime"
 	"strings"
 	"sync"
 	"testing"
@@ -25,6 +25,7 @@ import (
 	ffs "github.com/helmfile/helmfile/pkg/filesystem"
 	"github.com/helmfile/helmfile/pkg/helmexec"
 	"github.com/helmfile/helmfile/pkg/remote"
+	"github.com/helmfile/helmfile/pkg/runtime"
 	"github.com/helmfile/helmfile/pkg/state"
 	"github.com/helmfile/helmfile/pkg/testhelper"
 	"github.com/helmfile/helmfile/pkg/testutil"
@@ -4338,7 +4339,15 @@ myrelease4	testNamespace	true   	true     	id:myrelease1             	mychart1
 	assert.Equal(t, expected, out)
 }
 
-func TestSetValuesTemplate(t *testing.T) {
+func testSetValuesTemplate(t *testing.T, goccyGoYaml bool) {
+	t.Helper()
+
+	v := runtime.GoccyGoYaml
+	runtime.GoccyGoYaml = goccyGoYaml
+	t.Cleanup(func() {
+		runtime.GoccyGoYaml = v
+	})
+
 	files := map[string]string{
 		"/path/to/helmfile.yaml": `
 releases:
@@ -4357,7 +4366,7 @@ releases:
 `,
 	}
 	expectedValues := []interface{}{
-		map[interface{}]interface{}{"val1": "zipkin"},
+		map[string]interface{}{"val1": "zipkin"},
 		map[string]interface{}{"val2": "val2"}}
 	expectedSetValues := []state.SetValue{
 		{Name: "name-zipkin", Value: "val-zipkin"},
@@ -4402,7 +4411,17 @@ releases:
 	}
 }
 
+func TestSetValuesTemplate(t *testing.T) {
+	t.Run("with goccy/go-yaml", func(t *testing.T) {
+		testSetValuesTemplate(t, true)
+	})
+
+	t.Run("with gopkg.in/yaml.v2", func(t *testing.T) {
+		testSetValuesTemplate(t, false)
+	})
+}
+
 func location() string {
-	_, fn, line, _ := runtime.Caller(1)
+	_, fn, line, _ := goruntime.Caller(1)
 	return fmt.Sprintf("%s:%d", filepath.Base(fn), line)
 }

pkg/envvar/const.go

@@ -10,4 +10,6 @@ const (
 	Helm3                         = "HELMFILE_HELM3"
 	UpgradeNoticeDisabled         = "HELMFILE_UPGRADE_NOTICE_DISABLED"
 	V1Mode                        = "HELMFILE_V1MODE"
+	GoccyGoYaml                   = "HELMFILE_GOCCY_GOYAML"
+	CacheHome                     = "HELMFILE_CACHE_HOME"
 )

pkg/remote/remote.go

@@ -27,6 +27,10 @@ func init() {
 }
 
 func CacheDir() string {
+	if h := os.Getenv(envvar.CacheHome); h != "" {
+		return h
+	}
+
 	dir, err := os.UserCacheDir()
 	if err != nil {
 		// fall back to relative path with hidden directory

pkg/runtime/runtime.go

@@ -13,13 +13,23 @@ import (
 var (
 	V1Mode bool
 
+	// GoccyGoYaml is set to true in order to let Helmfile use
+	// goccy/go-yaml instead of gopkg.in/yaml.v2.
+	// It's false by default in Helmfile v0.x and true by default for Helmfile v1.x.
+	GoccyGoYaml bool
+
 	// We set this via ldflags at build-time so that we can use the
 	// value specified at the build time as the runtime default.
 	v1Mode string
 )
 
 func Info() string {
-	return fmt.Sprintf("V1 mode = %v", V1Mode)
+	yamlLib := "gopkg.in/yaml.v2"
+	if GoccyGoYaml {
+		yamlLib = "goccy/go-yaml"
+	}
+
+	return fmt.Sprintf("V1 mode = %v\nYAML library = %v", V1Mode, yamlLib)
 }
 
 func init() {
@@ -34,4 +44,14 @@ func init() {
 	default:
 		V1Mode, _ = strconv.ParseBool(v1Mode)
 	}
+
+	// You can switch the YAML library at runtime via an envvar:
+	switch os.Getenv(envvar.GoccyGoYaml) {
+	case "true":
+		GoccyGoYaml = true
+	case "false":
+		GoccyGoYaml = false
+	default:
+		GoccyGoYaml = V1Mode
+	}
 }

pkg/state/release.go

@@ -3,6 +3,7 @@ package state
 import (
 	"fmt"
 
+	"github.com/helmfile/helmfile/pkg/maputil"
 	"github.com/helmfile/helmfile/pkg/tmpl"
 	"github.com/helmfile/helmfile/pkg/yaml"
 )
@@ -107,13 +108,18 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
 					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, string(serialized), err)
 				}
 
-				var deserialized map[interface{}]interface{}
+				var deserialized map[string]interface{}
 
 				if err := yaml.Unmarshal(s.Bytes(), &deserialized); err != nil {
 					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err)
 				}
 
-				result.ValuesTemplate[i] = deserialized
+				m, err := maputil.CastKeysToStrings(deserialized)
+				if err != nil {
+					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err)
+				}
+
+				result.ValuesTemplate[i] = m
 			}
 		}
 
@@ -130,6 +136,12 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
 				return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%s\": %v", r.Name, i, ts, err)
 			}
 			result.Values[i] = s.String()
+		case map[interface{}]interface{}:
+			m, err := maputil.CastKeysToStrings(ts)
+			if err != nil {
+				return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%s\": %v", r.Name, i, ts, err)
+			}
+			result.Values[i] = m
 		}
 	}
 

pkg/state/state_exec_tmpl_test.go

@@ -121,7 +121,7 @@ func TestHelmState_executeTemplates(t *testing.T) {
 				Verify:    nil,
 				Name:      "app",
 				Namespace: "dev",
-				Values:    []interface{}{map[interface{}]interface{}{"key": "app-val0"}},
+				Values:    []interface{}{map[string]interface{}{"key": "app-val0"}},
 			},
 		},
 	}

pkg/tmpl/context_funcs.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/helmfile/helmfile/pkg/envvar"
 	"github.com/helmfile/helmfile/pkg/helmexec"
+	"github.com/helmfile/helmfile/pkg/maputil"
 	"github.com/helmfile/helmfile/pkg/yaml"
 )
 
@@ -286,11 +287,17 @@ func ToYaml(v interface{}) (string, error) {
 }
 
 func FromYaml(str string) (Values, error) {
-	m := Values{}
+	m := map[string]interface{}{}
 
 	if err := yaml.Unmarshal([]byte(str), &m); err != nil {
 		return nil, fmt.Errorf("%s, offending yaml: %s", err, str)
 	}
+
+	m, err := maputil.CastKeysToStrings(m)
+	if err != nil {
+		return nil, fmt.Errorf("%s, offending yaml: %s", err, str)
+	}
+
 	return m, nil
 }
 

pkg/tmpl/context_funcs_test.go

@@ -5,12 +5,13 @@ import (
 	"fmt"
 	"io/fs"
 	"path/filepath"
-	"runtime"
+	goruntime "runtime"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/helmfile/helmfile/pkg/filesystem"
+	"github.com/helmfile/helmfile/pkg/runtime"
 )
 
 func TestCreateFuncMap(t *testing.T) {
@@ -126,7 +127,7 @@ func TestReadDir(t *testing.T) {
 		"sampleDirectory/file3.yaml",
 	}
 	var expectedArray []string
-	if runtime.GOOS == "windows" {
+	if goruntime.GOOS == "windows" {
 		expectedArray = expectedArrayWindows
 	} else {
 		expectedArray = expectedArrayUnix
@@ -177,16 +178,29 @@ func TestReadFile_PassAbsPath(t *testing.T) {
 	require.Equal(t, actual, expected)
 }
 
-func TestToYaml_UnsupportedNestedMapKey(t *testing.T) {
-	expected := "foo:\n  bar: BAR\n"
+func TestToYaml_NestedMapInterfaceKey(t *testing.T) {
+	v := runtime.GoccyGoYaml
+	t.Cleanup(func() {
+		runtime.GoccyGoYaml = v
+	})
+
 	// nolint: unconvert
 	vals := Values(map[string]interface{}{
 		"foo": map[interface{}]interface{}{
 			"bar": "BAR",
 		},
 	})
+
+	runtime.GoccyGoYaml = true
+
 	actual, err := ToYaml(vals)
-	require.Equal(t, expected, actual)
+	require.Equal(t, "foo:\n  bar: BAR\n", actual)
+	require.NoError(t, err, "expected nil, but got: %v, when type: map[interface {}]interface {}", err)
+
+	runtime.GoccyGoYaml = false
+
+	actual, err = ToYaml(vals)
+	require.Equal(t, "foo:\n  bar: BAR\n", actual)
 	require.NoError(t, err, "expected nil, but got: %v, when type: map[interface {}]interface {}", err)
 }
 
@@ -205,21 +219,51 @@ func TestToYaml(t *testing.T) {
 	require.Equal(t, expected, actual)
 }
 
-func TestFromYaml(t *testing.T) {
+func testFromYaml(t *testing.T, goccyGoYaml bool, expected Values) {
+	t.Helper()
+
+	v := runtime.GoccyGoYaml
+	runtime.GoccyGoYaml = goccyGoYaml
+	t.Cleanup(func() {
+		runtime.GoccyGoYaml = v
+	})
+
 	raw := `foo:
   bar: BAR
 `
-	// nolint: unconvert
-	expected := Values(map[string]interface{}{
-		"foo": map[string]interface{}{
-			"bar": "BAR",
-		},
-	})
 	actual, err := FromYaml(raw)
 	require.NoError(t, err)
 	require.Equal(t, expected, actual)
 }
 
+func TestFromYaml(t *testing.T) {
+	t.Run("with goccy/go-yaml", func(t *testing.T) {
+		testFromYaml(
+			t,
+			true,
+			// nolint: unconvert
+			Values(map[string]interface{}{
+				"foo": map[string]interface{}{
+					"bar": "BAR",
+				},
+			}),
+		)
+	})
+
+	t.Run("with gopkg.in/yaml.v2", func(t *testing.T) {
+		testFromYaml(
+			t,
+			false,
+			// nolint: unconvert
+			Values(map[string]interface{}{
+				"foo": map[string]interface{}{
+					"bar": "BAR",
+				},
+			}),
+		)
+	})
+}
+
 func TestFromYamlToJson(t *testing.T) {
 	input := `foo:
   bar: BAR

pkg/yaml/yaml.go

@@ -6,12 +6,8 @@ import (
 
 	"github.com/goccy/go-yaml"
 	v2 "gopkg.in/yaml.v2"
-)
 
-var (
-	// We'll derive the default from the build once
-	// is merged
-	GoccyGoYaml bool = true
+	"github.com/helmfile/helmfile/pkg/runtime"
 )
 
 type Encoder interface {
@@ -21,7 +17,7 @@ type Encoder interface {
 
 // NewEncoder creates and returns a function that is used to encode a Go object to a YAML document
 func NewEncoder(w io.Writer) Encoder {
-	if GoccyGoYaml {
+	if runtime.GoccyGoYaml {
 		return yaml.NewEncoder(w)
 	}
 
@@ -29,7 +25,7 @@ func NewEncoder(w io.Writer) Encoder {
 }
 
 func Unmarshal(data []byte, v interface{}) error {
-	if GoccyGoYaml {
+	if runtime.GoccyGoYaml {
 		return yaml.Unmarshal(data, v)
 	}
 
@@ -41,7 +37,7 @@ func Unmarshal(data []byte, v interface{}) error {
 // When strict is true, this function ensures that every field found in the YAML document
 // to have the corresponding field in the decoded Go struct.
 func NewDecoder(data []byte, strict bool) func(interface{}) error {
-	if GoccyGoYaml {
+	if runtime.GoccyGoYaml {
 		var opts []yaml.DecodeOption
 		if strict {
 			opts = append(opts, yaml.DisallowUnknownField())
@@ -66,11 +62,10 @@ func NewDecoder(data []byte, strict bool) func(interface{}) error {
 }
 
 func Marshal(v interface{}) ([]byte, error) {
-	if GoccyGoYaml {
+	if runtime.GoccyGoYaml {
 		var b bytes.Buffer
 		yamlEncoder := yaml.NewEncoder(
 			&b,
-			yaml.IndentSequence(true),
 			yaml.Indent(2),
 		)
 		err := yamlEncoder.Encode(v)

pkg/yaml/yaml_test.go

@@ -4,9 +4,19 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/helmfile/helmfile/pkg/runtime"
 )
 
-func TestYamlMarshal(t *testing.T) {
+func testYamlMarshal(t *testing.T, goccyGoYaml bool) {
+	t.Helper()
+
+	v := runtime.GoccyGoYaml
+	runtime.GoccyGoYaml = goccyGoYaml
+	t.Cleanup(func() {
+		runtime.GoccyGoYaml = v
+	})
+
 	tests := []struct {
 		Name string `yaml:"name"`
 		Info []struct {
@@ -22,7 +32,7 @@ func TestYamlMarshal(t *testing.T) {
 				Age     int    `yaml:"age"`
 				Address string `yaml:"address"`
 			}{{Age: 20, Address: "New York"}},
-			expected: "name: John\ninfo:\n  - age: 20\n    address: New York\n",
+			expected: "name: John\ninfo:\n- age: 20\n  address: New York\n",
 		},
 	}
 
@@ -32,3 +42,13 @@ func TestYamlMarshal(t *testing.T) {
 		require.Equal(t, tt.expected, string(actual))
 	}
 }
+
+func TestYamlMarshal(t *testing.T) {
+	t.Run("with goccy/go-yaml", func(t *testing.T) {
+		testYamlMarshal(t, true)
+	})
+
+	t.Run("with gopkg.in/yaml.v2", func(t *testing.T) {
+		testYamlMarshal(t, false)
+	})
+}