Merge pull request #396 from helmfile/bump_yaml_v3

bump: upgrade gopkg.in/yaml.v2 to gopkg.in/yaml.v3

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/Masterminds/sprig/v3 v3.2.2
 	github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a
 	github.com/davecgh/go-spew v1.1.1
-	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/go-test/deep v1.0.8
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.5.9
@@ -31,7 +30,7 @@ require (
 	go.uber.org/zap v1.23.0
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
-	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools v2.2.0+incompatible
 	helm.sh/helm/v3 v3.10.0
 	k8s.io/apimachinery v0.25.2
@@ -130,7 +129,6 @@ require (
 	gopkg.in/ini.v1 v1.66.2 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
@@ -191,6 +189,7 @@ require (
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/api v0.25.0 // indirect
 	k8s.io/cli-runtime v0.25.0 // indirect
 	k8s.io/client-go v0.25.0 // indirect

go.sum

@@ -267,8 +267,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fujiwara/tfstate-lookup v0.4.4 h1:HxVsq62Tk9W1RcbPPq7/c1s0Qd4bBFkccciFxMXZpLI=
 github.com/fujiwara/tfstate-lookup v0.4.4/go.mod h1:WaSNeC69vL1r+l4YDFTyJJDR2BrYofJmiHZtEBAt82M=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew=
-github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I=
 github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=

pkg/app/app_test.go

@@ -4660,7 +4660,7 @@ releases:
 	}
 	expectedValues := []interface{}{
 		map[interface{}]interface{}{"val1": "zipkin"},
-		map[interface{}]interface{}{"val2": "val2"}}
+		map[string]interface{}{"val2": "val2"}}
 	expectedSetValues := []state.SetValue{
 		{Name: "name-zipkin", Value: "val-zipkin"},
 		{Name: "name", Value: "val"}}

pkg/app/load_opts.go

@@ -1,8 +1,9 @@
 package app
 
 import (
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
+	"github.com/helmfile/helmfile/pkg/maputil"
 	"github.com/helmfile/helmfile/pkg/state"
 )
 
@@ -21,7 +22,7 @@ type LoadOpts struct {
 }
 
 func (o LoadOpts) DeepCopy() LoadOpts {
-	bytes, err := yaml.Marshal(o)
+	bytes, err := maputil.YamlMarshal(o)
 	if err != nil {
 		panic(err)
 	}

pkg/app/two_pass_renderer_test.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 	"testing"
 
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/helmfile/helmfile/pkg/helmexec"
 	"github.com/helmfile/helmfile/pkg/remote"

pkg/environment/environment.go

@@ -2,7 +2,7 @@ package environment
 
 import (
 	"github.com/imdario/mergo"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/helmfile/helmfile/pkg/maputil"
 )
@@ -16,7 +16,7 @@ type Environment struct {
 var EmptyEnvironment Environment
 
 func (e Environment) DeepCopy() Environment {
-	valuesBytes, err := yaml.Marshal(e.Values)
+	valuesBytes, err := maputil.YamlMarshal(e.Values)
 	if err != nil {
 		panic(err)
 	}
@@ -29,7 +29,7 @@ func (e Environment) DeepCopy() Environment {
 		panic(err)
 	}
 
-	defaultsBytes, err := yaml.Marshal(e.Defaults)
+	defaultsBytes, err := maputil.YamlMarshal(e.Defaults)
 	if err != nil {
 		panic(err)
 	}

pkg/maputil/yamlutil.go

@@ -0,0 +1,18 @@
+package maputil
+
+import (
+	"bytes"
+
+	"gopkg.in/yaml.v3"
+)
+
+func YamlMarshal(v interface{}) ([]byte, error) {
+	var b bytes.Buffer
+	yamlEncoder := yaml.NewEncoder(&b)
+	yamlEncoder.SetIndent(2)
+	err := yamlEncoder.Encode(v)
+	defer func() {
+		_ = yamlEncoder.Close()
+	}()
+	return b.Bytes(), err
+}

pkg/maputil/yamlutil_test.go

@@ -0,0 +1,34 @@
+package maputil
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestYamlMarshal(t *testing.T) {
+	tests := []struct {
+		Name string `yaml:"name"`
+		Info []struct {
+			Age     int    `yaml:"age"`
+			Address string `yaml:"address"`
+		} `yaml:"info"`
+
+		expected string
+	}{
+		{
+			Name: "John",
+			Info: []struct {
+				Age     int    `yaml:"age"`
+				Address string `yaml:"address"`
+			}{{Age: 20, Address: "New York"}},
+			expected: "name: John\ninfo:\n  - age: 20\n    address: New York\n",
+		},
+	}
+
+	for _, tt := range tests {
+		actual, err := YamlMarshal(tt)
+		require.NoError(t, err)
+		require.Equal(t, tt.expected, string(actual))
+	}
+}

pkg/remote/remote.go

@@ -14,7 +14,7 @@ import (
 	"github.com/hashicorp/go-getter/helper/url"
 	"go.uber.org/multierr"
 	"go.uber.org/zap"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/helmfile/helmfile/pkg/envvar"
 	"github.com/helmfile/helmfile/pkg/filesystem"

pkg/state/chart_dependency.go

@@ -11,10 +11,11 @@ import (
 	goversion "github.com/hashicorp/go-version"
 	"github.com/r3labs/diff"
 	"go.uber.org/zap"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/helmfile/helmfile/pkg/app/version"
 	"github.com/helmfile/helmfile/pkg/helmexec"
+	"github.com/helmfile/helmfile/pkg/maputil"
 )
 
 type ChartMeta struct {
@@ -299,7 +300,7 @@ func (m *chartDependencyManager) updateHelm3(shell helmexec.DependencyUpdater, w
 	chartMetaContent := fmt.Sprintf("name: %s\nversion: 1.0.0\napiVersion: v2\n", m.Name)
 
 	// Generate `requirements.yaml` of the temporary local chart from the helmfile state
-	reqsContent, err := yaml.Marshal(unresolved.ToChartRequirements())
+	reqsContent, err := maputil.YamlMarshal(unresolved.ToChartRequirements())
 	if err != nil {
 		return nil, err
 	}
@@ -317,7 +318,7 @@ func (m *chartDependencyManager) updateHelm2(shell helmexec.DependencyUpdater, w
 	}
 
 	// Generate `requirements.yaml` of the temporary local chart from the helmfile state
-	reqsContent, err := yaml.Marshal(unresolved.ToChartRequirements())
+	reqsContent, err := maputil.YamlMarshal(unresolved.ToChartRequirements())
 	if err != nil {
 		return nil, err
 	}
@@ -383,7 +384,7 @@ func (m *chartDependencyManager) doUpdate(chartLockFile string, unresolved *Unre
 
 	lockedReqs.Version = version.Version
 
-	updatedLockFileContent, err = yaml.Marshal(lockedReqs)
+	updatedLockFileContent, err = maputil.YamlMarshal(lockedReqs)
 
 	if err != nil {
 		return nil, err

pkg/state/create.go

@@ -9,7 +9,7 @@ import (
 	"github.com/imdario/mergo"
 	"github.com/variantdev/vals"
 	"go.uber.org/zap"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/helmfile/helmfile/pkg/environment"
 	"github.com/helmfile/helmfile/pkg/filesystem"
@@ -86,7 +86,7 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState,
 
 	decoder := yaml.NewDecoder(bytes.NewReader(content))
 
-	decoder.SetStrict(c.Strict)
+	decoder.KnownFields(c.Strict)
 
 	i := 0
 	for {

pkg/state/envvals_loader.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/imdario/mergo"
 	"go.uber.org/zap"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/helmfile/helmfile/pkg/environment"
 	"github.com/helmfile/helmfile/pkg/filesystem"
@@ -79,7 +79,7 @@ func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *str
 					ld.logger.Debugf("envvals_loader: loaded %s:%v", strOrMap, m)
 				}
 			}
-		case map[interface{}]interface{}:
+		case map[interface{}]interface{}, map[string]interface{}:
 			maps = append(maps, strOrMap)
 		default:
 			return nil, fmt.Errorf("unexpected type of value: value=%v, type=%T", strOrMap, strOrMap)

pkg/state/release.go

@@ -3,8 +3,9 @@ package state
 import (
 	"fmt"
 
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
+	"github.com/helmfile/helmfile/pkg/maputil"
 	"github.com/helmfile/helmfile/pkg/tmpl"
 )
 
@@ -97,8 +98,8 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
 	if len(result.ValuesTemplate) > 0 {
 		for i, t := range result.ValuesTemplate {
 			switch ts := t.(type) {
-			case map[interface{}]interface{}:
-				serialized, err := yaml.Marshal(ts)
+			case map[interface{}]interface{}, map[string]interface{}:
+				serialized, err := maputil.YamlMarshal(ts)
 				if err != nil {
 					return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err)
 				}
@@ -201,7 +202,7 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
 }
 
 func (r ReleaseSpec) Clone() (*ReleaseSpec, error) {
-	serialized, err := yaml.Marshal(r)
+	serialized, err := maputil.YamlMarshal(r)
 	if err != nil {
 		return nil, fmt.Errorf("failed cloning release \"%s\": %v", r.Name, err)
 	}

pkg/state/state.go

@@ -22,12 +22,13 @@ import (
 	"github.com/variantdev/chartify"
 	"github.com/variantdev/vals"
 	"go.uber.org/zap"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/helmfile/helmfile/pkg/environment"
 	"github.com/helmfile/helmfile/pkg/event"
 	"github.com/helmfile/helmfile/pkg/filesystem"
 	"github.com/helmfile/helmfile/pkg/helmexec"
+	"github.com/helmfile/helmfile/pkg/maputil"
 	"github.com/helmfile/helmfile/pkg/remote"
 	"github.com/helmfile/helmfile/pkg/tmpl"
 )
@@ -2663,7 +2664,7 @@ func (st *HelmState) RenderReleaseValuesFileToBytes(release *ReleaseSpec, path s
 			return nil, err
 		}
 
-		return yaml.Marshal(parsedYaml)
+		return maputil.YamlMarshal(parsedYaml)
 	}
 
 	return rawBytes, nil
@@ -2830,7 +2831,7 @@ func (st *HelmState) generateSecretValuesFiles(helm helmexec.Interface, release
 				return nil, err
 			}
 		default:
-			bs, err := yaml.Marshal(value)
+			bs, err := maputil.YamlMarshal(value)
 			if err != nil {
 				return nil, err
 			}
@@ -3071,7 +3072,7 @@ func (hf *SubHelmfileSpec) UnmarshalYAML(unmarshal func(interface{}) error) erro
 	switch i := tmp.(type) {
 	case string: // single path definition without sub items, legacy sub helmfile definition
 		hf.Path = i
-	case map[interface{}]interface{}: // helmfile path with sub section
+	case map[interface{}]interface{}, map[string]interface{}: // helmfile path with sub section
 		var subHelmfileSpecTmp struct {
 			Path               string   `yaml:"path"`
 			Selectors          []string `yaml:"selectors"`
@@ -3232,7 +3233,7 @@ func (st *HelmState) GenerateOutputFilePath(release *ReleaseSpec, outputFileTemp
 }
 
 func (st *HelmState) ToYaml() (string, error) {
-	if result, err := yaml.Marshal(st); err != nil {
+	if result, err := maputil.YamlMarshal(st); err != nil {
 		return "", err
 	} else {
 		return string(result), nil

pkg/state/state_exec_tmpl.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"reflect"
 
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/helmfile/helmfile/pkg/tmpl"
 )

pkg/tmpl/context_funcs.go

@@ -13,11 +13,12 @@ import (
 	"strings"
 	"text/template"
 
-	"github.com/ghodss/yaml"
 	"golang.org/x/sync/errgroup"
+	"gopkg.in/yaml.v3"
 
 	"github.com/helmfile/helmfile/pkg/envvar"
 	"github.com/helmfile/helmfile/pkg/helmexec"
+	"github.com/helmfile/helmfile/pkg/maputil"
 )
 
 type Values = map[string]interface{}
@@ -278,7 +279,7 @@ func (c *Context) Tpl(text string, data interface{}) (string, error) {
 }
 
 func ToYaml(v interface{}) (string, error) {
-	data, err := yaml.Marshal(v)
+	data, err := maputil.YamlMarshal(v)
 	if err != nil {
 		return "", err
 	}

pkg/tmpl/context_funcs_test.go

@@ -178,7 +178,7 @@ func TestReadFile_PassAbsPath(t *testing.T) {
 }
 
 func TestToYaml_UnsupportedNestedMapKey(t *testing.T) {
-	expected := ``
+	expected := "foo:\n  bar: BAR\n"
 	// nolint: unconvert
 	vals := Values(map[string]interface{}{
 		"foo": map[interface{}]interface{}{
@@ -186,8 +186,8 @@ func TestToYaml_UnsupportedNestedMapKey(t *testing.T) {
 		},
 	})
 	actual, err := ToYaml(vals)
-	require.Error(t, err, "error marshaling into JSON: json: unsupported type: map[interface {}]interface {}")
 	require.Equal(t, expected, actual)
+	require.NoError(t, err, "expected nil, but got: %v, when type: map[interface {}]interface {}", err)
 }
 
 func TestToYaml(t *testing.T) {

test/e2e/template/helmfile/testdata/snapshot/issue_2098_release_template_needs/output.yaml

@@ -6,23 +6,23 @@ helmBinary: helm
 environments:
   default: {}
 repositories:
-- name: aservo
-  url: https://aservo.github.io/charts
+  - name: aservo
+    url: https://aservo.github.io/charts
 releases:
-- chart: aservo/util
-  version: 0.0.1
-  name: default-shared-resources
-  namespace: default
-  labels:
-    service: shared-resources
-- chart: aservo/util
-  version: 0.0.1
-  needs:
-  - default-shared-resources
-  name: default-release-resources
-  namespace: default
-  labels:
-    service: release-resources
+  - chart: aservo/util
+    version: 0.0.1
+    name: default-shared-resources
+    namespace: default
+    labels:
+      service: shared-resources
+  - chart: aservo/util
+    version: 0.0.1
+    needs:
+      - default-shared-resources
+    name: default-release-resources
+    namespace: default
+    labels:
+      service: release-resources
 templates:
   defaults:
     name: default-{{ .Release.Labels.service }}

test/integration/issue.657.yaml

@@ -0,0 +1,16 @@
+templates:
+  default: &default
+    values:
+      - image:
+          repository: docker.io/citizenstig/httpbin
+          tag: latest
+          pullPolicy: IfNotPresent
+
+releases:
+  - name: httpbin
+    chart: ./charts/httpbin
+    <<: *default
+    values:
+      - image:
+          tag: 0.0.1
+          pullPolicy: Never

test/integration/run.sh

@@ -202,32 +202,44 @@ if [[ helm_major_version -eq 3 ]]; then
 
   test_pass "secretssops.2"
 
-    test_start "secretssops.3 - should order secrets correctly"
+  test_start "secretssops.3 - should order secrets correctly"
 
-    tmp=$(mktemp -d)
-    direct=${tmp}/direct.build.yaml
-    reverse=${tmp}/reverse.build.yaml
-    golden_dir=${dir}/secrets-golden
+  tmp=$(mktemp -d)
+  direct=${tmp}/direct.build.yaml
+  reverse=${tmp}/reverse.build.yaml
+  yaml_overwrite_reverse=${tmp}/yaml.override.build.yaml
+  secrets_golden_dir=${dir}/secrets-golden
+  feature_golden_dir=${dir}/yaml-features-golden
 
-    info "Building secrets output"
+  info "Building secrets output"
 
-    info "Comparing build/direct output ${direct} with ${golden_dir}"
-    for i in $(seq 10); do
-        info "Comparing build/direct #$i"
-        ${helmfile} -f ${dir}/secretssops.yaml -e direct template --skip-deps > ${direct} || fail "\"helmfile template\" shouldn't fail"
-        ./yamldiff ${golden_dir}/direct.build.yaml ${direct} || fail "\"helmfile template\" should be consistent"
-        echo code=$?
-    done
+  info "Comparing build/direct output ${direct} with ${secrets_golden_dir}"
+  for i in $(seq 10); do
+      info "Comparing build/direct #$i"
+      ${helmfile} -f ${dir}/secretssops.yaml -e direct template --skip-deps > ${direct} || fail "\"helmfile template\" shouldn't fail"
+      ./yamldiff ${secrets_golden_dir}/direct.build.yaml ${direct} || fail "\"helmfile template\" should be consistent"
+      echo code=$?
+  done
 
-    info "Comparing build/reverse output ${direct} with ${golden_dir}"
-    for i in $(seq 10); do
-        info "Comparing build/reverse #$i"
-        ${helmfile} -f ${dir}/secretssops.yaml -e reverse template --skip-deps > ${reverse} || fail "\"helmfile template\" shouldn't fail"
-        ./yamldiff ${golden_dir}/reverse.build.yaml ${reverse} || fail "\"helmfile template\" should be consistent"
-        echo code=$?
-    done
+  info "Comparing build/reverse output ${direct} with ${secrets_golden_dir}"
+  for i in $(seq 10); do
+      info "Comparing build/reverse #$i"
+      ${helmfile} -f ${dir}/secretssops.yaml -e reverse template --skip-deps > ${reverse} || fail "\"helmfile template\" shouldn't fail"
+      ./yamldiff ${secrets_golden_dir}/reverse.build.yaml ${reverse} || fail "\"helmfile template\" should be consistent"
+      echo code=$?
+  done
 
-    test_pass "secretssops.3"
+  test_pass "secretssops.3"
+
+  test_start "yaml overwrite feature"
+  info "Comparing yaml overwrite feature output ${yaml_overwrite_reverse} with ${feature_golden_dir}/overwritten.yaml"
+  for i in $(seq 10); do
+      info "Comparing build/yaml-overwrite #$i"
+      ${helmfile} -f ${dir}/issue.657.yaml template --skip-deps > ${yaml_overwrite_reverse} || fail "\"helmfile template\" shouldn't fail"
+      ./yamldiff ${feature_golden_dir}/overwritten.yaml ${yaml_overwrite_reverse} || fail "\"helmfile template\" should be consistent"
+      echo code=$?
+  done
+  test_pass "yaml overwrite feature"
 
 fi
 

test/integration/yaml-features-golden/overwritten.yaml

@@ -0,0 +1,63 @@
+---
+# Source: httpbin/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: httpbin-httpbin
+  labels:
+    app: httpbin
+    chart: httpbin-0.1.0
+    release: httpbin
+    heritage: Helm
+spec:
+  type: LoadBalancer
+  ports:
+    - port: 8000
+      targetPort: 8000
+      protocol: TCP
+      name: httpbin
+  selector:
+    app: httpbin
+    release: httpbin
+---
+# Source: httpbin/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+metadata:
+  name: httpbin-httpbin
+  labels:
+    app: httpbin
+    chart: httpbin-0.1.0
+    release: httpbin
+    heritage: Helm  
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: httpbin
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        app: httpbin
+        release: httpbin
+    spec:
+      containers:
+      - name: httpbin
+        image: "docker.io/citizenstig/httpbin:0.0.1"
+        imagePullPolicy: Never
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8000
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 8000        
+        ports:
+          - containerPort: 8000
+        resources:
+          {}
+
+status: {}

test/yamldiff/yamldiff.go

@@ -10,7 +10,7 @@ import (
 	"github.com/kylelemons/godebug/pretty"
 	"github.com/logrusorgru/aurora"
 	"github.com/mattn/go-isatty"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 var version = "latest"