From 6664f0159609001934e612243a421aab5f33b612 Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Tue, 27 Dec 2022 10:14:35 +0900 Subject: [PATCH] Use goccy/go-yaml for v1 / Prep bringing back go-yaml v2 for v0.x (#604) This is a successor to #596. We need a smooth migration path from `gopkg.in/yaml.v2`, and this pull request moves it forward with `goccy/go-yaml` instead of `gopkg.in/yaml.v3`. Merging this unblocks users stuck in Helmfile v0.146.x or earlier due to #435, so that they can upgrade to 0.147.x or greater without updating their helmfile configs. We previously tried to upgrade to `yaml.v3` (https://github.com/helmfile/helmfile/issues/394) in Helmfile v0.x, presuming it won't break anything. Apparently, it broke use-cases where you want to layer release's `values` field over three or more release templates and releases (#435). We then tried to bring back `yaml.v2` for Helmfile v0.x and keep `yaml.v3` for the upcoming Helmfile v1. However, it failed due to incompatibility in the Unmarshaller interface between `yaml.v2` and `yaml.v3` (https://github.com/helmfile/helmfile/pull/596). `goccy/go-yaml` is, from my observation, a well-maintained alternative to `yaml.v2`. One of its premises is that it enables us to swap the implementation from `gopkg.in/yaml.v2` to `goccy/go-yaml` just by replacing the import directive. It seems to use the same `Unmarshaller` interface as yaml.v2 too. Once this PR gets merged, I'd like to follow-up with adding a new build-time variable and an envvar to set the proper default for the yaml parser Helmfile uses and the ability to switch the parser at runtime. All in all, the next Helmfile release, v0.150.0 will get reverted to use `gopkg.in/yaml.v2` by default which resolves #435. New users who started using Helmfile since any of v0.148.0, v0.148.1, and v0.149.0 might be already relying on the new behavior, They might need to specify a new envvar to enable `goccy/go-yaml`. Signed-off-by: yxxhero Signed-off-by: yxxhero Co-authored-by: yxxhero --- go.mod | 6 +- go.sum | 9 +- pkg/app/app_list_test.go | 4 +- pkg/app/app_template_test.go | 86 +++++++++++++++++++ pkg/app/app_test.go | 4 +- pkg/app/load_opts.go | 6 +- pkg/app/two_pass_renderer_test.go | 3 +- pkg/environment/environment.go | 6 +- pkg/helmexec/exec.go | 2 +- pkg/maputil/yamlutil.go | 18 ---- pkg/remote/remote.go | 2 +- pkg/state/chart_dependency.go | 9 +- pkg/state/create.go | 9 +- pkg/state/envvals_loader.go | 2 +- pkg/state/release.go | 8 +- pkg/state/state.go | 31 +++---- pkg/state/state_exec_tmpl.go | 3 +- pkg/state/state_exec_tmpl_test.go | 4 +- pkg/tmpl/context_funcs.go | 5 +- pkg/yaml/yaml.go | 84 ++++++++++++++++++ .../yamlutil_test.go => yaml/yaml_test.go} | 4 +- test/e2e/template/helmfile/snapshot_test.go | 2 +- 22 files changed, 224 insertions(+), 83 deletions(-) delete mode 100644 pkg/maputil/yamlutil.go create mode 100644 pkg/yaml/yaml.go rename pkg/{maputil/yamlutil_test.go => yaml/yaml_test.go} (92%) diff --git a/go.mod b/go.mod index 7782932c..092aad8c 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a github.com/davecgh/go-spew v1.1.1 github.com/go-test/deep v1.1.0 + github.com/goccy/go-yaml v1.9.8 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 github.com/gosuri/uitable v0.0.4 @@ -27,7 +28,7 @@ require ( go.uber.org/zap v1.24.0 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 golang.org/x/term v0.3.0 - gopkg.in/yaml.v3 v3.0.1 + gopkg.in/yaml.v2 v2.4.0 helm.sh/helm/v3 v3.10.3 k8s.io/apimachinery v0.26.0 ) @@ -163,7 +164,6 @@ require ( github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.5 // indirect github.com/go-openapi/swag v0.19.14 // indirect - github.com/goccy/go-yaml v1.9.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect @@ -206,7 +206,7 @@ require ( go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect golang.org/x/crypto v0.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.25.2 // indirect k8s.io/cli-runtime v0.25.2 // indirect k8s.io/client-go v0.25.2 // indirect diff --git a/go.sum b/go.sum index f53be255..ded9af78 100644 --- a/go.sum +++ b/go.sum @@ -322,15 +322,18 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0= -github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= +github.com/goccy/go-yaml v1.9.8 h1:5gMyLUeU1/6zl+WFfR1hN7D2kf+1/eRGa7DFtToiBvQ= +github.com/goccy/go-yaml v1.9.8/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -582,6 +585,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= @@ -1064,6 +1068,7 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/app/app_list_test.go b/pkg/app/app_list_test.go index 5fc5ebaf..0e4de724 100644 --- a/pkg/app/app_list_test.go +++ b/pkg/app/app_list_test.go @@ -242,7 +242,7 @@ environments: releases: - name: myrelease1 chart: mychart1 - installed: no + installed: false labels: id: myrelease1 - name: myrelease2 @@ -253,7 +253,7 @@ releases: releases: - name: myrelease3 chart: mychart1 - installed: yes + installed: true - name: myrelease4 chart: mychart1 labels: diff --git a/pkg/app/app_template_test.go b/pkg/app/app_template_test.go index 83a7a3a4..921cc4c7 100644 --- a/pkg/app/app_template_test.go +++ b/pkg/app/app_template_test.go @@ -16,6 +16,7 @@ import ( ffs "github.com/helmfile/helmfile/pkg/filesystem" "github.com/helmfile/helmfile/pkg/helmexec" "github.com/helmfile/helmfile/pkg/testhelper" + "github.com/helmfile/helmfile/pkg/yaml" ) func TestTemplate(t *testing.T) { @@ -306,3 +307,88 @@ releases: }) }) } + +func TestTemplate_StrictParsing(t *testing.T) { + v := yaml.GoccyGoYaml + yaml.GoccyGoYaml = true + t.Cleanup(func() { + yaml.GoccyGoYaml = v + }) + + type testcase struct { + ns string + error string + } + + check := func(t *testing.T, tc testcase) { + t.Helper() + + var helm = &exectest.Helm{ + FailOnUnexpectedList: true, + FailOnUnexpectedDiff: true, + DiffMutex: &sync.Mutex{}, + ChartsMutex: &sync.Mutex{}, + ReleasesMutex: &sync.Mutex{}, + } + + _ = runWithLogCapture(t, "debug", func(t *testing.T, logger *zap.SugaredLogger) { + t.Helper() + + valsRuntime, err := vals.New(vals.Options{CacheSize: 32}) + if err != nil { + t.Errorf("unexpected error creating vals runtime: %v", err) + } + + files := map[string]string{ + "/path/to/helmfile.yaml": ` +releases: +- name: app1 + foobar: FOOBAR + chart: incubator/raw +`, + } + + app := appWithFs(&App{ + OverrideHelmBinary: DefaultHelmBinary, + fs: &ffs.FileSystem{Glob: filepath.Glob}, + OverrideKubeContext: "default", + Env: "default", + Logger: logger, + helms: map[helmKey]helmexec.Interface{ + createHelmKey("helm", "default"): helm, + }, + valsRuntime: valsRuntime, + }, files) + + if tc.ns != "" { + app.Namespace = tc.ns + } + + tmplErr := app.Template(applyConfig{ + // if we check log output, concurrency must be 1. otherwise the test becomes non-deterministic. + concurrency: 1, + logger: logger, + }) + + var gotErr string + if tmplErr != nil { + gotErr = tmplErr.Error() + } + + if d := cmp.Diff(tc.error, gotErr); d != "" { + t.Fatalf("unexpected error: want (-), got (+): %s", d) + } + }) + } + + t.Run("fail due to known field", func(t *testing.T) { + check(t, testcase{ + error: `in ./helmfile.yaml: failed to read helmfile.yaml: reading document at index 1: [4:3] unknown field "foobar" + 2 | releases: + 3 | - name: app1 + > 4 | foobar: FOOBAR + ^ + 5 | chart: incubator/raw`, + }) + }) +} diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 5732425f..36a28c42 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -4288,7 +4288,7 @@ environments: releases: - name: myrelease1 chart: mychart1 - installed: no + installed: false labels: id: myrelease1 - name: myrelease2 @@ -4299,7 +4299,7 @@ releases: releases: - name: myrelease3 chart: mychart1 - installed: yes + installed: true - name: myrelease4 chart: mychart1 labels: diff --git a/pkg/app/load_opts.go b/pkg/app/load_opts.go index cd2726db..f321237b 100644 --- a/pkg/app/load_opts.go +++ b/pkg/app/load_opts.go @@ -1,10 +1,8 @@ package app import ( - "gopkg.in/yaml.v3" - - "github.com/helmfile/helmfile/pkg/maputil" "github.com/helmfile/helmfile/pkg/state" + "github.com/helmfile/helmfile/pkg/yaml" ) type LoadOpts struct { @@ -22,7 +20,7 @@ type LoadOpts struct { } func (o LoadOpts) DeepCopy() LoadOpts { - bytes, err := maputil.YamlMarshal(o) + bytes, err := yaml.Marshal(o) if err != nil { panic(err) } diff --git a/pkg/app/two_pass_renderer_test.go b/pkg/app/two_pass_renderer_test.go index 2d0e89c9..98d6f951 100644 --- a/pkg/app/two_pass_renderer_test.go +++ b/pkg/app/two_pass_renderer_test.go @@ -4,11 +4,10 @@ import ( "strings" "testing" - "gopkg.in/yaml.v3" - "github.com/helmfile/helmfile/pkg/remote" "github.com/helmfile/helmfile/pkg/state" "github.com/helmfile/helmfile/pkg/testhelper" + "github.com/helmfile/helmfile/pkg/yaml" ) // nolint: unparam diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go index 6bc76080..58f02f28 100644 --- a/pkg/environment/environment.go +++ b/pkg/environment/environment.go @@ -2,9 +2,9 @@ package environment import ( "github.com/imdario/mergo" - "gopkg.in/yaml.v3" "github.com/helmfile/helmfile/pkg/maputil" + "github.com/helmfile/helmfile/pkg/yaml" ) type Environment struct { @@ -16,7 +16,7 @@ type Environment struct { var EmptyEnvironment Environment func (e Environment) DeepCopy() Environment { - valuesBytes, err := maputil.YamlMarshal(e.Values) + valuesBytes, err := yaml.Marshal(e.Values) if err != nil { panic(err) } @@ -29,7 +29,7 @@ func (e Environment) DeepCopy() Environment { panic(err) } - defaultsBytes, err := maputil.YamlMarshal(e.Defaults) + defaultsBytes, err := yaml.Marshal(e.Defaults) if err != nil { panic(err) } diff --git a/pkg/helmexec/exec.go b/pkg/helmexec/exec.go index 41daa610..328ef392 100644 --- a/pkg/helmexec/exec.go +++ b/pkg/helmexec/exec.go @@ -14,12 +14,12 @@ import ( "github.com/Masterminds/semver/v3" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "gopkg.in/yaml.v3" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/plugin" "github.com/helmfile/helmfile/pkg/envvar" + "github.com/helmfile/helmfile/pkg/yaml" ) type decryptedSecret struct { diff --git a/pkg/maputil/yamlutil.go b/pkg/maputil/yamlutil.go deleted file mode 100644 index a2e15c7b..00000000 --- a/pkg/maputil/yamlutil.go +++ /dev/null @@ -1,18 +0,0 @@ -package maputil - -import ( - "bytes" - - "gopkg.in/yaml.v3" -) - -func YamlMarshal(v interface{}) ([]byte, error) { - var b bytes.Buffer - yamlEncoder := yaml.NewEncoder(&b) - yamlEncoder.SetIndent(2) - err := yamlEncoder.Encode(v) - defer func() { - _ = yamlEncoder.Close() - }() - return b.Bytes(), err -} diff --git a/pkg/remote/remote.go b/pkg/remote/remote.go index 563f47db..72ace9bb 100644 --- a/pkg/remote/remote.go +++ b/pkg/remote/remote.go @@ -14,10 +14,10 @@ import ( "github.com/hashicorp/go-getter/helper/url" "go.uber.org/multierr" "go.uber.org/zap" - "gopkg.in/yaml.v3" "github.com/helmfile/helmfile/pkg/envvar" "github.com/helmfile/helmfile/pkg/filesystem" + "github.com/helmfile/helmfile/pkg/yaml" ) var disableInsecureFeatures bool diff --git a/pkg/state/chart_dependency.go b/pkg/state/chart_dependency.go index a8b22a60..72872c6b 100644 --- a/pkg/state/chart_dependency.go +++ b/pkg/state/chart_dependency.go @@ -11,11 +11,10 @@ import ( goversion "github.com/hashicorp/go-version" "github.com/r3labs/diff" "go.uber.org/zap" - "gopkg.in/yaml.v3" "github.com/helmfile/helmfile/pkg/app/version" "github.com/helmfile/helmfile/pkg/helmexec" - "github.com/helmfile/helmfile/pkg/maputil" + "github.com/helmfile/helmfile/pkg/yaml" ) type ChartMeta struct { @@ -309,7 +308,7 @@ func (m *chartDependencyManager) updateHelm3(shell helmexec.DependencyUpdater, w chartMetaContent := fmt.Sprintf("name: %s\nversion: 1.0.0\napiVersion: v2\n", m.Name) // Generate `requirements.yaml` of the temporary local chart from the helmfile state - reqsContent, err := maputil.YamlMarshal(unresolved.ToChartRequirements()) + reqsContent, err := yaml.Marshal(unresolved.ToChartRequirements()) if err != nil { return nil, err } @@ -327,7 +326,7 @@ func (m *chartDependencyManager) updateHelm2(shell helmexec.DependencyUpdater, w } // Generate `requirements.yaml` of the temporary local chart from the helmfile state - reqsContent, err := maputil.YamlMarshal(unresolved.ToChartRequirements()) + reqsContent, err := yaml.Marshal(unresolved.ToChartRequirements()) if err != nil { return nil, err } @@ -393,7 +392,7 @@ func (m *chartDependencyManager) doUpdate(chartLockFile string, unresolved *Unre lockedReqs.Version = version.Version() - updatedLockFileContent, err = maputil.YamlMarshal(lockedReqs) + updatedLockFileContent, err = yaml.Marshal(lockedReqs) if err != nil { return nil, err diff --git a/pkg/state/create.go b/pkg/state/create.go index 27fc2fda..0654f684 100644 --- a/pkg/state/create.go +++ b/pkg/state/create.go @@ -1,7 +1,6 @@ package state import ( - "bytes" "errors" "fmt" "io" @@ -9,13 +8,13 @@ import ( "github.com/imdario/mergo" "github.com/variantdev/vals" "go.uber.org/zap" - "gopkg.in/yaml.v3" "github.com/helmfile/helmfile/pkg/environment" "github.com/helmfile/helmfile/pkg/filesystem" "github.com/helmfile/helmfile/pkg/helmexec" "github.com/helmfile/helmfile/pkg/maputil" "github.com/helmfile/helmfile/pkg/remote" + "github.com/helmfile/helmfile/pkg/yaml" ) const ( @@ -89,9 +88,7 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState, state.LockFile = c.lockFile - decoder := yaml.NewDecoder(bytes.NewReader(content)) - - decoder.KnownFields(c.Strict) + decode := yaml.NewDecoder(content, c.Strict) i := 0 for { @@ -99,7 +96,7 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState, var intermediate HelmState - err := decoder.Decode(&intermediate) + err := decode(&intermediate) if err == io.EOF { break } else if err != nil { diff --git a/pkg/state/envvals_loader.go b/pkg/state/envvals_loader.go index 932ad4f2..4eb07498 100644 --- a/pkg/state/envvals_loader.go +++ b/pkg/state/envvals_loader.go @@ -6,13 +6,13 @@ import ( "github.com/imdario/mergo" "go.uber.org/zap" - "gopkg.in/yaml.v3" "github.com/helmfile/helmfile/pkg/environment" "github.com/helmfile/helmfile/pkg/filesystem" "github.com/helmfile/helmfile/pkg/maputil" "github.com/helmfile/helmfile/pkg/remote" "github.com/helmfile/helmfile/pkg/tmpl" + "github.com/helmfile/helmfile/pkg/yaml" ) type EnvironmentValuesLoader struct { diff --git a/pkg/state/release.go b/pkg/state/release.go index f4afdb79..f1391bbe 100644 --- a/pkg/state/release.go +++ b/pkg/state/release.go @@ -3,10 +3,8 @@ package state import ( "fmt" - "gopkg.in/yaml.v3" - - "github.com/helmfile/helmfile/pkg/maputil" "github.com/helmfile/helmfile/pkg/tmpl" + "github.com/helmfile/helmfile/pkg/yaml" ) func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*ReleaseSpec, error) { @@ -99,7 +97,7 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R for i, t := range result.ValuesTemplate { switch ts := t.(type) { case map[interface{}]interface{}, map[string]interface{}: - serialized, err := maputil.YamlMarshal(ts) + serialized, err := yaml.Marshal(ts) if err != nil { return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err) } @@ -202,7 +200,7 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R } func (r ReleaseSpec) Clone() (*ReleaseSpec, error) { - serialized, err := maputil.YamlMarshal(r) + serialized, err := yaml.Marshal(r) if err != nil { return nil, fmt.Errorf("failed cloning release \"%s\": %v", r.Name, err) } diff --git a/pkg/state/state.go b/pkg/state/state.go index 39842a03..c2e90801 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -22,16 +22,15 @@ import ( "github.com/variantdev/chartify" "github.com/variantdev/vals" "go.uber.org/zap" - "gopkg.in/yaml.v3" "github.com/helmfile/helmfile/pkg/environment" "github.com/helmfile/helmfile/pkg/event" "github.com/helmfile/helmfile/pkg/filesystem" "github.com/helmfile/helmfile/pkg/helmexec" - "github.com/helmfile/helmfile/pkg/maputil" "github.com/helmfile/helmfile/pkg/policy" "github.com/helmfile/helmfile/pkg/remote" "github.com/helmfile/helmfile/pkg/tmpl" + "github.com/helmfile/helmfile/pkg/yaml" ) const ( @@ -87,13 +86,9 @@ type ReleaseSetSpec struct { // helmStateAlias is helm state alias type helmStateAlias HelmState -func (hs HelmState) MarshalYAML() (interface{}, error) { - return helmStateAlias(hs), nil -} - -func (hs *HelmState) UnmarshalYAML(value *yaml.Node) error { +func (hs *HelmState) UnmarshalYAML(unmarshal func(interface{}) error) error { helmStateInfo := make(map[string]interface{}) - if err := value.DecodeWithOptions(&helmStateInfo, yaml.DecodeOptions{KnownFields: true}); err != nil { + if err := unmarshal(&helmStateInfo); err != nil { return err } @@ -105,7 +100,7 @@ func (hs *HelmState) UnmarshalYAML(value *yaml.Node) error { fmt.Fprintf(os.Stderr, "Warning: %v\n", err) } - return value.DecodeWithOptions((*helmStateAlias)(hs), yaml.DecodeOptions{KnownFields: true}) + return unmarshal((*helmStateAlias)(hs)) } // HelmState structure for the helmfile @@ -1552,8 +1547,8 @@ func (st *HelmState) WriteReleasesValues(helm helmexec.Interface, additionalValu var buf bytes.Buffer - y := yaml.NewEncoder(&buf) - if err := y.Encode(merged); err != nil { + encoder := yaml.NewEncoder(&buf) + if err := encoder.Encode(merged); err != nil { return []error{err} } @@ -2708,7 +2703,7 @@ func (st *HelmState) RenderReleaseValuesFileToBytes(release *ReleaseSpec, path s return nil, err } - return maputil.YamlMarshal(parsedYaml) + return yaml.Marshal(parsedYaml) } return rawBytes, nil @@ -2896,7 +2891,7 @@ func (st *HelmState) generateSecretValuesFiles(helm helmexec.Interface, release return nil, err } default: - bs, err := maputil.YamlMarshal(value) + bs, err := yaml.Marshal(value) if err != nil { return nil, err } @@ -3127,10 +3122,10 @@ func (p SubHelmfileSpec) MarshalYAML() (interface{}, error) { } // UnmarshalYAML will unmarshal the helmfile yaml section and fill the SubHelmfileSpec structure -// this is required to keep allowing string scalar for defining helmfile -func (hf *SubHelmfileSpec) UnmarshalYAML(value *yaml.Node) error { +// this is required go-yto keep allowing string scalar for defining helmfile +func (hf *SubHelmfileSpec) UnmarshalYAML(unmarshal func(interface{}) error) error { var tmp interface{} - if err := value.Decode(&tmp); err != nil { + if err := unmarshal(&tmp); err != nil { return err } @@ -3145,7 +3140,7 @@ func (hf *SubHelmfileSpec) UnmarshalYAML(value *yaml.Node) error { Environment SubhelmfileEnvironmentSpec `yaml:",inline"` } - if err := value.Decode(&subHelmfileSpecTmp); err != nil { + if err := unmarshal(&subHelmfileSpecTmp); err != nil { return err } hf.Path = subHelmfileSpecTmp.Path @@ -3330,7 +3325,7 @@ func (st *HelmState) GenerateOutputFilePath(release *ReleaseSpec, outputFileTemp } func (st *HelmState) ToYaml() (string, error) { - if result, err := maputil.YamlMarshal(st); err != nil { + if result, err := yaml.Marshal(st); err != nil { return "", err } else { return string(result), nil diff --git a/pkg/state/state_exec_tmpl.go b/pkg/state/state_exec_tmpl.go index f53906bd..e09e7ca4 100644 --- a/pkg/state/state_exec_tmpl.go +++ b/pkg/state/state_exec_tmpl.go @@ -4,9 +4,8 @@ import ( "fmt" "reflect" - "gopkg.in/yaml.v3" - "github.com/helmfile/helmfile/pkg/tmpl" + "github.com/helmfile/helmfile/pkg/yaml" ) func (st *HelmState) Values() map[string]interface{} { diff --git a/pkg/state/state_exec_tmpl_test.go b/pkg/state/state_exec_tmpl_test.go index 5ce5c0cd..91b7945e 100644 --- a/pkg/state/state_exec_tmpl_test.go +++ b/pkg/state/state_exec_tmpl_test.go @@ -67,11 +67,11 @@ func TestHelmState_executeTemplates(t *testing.T) { Name: "app-dev", Namespace: "dev", Labels: map[string]string{"id": "app"}, - InstalledTemplate: func(i string) *string { return &i }(`{{ eq .Release.Labels.id "app" | ternary "yes" "no" }}`), + InstalledTemplate: func(i string) *string { return &i }(`{{ eq .Release.Labels.id "app" | ternary "true" "false" }}`), VerifyTemplate: func(i string) *string { return &i }(`{{ true }}`), Verify: func(i bool) *bool { return &i }(false), WaitTemplate: func(i string) *string { return &i }(`{{ false }}`), - TillerlessTemplate: func(i string) *string { return &i }(`yes`), + TillerlessTemplate: func(i string) *string { return &i }(`true`), }, want: ReleaseSpec{ Chart: "test-chart", diff --git a/pkg/tmpl/context_funcs.go b/pkg/tmpl/context_funcs.go index 5e09b88e..9b671d25 100644 --- a/pkg/tmpl/context_funcs.go +++ b/pkg/tmpl/context_funcs.go @@ -14,11 +14,10 @@ import ( "text/template" "golang.org/x/sync/errgroup" - "gopkg.in/yaml.v3" "github.com/helmfile/helmfile/pkg/envvar" "github.com/helmfile/helmfile/pkg/helmexec" - "github.com/helmfile/helmfile/pkg/maputil" + "github.com/helmfile/helmfile/pkg/yaml" ) type Values = map[string]interface{} @@ -279,7 +278,7 @@ func (c *Context) Tpl(text string, data interface{}) (string, error) { } func ToYaml(v interface{}) (string, error) { - data, err := maputil.YamlMarshal(v) + data, err := yaml.Marshal(v) if err != nil { return "", err } diff --git a/pkg/yaml/yaml.go b/pkg/yaml/yaml.go new file mode 100644 index 00000000..5c88134f --- /dev/null +++ b/pkg/yaml/yaml.go @@ -0,0 +1,84 @@ +package yaml + +import ( + "bytes" + "io" + + "github.com/goccy/go-yaml" + v2 "gopkg.in/yaml.v2" +) + +var ( + // We'll derive the default from the build once + // is merged + GoccyGoYaml bool = true +) + +type Encoder interface { + Encode(interface{}) error + Close() error +} + +// NewEncoder creates and returns a function that is used to encode a Go object to a YAML document +func NewEncoder(w io.Writer) Encoder { + if GoccyGoYaml { + return yaml.NewEncoder(w) + } + + return v2.NewEncoder(w) +} + +func Unmarshal(data []byte, v interface{}) error { + if GoccyGoYaml { + return yaml.Unmarshal(data, v) + } + + return v2.Unmarshal(data, v) +} + +// NewDecoder creates and returns a function that is used to decode a YAML document +// contained within the YAML document stream per each call. +// When strict is true, this function ensures that every field found in the YAML document +// to have the corresponding field in the decoded Go struct. +func NewDecoder(data []byte, strict bool) func(interface{}) error { + if GoccyGoYaml { + var opts []yaml.DecodeOption + if strict { + opts = append(opts, yaml.DisallowUnknownField()) + } + + decoder := yaml.NewDecoder( + bytes.NewReader(data), + opts..., + ) + + return func(v interface{}) error { + return decoder.Decode(v) + } + } + + decoder := v2.NewDecoder(bytes.NewReader(data)) + decoder.SetStrict(strict) + + return func(v interface{}) error { + return decoder.Decode(v) + } +} + +func Marshal(v interface{}) ([]byte, error) { + if GoccyGoYaml { + var b bytes.Buffer + yamlEncoder := yaml.NewEncoder( + &b, + yaml.IndentSequence(true), + yaml.Indent(2), + ) + err := yamlEncoder.Encode(v) + defer func() { + _ = yamlEncoder.Close() + }() + return b.Bytes(), err + } + + return v2.Marshal(v) +} diff --git a/pkg/maputil/yamlutil_test.go b/pkg/yaml/yaml_test.go similarity index 92% rename from pkg/maputil/yamlutil_test.go rename to pkg/yaml/yaml_test.go index 70377a3c..758a8bc7 100644 --- a/pkg/maputil/yamlutil_test.go +++ b/pkg/yaml/yaml_test.go @@ -1,4 +1,4 @@ -package maputil +package yaml import ( "testing" @@ -27,7 +27,7 @@ func TestYamlMarshal(t *testing.T) { } for _, tt := range tests { - actual, err := YamlMarshal(tt) + actual, err := Marshal(tt) require.NoError(t, err) require.Equal(t, tt.expected, string(actual)) } diff --git a/test/e2e/template/helmfile/snapshot_test.go b/test/e2e/template/helmfile/snapshot_test.go index e1a8e3d6..03d911cd 100644 --- a/test/e2e/template/helmfile/snapshot_test.go +++ b/test/e2e/template/helmfile/snapshot_test.go @@ -15,11 +15,11 @@ import ( "github.com/stretchr/testify/require" "github.com/variantdev/chartify/helmtesting" - "gopkg.in/yaml.v3" "github.com/helmfile/helmfile/pkg/app" "github.com/helmfile/helmfile/pkg/envvar" "github.com/helmfile/helmfile/pkg/helmexec" + "github.com/helmfile/helmfile/pkg/yaml" ) var (