From 9a88372449962ae22a046620917093585ab0a1dc Mon Sep 17 00:00:00 2001 From: Jess <639053+jess-sol@users.noreply.github.com> Date: Wed, 30 Jul 2025 23:38:36 -0600 Subject: [PATCH 1/2] Allow caching of remote files to be disabled (#2112) * Allow caching of remote files to be disabled Make it possible to automatically update the cache of remote resources by disabling the caching of those resources using a query string parameter (`cache=false`). Signed-off-by: Jess * Fix test that broke Because query parameters are being re-encoded, = is being encoded to %3D. Signed-off-by: Jess * Add test for disabling caching of remote resources Signed-off-by: Jess * Include example usage in docs Signed-off-by: Jess --------- Signed-off-by: Jess --- docs/index.md | 3 ++ pkg/remote/remote.go | 12 ++++--- pkg/remote/remote_test.go | 69 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/docs/index.md b/docs/index.md index ec7ee6ff..348bf3aa 100644 --- a/docs/index.md +++ b/docs/index.md @@ -409,6 +409,9 @@ helmfiles: # The nested-state file is locally checked-out along with the remote directory containing it. # Therefore all the local paths in the file are resolved relative to the file path: git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0 +- # By default git repositories aren't updated unless the ref is updated. + # Alternatively, refer to a named ref and disable the caching. + path: git::ssh://git@github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=main&cache=false # If set to "Error", return an error when a subhelmfile points to a # non-existent path. The default behavior is to print a warning and continue. missingFileHandler: Error diff --git a/pkg/remote/remote.go b/pkg/remote/remote.go index c0845f23..8b6006f6 100644 --- a/pkg/remote/remote.go +++ b/pkg/remote/remote.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "maps" "net/http" neturl "net/url" "os" @@ -213,13 +214,16 @@ func (r *Remote) Fetch(path string, cacheDirOpt ...string) (string, error) { return "", fmt.Errorf("[bug] cacheDirOpt's length: want 0 or 1, got %d", len(cacheDirOpt)) } - query := u.RawQuery + query, _ := neturl.ParseQuery(u.RawQuery) + + should_cache := query.Get("cache") != "false" + delete(query, "cache") var cacheKey string replacer := strings.NewReplacer(":", "", "//", "_", "/", "_", ".", "_") dirKey := replacer.Replace(srcDir) if len(query) > 0 { - q, _ := neturl.ParseQuery(query) + q := maps.Clone(query) if q.Has("sshkey") { q.Set("sshkey", "redacted") } @@ -262,7 +266,7 @@ func (r *Remote) Fetch(path string, cacheDirOpt ...string) (string, error) { } } - if !cached { + if !cached || !should_cache { var getterSrc string if u.User != "" { getterSrc = fmt.Sprintf("%s://%s@%s%s", u.Scheme, u.User, u.Host, u.Dir) @@ -271,7 +275,7 @@ func (r *Remote) Fetch(path string, cacheDirOpt ...string) (string, error) { } if len(query) > 0 { - getterSrc = strings.Join([]string{getterSrc, query}, "?") + getterSrc = strings.Join([]string{getterSrc, query.Encode()}, "?") } r.Logger.Debugf("remote> downloading %s to %s", getterSrc, getterDst) diff --git a/pkg/remote/remote_test.go b/pkg/remote/remote_test.go index c91627df..a983acca 100644 --- a/pkg/remote/remote_test.go +++ b/pkg/remote/remote_test.go @@ -179,7 +179,7 @@ func TestRemote_SShGitHub_WithSshKey(t *testing.T) { if wd != CacheDir() { return fmt.Errorf("unexpected wd: %s", wd) } - if src != "git::ssh://git@github.com/helmfile/helmfiles.git?ref=0.40.0&sshkey=ZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkJTU3dOY2xoVzQ2Vm9VR3dMQ3JscVRHYUdOVWdRVUVEUEptc1ZzdUViL2RBNUcrQk9YMWxGaUVMYU9HQ2F6bS9KQkR2V3Y2Y0ZDQUtVRjVocVJOUjdJPSA=" { + if src != "git::ssh://git@github.com/helmfile/helmfiles.git?ref=0.40.0&sshkey=ZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkJTU3dOY2xoVzQ2Vm9VR3dMQ3JscVRHYUdOVWdRVUVEUEptc1ZzdUViL2RBNUcrQk9YMWxGaUVMYU9HQ2F6bS9KQkR2V3Y2Y0ZDQUtVRjVocVJOUjdJPSA%3D" { return fmt.Errorf("unexpected src: %s", src) } @@ -219,6 +219,73 @@ func TestRemote_SShGitHub_WithSshKey(t *testing.T) { } } +func TestRemote_SShGitHub_WithDisableCacheKey(t *testing.T) { + cleanfs := map[string]string{ + CacheDir(): "", + } + cachefs := map[string]string{ + filepath.Join(CacheDir(), "ssh_github_com_helmfile_helmfiles_git.ref=main/releases/kiam.yaml"): "foo: bar", + } + + testcases := []struct { + name string + files map[string]string + expectCacheHit bool + }{ + {name: "not expectCacheHit", files: cleanfs, expectCacheHit: false}, + {name: "forceNoCacheHit", files: cachefs, expectCacheHit: false}, + } + + for _, tt := range testcases { + t.Run(tt.name, func(t *testing.T) { + testfs := testhelper.NewTestFs(tt.files) + + hit := true + + get := func(wd, src, dst string) error { + if wd != CacheDir() { + return fmt.Errorf("unexpected wd: %s", wd) + } + if src != "git::ssh://git@github.com/helmfile/helmfiles.git?ref=main" { + return fmt.Errorf("unexpected src: %s", src) + } + + hit = false + + return nil + } + + getter := &testGetter{ + get: get, + } + remote := &Remote{ + Logger: helmexec.NewLogger(io.Discard, "debug"), + Home: CacheDir(), + Getter: getter, + fs: testfs.ToFileSystem(), + } + + url := "git::ssh://git@github.com/helmfile/helmfiles.git@releases/kiam.yaml?ref=main&cache=false" + file, err := remote.Locate(url) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + expectedFile := filepath.Join(CacheDir(), "ssh_github_com_helmfile_helmfiles_git.ref=main/releases/kiam.yaml") + if file != expectedFile { + t.Errorf("unexpected file located: %s vs expected: %s", file, expectedFile) + } + + if tt.expectCacheHit && !hit { + t.Errorf("unexpected result: unexpected cache miss") + } + if !tt.expectCacheHit && hit { + t.Errorf("unexpected result: unexpected cache hit") + } + }) + } +} + func TestRemote_S3(t *testing.T) { cleanfs := map[string]string{ CacheDir(): "", From 959aae579144213df9a5a54b27a32dda9513d7b0 Mon Sep 17 00:00:00 2001 From: yxxhero <11087727+yxxhero@users.noreply.github.com> Date: Fri, 1 Aug 2025 11:28:39 +0800 Subject: [PATCH 2/2] refactor(yaml): switch yaml library import paths from gopkg.in to go.yaml.in (#2114) Signed-off-by: yxxhero --- docs/index.md | 4 ++-- go.mod | 6 ++++-- go.sum | 4 ++++ pkg/app/app_template_test.go | 4 ++-- pkg/app/app_test.go | 8 ++++---- pkg/runtime/runtime.go | 6 +++--- pkg/tmpl/context_funcs_test.go | 4 ++-- pkg/yaml/yaml.go | 4 ++-- pkg/yaml/yaml_test.go | 12 ++++++------ test/e2e/template/helmfile/snapshot_test.go | 4 ++-- 10 files changed, 31 insertions(+), 25 deletions(-) diff --git a/docs/index.md b/docs/index.md index 348bf3aa..21099f11 100644 --- a/docs/index.md +++ b/docs/index.md @@ -574,7 +574,7 @@ Helmfile uses some OS environment variables to override default behaviour: * `HELMFILE_ENVIRONMENT` - specify [Helmfile environment](https://helmfile.readthedocs.io/en/latest/#environment), it has lower priority than CLI argument `--environment` * `HELMFILE_TEMPDIR` - specify directory to store temporary files * `HELMFILE_UPGRADE_NOTICE_DISABLED` - expecting any non-empty value to skip the check for the latest version of Helmfile in [helmfile version](https://helmfile.readthedocs.io/en/latest/#version) -* `HELMFILE_GO_YAML_V3` - use *gopkg.in/yaml.v3* instead of *gopkg.in/yaml.v2*. It's `false` by default in Helmfile v0.x, and `true` in Helmfile v1.x. +* `HELMFILE_GO_YAML_V3` - use *go.yaml.in/yaml/v3* instead of *go.yaml.in/yaml/v2*. It's `false` by default in Helmfile v0.x, and `true` in Helmfile v1.x. * `HELMFILE_CACHE_HOME` - specify directory to store cached files for remote operations * `HELMFILE_FILE_PATH` - specify the path to the helmfile.yaml file * `HELMFILE_INTERACTIVE` - enable interactive mode, expecting `true` lower case. The same as `--interactive` CLI flag @@ -584,7 +584,7 @@ Helmfile uses some OS environment variables to override default behaviour: ``` Declaratively deploy your Kubernetes manifests, Kustomize configs, and Charts as Helm releases in one shot V1 mode = false -YAML library = gopkg.in/yaml.v3 +YAML library = go.yaml.in/yaml/v3 Usage: helmfile [command] diff --git a/go.mod b/go.mod index 635dd888..9a4d1795 100644 --- a/go.mod +++ b/go.mod @@ -25,10 +25,10 @@ require ( github.com/zclconf/go-cty-yaml v1.1.0 go.szostok.io/version v1.2.0 go.uber.org/zap v1.27.0 + go.yaml.in/yaml/v2 v2.4.2 + go.yaml.in/yaml/v3 v3.0.4 golang.org/x/sync v0.16.0 golang.org/x/term v0.33.0 - gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.1 helm.sh/helm/v3 v3.18.4 k8s.io/apimachinery v0.33.3 ) @@ -294,6 +294,8 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/gookit/color.v1 v1.1.6 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.33.2 // indirect k8s.io/apiextensions-apiserver v0.33.2 // indirect k8s.io/cli-runtime v0.33.2 // indirect diff --git a/go.sum b/go.sum index 9c8d4588..3f5db0f9 100644 --- a/go.sum +++ b/go.sum @@ -1577,6 +1577,10 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/pkg/app/app_template_test.go b/pkg/app/app_template_test.go index 28311a94..362021f5 100644 --- a/pkg/app/app_template_test.go +++ b/pkg/app/app_template_test.go @@ -411,7 +411,7 @@ releases: }) } - t.Run("fail due to unknown field with gopkg.in/yaml.v3", func(t *testing.T) { + t.Run("fail due to unknown field with go.yaml.in/yaml/v3", func(t *testing.T) { check(t, testcase{ GoYamlV3: true, error: `in ./helmfile.yaml: failed to read helmfile.yaml: reading document at index 1. Started seeing this since Helmfile v1? Add the .gotmpl file extension: yaml: unmarshal errors: @@ -419,7 +419,7 @@ releases: }) }) - t.Run("fail due to unknown field with gopkg.in/yaml.v2", func(t *testing.T) { + t.Run("fail due to unknown field with go.yaml.in/yaml/v2", func(t *testing.T) { check(t, testcase{ GoYamlV3: false, error: `in ./helmfile.yaml: failed to read helmfile.yaml: reading document at index 1. Started seeing this since Helmfile v1? Add the .gotmpl file extension: yaml: unmarshal errors: diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 9494c2ad..fde002c3 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -4161,21 +4161,21 @@ releases: } func TestSetValuesTemplate(t *testing.T) { - t.Run("with gopkg.in/yaml.v3", func(t *testing.T) { + t.Run("with go.yaml.in/yaml/v3", func(t *testing.T) { testSetValuesTemplate(t, true) }) - t.Run("with gopkg.in/yaml.v2", func(t *testing.T) { + t.Run("with go.yaml.in/yaml/v2", func(t *testing.T) { testSetValuesTemplate(t, false) }) } func TestSetStringValuesTemplate(t *testing.T) { - t.Run("with gopkg.in/yaml.v3", func(t *testing.T) { + t.Run("with go.yaml.in/yaml/v3", func(t *testing.T) { testSetStringValuesTemplate(t, true) }) - t.Run("with gopkg.in/yaml.v2", func(t *testing.T) { + t.Run("with go.yaml.in/yaml/v2", func(t *testing.T) { testSetStringValuesTemplate(t, false) }) } diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 214e046c..5d9f0ff3 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -9,15 +9,15 @@ import ( var ( // GoYamlV3 is set to true in order to let Helmfile use - // gopkg.in/yaml.v3 instead of gopkg.in/yaml.v2. + // go.yaml.in/yaml/v3 instead of go.yaml.in/yaml/v2. // It's false by default in Helmfile v0.x and true in Helmfile v1.x. GoYamlV3 bool ) func Info() string { - yamlLib := "gopkg.in/yaml.v2" + yamlLib := "go.yaml.in/yaml/v2" if GoYamlV3 { - yamlLib = "gopkg.in/yaml.v3" + yamlLib = "go.yaml.in/yaml/v3" } return fmt.Sprintf("YAML library = %v", yamlLib) diff --git a/pkg/tmpl/context_funcs_test.go b/pkg/tmpl/context_funcs_test.go index 58fdab40..a72e39a9 100644 --- a/pkg/tmpl/context_funcs_test.go +++ b/pkg/tmpl/context_funcs_test.go @@ -358,11 +358,11 @@ func testFromYaml(t *testing.T, GoYamlV3 bool) { } func TestFromYaml(t *testing.T) { - t.Run("with gopkg.in/yaml.v2", func(t *testing.T) { + t.Run("with go.yaml.in/yaml/v2", func(t *testing.T) { testFromYaml(t, true) }) - t.Run("with gopkg.in/yaml.v3", func(t *testing.T) { + t.Run("with go.yaml.in/yaml/v3", func(t *testing.T) { testFromYaml(t, false) }) } diff --git a/pkg/yaml/yaml.go b/pkg/yaml/yaml.go index ffde3622..17f820ed 100644 --- a/pkg/yaml/yaml.go +++ b/pkg/yaml/yaml.go @@ -4,8 +4,8 @@ import ( "bytes" "io" - v2 "gopkg.in/yaml.v2" - v3 "gopkg.in/yaml.v3" + v2 "go.yaml.in/yaml/v2" + v3 "go.yaml.in/yaml/v3" "github.com/helmfile/helmfile/pkg/runtime" ) diff --git a/pkg/yaml/yaml_test.go b/pkg/yaml/yaml_test.go index cce63e81..57ed6a4a 100644 --- a/pkg/yaml/yaml_test.go +++ b/pkg/yaml/yaml_test.go @@ -13,9 +13,9 @@ func testYamlMarshal(t *testing.T, GoYamlV3 bool) { var yamlLibraryName string if GoYamlV3 { - yamlLibraryName = "gopkg.in/yaml.v3" + yamlLibraryName = "go.yaml.in/yaml/v3" } else { - yamlLibraryName = "gopkg.in/yaml.v2" + yamlLibraryName = "go.yaml.in/yaml/v2" } v := runtime.GoYamlV3 @@ -49,8 +49,8 @@ func testYamlMarshal(t *testing.T, GoYamlV3 bool) { Annotation: "on", }}, expected: map[string]string{ - "gopkg.in/yaml.v2": "name: John\ninfo:\n- age: 20\n address: New York\n annotation: \"on\"\n", - "gopkg.in/yaml.v3": "name: John\ninfo:\n - age: 20\n address: New York\n annotation: \"on\"\n", + "go.yaml.in/yaml/v2": "name: John\ninfo:\n- age: 20\n address: New York\n annotation: \"on\"\n", + "go.yaml.in/yaml/v3": "name: John\ninfo:\n - age: 20\n address: New York\n annotation: \"on\"\n", }, }, } @@ -63,11 +63,11 @@ func testYamlMarshal(t *testing.T, GoYamlV3 bool) { } func TestYamlMarshal(t *testing.T) { - t.Run("with gopkg.in/yaml.v2", func(t *testing.T) { + t.Run("with go.yaml.in/yaml/v2", func(t *testing.T) { testYamlMarshal(t, true) }) - t.Run("with gopkg.in/yaml.v3", func(t *testing.T) { + t.Run("with go.yaml.in/yaml/v3", func(t *testing.T) { testYamlMarshal(t, false) }) } diff --git a/test/e2e/template/helmfile/snapshot_test.go b/test/e2e/template/helmfile/snapshot_test.go index 00184e0d..e99312f3 100644 --- a/test/e2e/template/helmfile/snapshot_test.go +++ b/test/e2e/template/helmfile/snapshot_test.go @@ -58,11 +58,11 @@ func (f fakeInit) Force() bool { } func TestHelmfileTemplateWithBuildCommand(t *testing.T) { - t.Run("with gopkg.in/yaml.v3", func(t *testing.T) { + t.Run("with go.yaml.in/yaml/v3", func(t *testing.T) { testHelmfileTemplateWithBuildCommand(t, true) }) - t.Run("with gopkg.in/yaml.v2", func(t *testing.T) { + t.Run("with go.yaml.in/yaml/v2", func(t *testing.T) { testHelmfileTemplateWithBuildCommand(t, false) }) }