Merge branch 'main' into update-strategy-feature

docs/index.md

@@ -413,6 +413,9 @@ helmfiles:
   # The nested-state file is locally checked-out along with the remote directory containing it.
   # Therefore all the local paths in the file are resolved relative to the file
   path: git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0
+- # By default git repositories aren't updated unless the ref is updated.
+  # Alternatively, refer to a named ref and disable the caching.
+  path: git::ssh://git@github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=main&cache=false
 # If set to "Error", return an error when a subhelmfile points to a
 # non-existent path. The default behavior is to print a warning and continue.
 missingFileHandler: Error
@@ -575,7 +578,7 @@ Helmfile uses some OS environment variables to override default behaviour:
 * `HELMFILE_ENVIRONMENT` - specify [Helmfile environment](https://helmfile.readthedocs.io/en/latest/#environment), it has lower priority than CLI argument `--environment`
 * `HELMFILE_TEMPDIR` - specify directory to store temporary files
 * `HELMFILE_UPGRADE_NOTICE_DISABLED` - expecting any non-empty value to skip the check for the latest version of Helmfile in [helmfile version](https://helmfile.readthedocs.io/en/latest/#version)
-* `HELMFILE_GO_YAML_V3` - use *gopkg.in/yaml.v3* instead of *gopkg.in/yaml.v2*.  It's `false` by default in Helmfile v0.x, and `true` in Helmfile v1.x.
+* `HELMFILE_GO_YAML_V3` - use *go.yaml.in/yaml/v3* instead of *go.yaml.in/yaml/v2*.  It's `false` by default in Helmfile v0.x, and `true` in Helmfile v1.x.
 * `HELMFILE_CACHE_HOME` - specify directory to store cached files for remote operations
 * `HELMFILE_FILE_PATH` - specify the path to the helmfile.yaml file
 * `HELMFILE_INTERACTIVE` - enable interactive mode, expecting `true` lower case. The same as `--interactive` CLI flag
@@ -585,7 +588,7 @@ Helmfile uses some OS environment variables to override default behaviour:
 ```
 Declaratively deploy your Kubernetes manifests, Kustomize configs, and Charts as Helm releases in one shot
 V1 mode = false
-YAML library = gopkg.in/yaml.v3
+YAML library = go.yaml.in/yaml/v3
 
 Usage:
   helmfile [command]

go.mod

@@ -25,10 +25,10 @@ require (
 	github.com/zclconf/go-cty-yaml v1.1.0
 	go.szostok.io/version v1.2.0
 	go.uber.org/zap v1.27.0
+	go.yaml.in/yaml/v2 v2.4.2
+	go.yaml.in/yaml/v3 v3.0.4
 	golang.org/x/sync v0.16.0
 	golang.org/x/term v0.33.0
-	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.18.4
 	k8s.io/apimachinery v0.33.3
 )
@@ -294,6 +294,8 @@ require (
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/gookit/color.v1 v1.1.6 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.33.2 // indirect
 	k8s.io/apiextensions-apiserver v0.33.2 // indirect
 	k8s.io/cli-runtime v0.33.2 // indirect

go.sum

@@ -1577,6 +1577,10 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

pkg/app/app_template_test.go

@@ -411,7 +411,7 @@ releases:
 		})
 	}
 
-	t.Run("fail due to unknown field with gopkg.in/yaml.v3", func(t *testing.T) {
+	t.Run("fail due to unknown field with go.yaml.in/yaml/v3", func(t *testing.T) {
 		check(t, testcase{
 			GoYamlV3: true,
 			error: `in ./helmfile.yaml: failed to read helmfile.yaml: reading document at index 1. Started seeing this since Helmfile v1? Add the .gotmpl file extension: yaml: unmarshal errors:
@@ -419,7 +419,7 @@ releases:
 		})
 	})
 
-	t.Run("fail due to unknown field with gopkg.in/yaml.v2", func(t *testing.T) {
+	t.Run("fail due to unknown field with go.yaml.in/yaml/v2", func(t *testing.T) {
 		check(t, testcase{
 			GoYamlV3: false,
 			error: `in ./helmfile.yaml: failed to read helmfile.yaml: reading document at index 1. Started seeing this since Helmfile v1? Add the .gotmpl file extension: yaml: unmarshal errors:

pkg/app/app_test.go

@@ -4352,21 +4352,21 @@ releases:
 }
 
 func TestSetValuesTemplate(t *testing.T) {
-	t.Run("with gopkg.in/yaml.v3", func(t *testing.T) {
+	t.Run("with go.yaml.in/yaml/v3", func(t *testing.T) {
 		testSetValuesTemplate(t, true)
 	})
 
-	t.Run("with gopkg.in/yaml.v2", func(t *testing.T) {
+	t.Run("with go.yaml.in/yaml/v2", func(t *testing.T) {
 		testSetValuesTemplate(t, false)
 	})
 }
 
 func TestSetStringValuesTemplate(t *testing.T) {
-	t.Run("with gopkg.in/yaml.v3", func(t *testing.T) {
+	t.Run("with go.yaml.in/yaml/v3", func(t *testing.T) {
 		testSetStringValuesTemplate(t, true)
 	})
 
-	t.Run("with gopkg.in/yaml.v2", func(t *testing.T) {
+	t.Run("with go.yaml.in/yaml/v2", func(t *testing.T) {
 		testSetStringValuesTemplate(t, false)
 	})
 }

pkg/remote/remote.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"maps"
 	"net/http"
 	neturl "net/url"
 	"os"
@@ -213,13 +214,16 @@ func (r *Remote) Fetch(path string, cacheDirOpt ...string) (string, error) {
 		return "", fmt.Errorf("[bug] cacheDirOpt's length: want 0 or 1, got %d", len(cacheDirOpt))
 	}
 
-	query := u.RawQuery
+	query, _ := neturl.ParseQuery(u.RawQuery)
+
+	should_cache := query.Get("cache") != "false"
+	delete(query, "cache")
 
 	var cacheKey string
 	replacer := strings.NewReplacer(":", "", "//", "_", "/", "_", ".", "_")
 	dirKey := replacer.Replace(srcDir)
 	if len(query) > 0 {
-		q, _ := neturl.ParseQuery(query)
+		q := maps.Clone(query)
 		if q.Has("sshkey") {
 			q.Set("sshkey", "redacted")
 		}
@@ -262,7 +266,7 @@ func (r *Remote) Fetch(path string, cacheDirOpt ...string) (string, error) {
 		}
 	}
 
-	if !cached {
+	if !cached || !should_cache {
 		var getterSrc string
 		if u.User != "" {
 			getterSrc = fmt.Sprintf("%s://%s@%s%s", u.Scheme, u.User, u.Host, u.Dir)
@@ -271,7 +275,7 @@ func (r *Remote) Fetch(path string, cacheDirOpt ...string) (string, error) {
 		}
 
 		if len(query) > 0 {
-			getterSrc = strings.Join([]string{getterSrc, query}, "?")
+			getterSrc = strings.Join([]string{getterSrc, query.Encode()}, "?")
 		}
 
 		r.Logger.Debugf("remote> downloading %s to %s", getterSrc, getterDst)

pkg/remote/remote_test.go

@@ -179,7 +179,7 @@ func TestRemote_SShGitHub_WithSshKey(t *testing.T) {
 				if wd != CacheDir() {
 					return fmt.Errorf("unexpected wd: %s", wd)
 				}
-				if src != "git::ssh://git@github.com/helmfile/helmfiles.git?ref=0.40.0&sshkey=ZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkJTU3dOY2xoVzQ2Vm9VR3dMQ3JscVRHYUdOVWdRVUVEUEptc1ZzdUViL2RBNUcrQk9YMWxGaUVMYU9HQ2F6bS9KQkR2V3Y2Y0ZDQUtVRjVocVJOUjdJPSA=" {
+				if src != "git::ssh://git@github.com/helmfile/helmfiles.git?ref=0.40.0&sshkey=ZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkJTU3dOY2xoVzQ2Vm9VR3dMQ3JscVRHYUdOVWdRVUVEUEptc1ZzdUViL2RBNUcrQk9YMWxGaUVMYU9HQ2F6bS9KQkR2V3Y2Y0ZDQUtVRjVocVJOUjdJPSA%3D" {
 					return fmt.Errorf("unexpected src: %s", src)
 				}
 
@@ -219,6 +219,73 @@ func TestRemote_SShGitHub_WithSshKey(t *testing.T) {
 	}
 }
 
+func TestRemote_SShGitHub_WithDisableCacheKey(t *testing.T) {
+	cleanfs := map[string]string{
+		CacheDir(): "",
+	}
+	cachefs := map[string]string{
+		filepath.Join(CacheDir(), "ssh_github_com_helmfile_helmfiles_git.ref=main/releases/kiam.yaml"): "foo: bar",
+	}
+
+	testcases := []struct {
+		name           string
+		files          map[string]string
+		expectCacheHit bool
+	}{
+		{name: "not expectCacheHit", files: cleanfs, expectCacheHit: false},
+		{name: "forceNoCacheHit", files: cachefs, expectCacheHit: false},
+	}
+
+	for _, tt := range testcases {
+		t.Run(tt.name, func(t *testing.T) {
+			testfs := testhelper.NewTestFs(tt.files)
+
+			hit := true
+
+			get := func(wd, src, dst string) error {
+				if wd != CacheDir() {
+					return fmt.Errorf("unexpected wd: %s", wd)
+				}
+				if src != "git::ssh://git@github.com/helmfile/helmfiles.git?ref=main" {
+					return fmt.Errorf("unexpected src: %s", src)
+				}
+
+				hit = false
+
+				return nil
+			}
+
+			getter := &testGetter{
+				get: get,
+			}
+			remote := &Remote{
+				Logger: helmexec.NewLogger(io.Discard, "debug"),
+				Home:   CacheDir(),
+				Getter: getter,
+				fs:     testfs.ToFileSystem(),
+			}
+
+			url := "git::ssh://git@github.com/helmfile/helmfiles.git@releases/kiam.yaml?ref=main&cache=false"
+			file, err := remote.Locate(url)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			expectedFile := filepath.Join(CacheDir(), "ssh_github_com_helmfile_helmfiles_git.ref=main/releases/kiam.yaml")
+			if file != expectedFile {
+				t.Errorf("unexpected file located: %s vs expected: %s", file, expectedFile)
+			}
+
+			if tt.expectCacheHit && !hit {
+				t.Errorf("unexpected result: unexpected cache miss")
+			}
+			if !tt.expectCacheHit && hit {
+				t.Errorf("unexpected result: unexpected cache hit")
+			}
+		})
+	}
+}
+
 func TestRemote_S3(t *testing.T) {
 	cleanfs := map[string]string{
 		CacheDir(): "",

pkg/runtime/runtime.go

@@ -9,15 +9,15 @@ import (
 
 var (
 	// GoYamlV3 is set to true in order to let Helmfile use
-	// gopkg.in/yaml.v3 instead of gopkg.in/yaml.v2.
+	// go.yaml.in/yaml/v3 instead of go.yaml.in/yaml/v2.
 	// It's false by default in Helmfile v0.x and true in Helmfile v1.x.
 	GoYamlV3 bool
 )
 
 func Info() string {
-	yamlLib := "gopkg.in/yaml.v2"
+	yamlLib := "go.yaml.in/yaml/v2"
 	if GoYamlV3 {
-		yamlLib = "gopkg.in/yaml.v3"
+		yamlLib = "go.yaml.in/yaml/v3"
 	}
 
 	return fmt.Sprintf("YAML library = %v", yamlLib)

pkg/tmpl/context_funcs_test.go

@@ -358,11 +358,11 @@ func testFromYaml(t *testing.T, GoYamlV3 bool) {
 }
 
 func TestFromYaml(t *testing.T) {
-	t.Run("with gopkg.in/yaml.v2", func(t *testing.T) {
+	t.Run("with go.yaml.in/yaml/v2", func(t *testing.T) {
 		testFromYaml(t, true)
 	})
 
-	t.Run("with gopkg.in/yaml.v3", func(t *testing.T) {
+	t.Run("with go.yaml.in/yaml/v3", func(t *testing.T) {
 		testFromYaml(t, false)
 	})
 }

pkg/yaml/yaml.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"io"
 
-	v2 "gopkg.in/yaml.v2"
+	v2 "go.yaml.in/yaml/v2"
-	v3 "gopkg.in/yaml.v3"
+	v3 "go.yaml.in/yaml/v3"
 
 	"github.com/helmfile/helmfile/pkg/runtime"
 )

pkg/yaml/yaml_test.go

@@ -13,9 +13,9 @@ func testYamlMarshal(t *testing.T, GoYamlV3 bool) {
 
 	var yamlLibraryName string
 	if GoYamlV3 {
-		yamlLibraryName = "gopkg.in/yaml.v3"
+		yamlLibraryName = "go.yaml.in/yaml/v3"
 	} else {
-		yamlLibraryName = "gopkg.in/yaml.v2"
+		yamlLibraryName = "go.yaml.in/yaml/v2"
 	}
 
 	v := runtime.GoYamlV3
@@ -49,8 +49,8 @@ func testYamlMarshal(t *testing.T, GoYamlV3 bool) {
 				Annotation: "on",
 			}},
 			expected: map[string]string{
-				"gopkg.in/yaml.v2": "name: John\ninfo:\n- age: 20\n  address: New York\n  annotation: \"on\"\n",
+				"go.yaml.in/yaml/v2": "name: John\ninfo:\n- age: 20\n  address: New York\n  annotation: \"on\"\n",
-				"gopkg.in/yaml.v3": "name: John\ninfo:\n  - age: 20\n    address: New York\n    annotation: \"on\"\n",
+				"go.yaml.in/yaml/v3": "name: John\ninfo:\n  - age: 20\n    address: New York\n    annotation: \"on\"\n",
 			},
 		},
 	}
@@ -63,11 +63,11 @@ func testYamlMarshal(t *testing.T, GoYamlV3 bool) {
 }
 
 func TestYamlMarshal(t *testing.T) {
-	t.Run("with gopkg.in/yaml.v2", func(t *testing.T) {
+	t.Run("with go.yaml.in/yaml/v2", func(t *testing.T) {
 		testYamlMarshal(t, true)
 	})
 
-	t.Run("with gopkg.in/yaml.v3", func(t *testing.T) {
+	t.Run("with go.yaml.in/yaml/v3", func(t *testing.T) {
 		testYamlMarshal(t, false)
 	})
 }

test/e2e/template/helmfile/snapshot_test.go

@@ -58,11 +58,11 @@ func (f fakeInit) Force() bool {
 }
 
 func TestHelmfileTemplateWithBuildCommand(t *testing.T) {
-	t.Run("with gopkg.in/yaml.v3", func(t *testing.T) {
+	t.Run("with go.yaml.in/yaml/v3", func(t *testing.T) {
 		testHelmfileTemplateWithBuildCommand(t, true)
 	})
 
-	t.Run("with gopkg.in/yaml.v2", func(t *testing.T) {
+	t.Run("with go.yaml.in/yaml/v2", func(t *testing.T) {
 		testHelmfileTemplateWithBuildCommand(t, false)
 	})
 }