Allow caching of remote files to be disabled (#2112)

* Allow caching of remote files to be disabled

Make it possible to automatically update the cache of remote
resources by disabling the caching of those resources using a query
string parameter (`cache=false`).

Signed-off-by: Jess <jess@ros.io>

* Fix test that broke

Because query parameters are being re-encoded, = is being encoded to %3D.

Signed-off-by: Jess <jess@ros.io>

* Add test for disabling caching of remote resources

Signed-off-by: Jess <jess@ros.io>

* Include example usage in docs

Signed-off-by: Jess <jess@ros.io>

---------

Signed-off-by: Jess <jess@ros.io>

docs/index.md

@@ -409,6 +409,9 @@ helmfiles:
   # The nested-state file is locally checked-out along with the remote directory containing it.
   # Therefore all the local paths in the file are resolved relative to the file
   path: git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0
+- # By default git repositories aren't updated unless the ref is updated.
+  # Alternatively, refer to a named ref and disable the caching.
+  path: git::ssh://git@github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=main&cache=false
 # If set to "Error", return an error when a subhelmfile points to a
 # non-existent path. The default behavior is to print a warning and continue.
 missingFileHandler: Error

pkg/remote/remote.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"maps"
 	"net/http"
 	neturl "net/url"
 	"os"
@@ -213,13 +214,16 @@ func (r *Remote) Fetch(path string, cacheDirOpt ...string) (string, error) {
 		return "", fmt.Errorf("[bug] cacheDirOpt's length: want 0 or 1, got %d", len(cacheDirOpt))
 	}
 
-	query := u.RawQuery
+	query, _ := neturl.ParseQuery(u.RawQuery)
+
+	should_cache := query.Get("cache") != "false"
+	delete(query, "cache")
 
 	var cacheKey string
 	replacer := strings.NewReplacer(":", "", "//", "_", "/", "_", ".", "_")
 	dirKey := replacer.Replace(srcDir)
 	if len(query) > 0 {
-		q, _ := neturl.ParseQuery(query)
+		q := maps.Clone(query)
 		if q.Has("sshkey") {
 			q.Set("sshkey", "redacted")
 		}
@@ -262,7 +266,7 @@ func (r *Remote) Fetch(path string, cacheDirOpt ...string) (string, error) {
 		}
 	}
 
-	if !cached {
+	if !cached || !should_cache {
 		var getterSrc string
 		if u.User != "" {
 			getterSrc = fmt.Sprintf("%s://%s@%s%s", u.Scheme, u.User, u.Host, u.Dir)
@@ -271,7 +275,7 @@ func (r *Remote) Fetch(path string, cacheDirOpt ...string) (string, error) {
 		}
 
 		if len(query) > 0 {
-			getterSrc = strings.Join([]string{getterSrc, query}, "?")
+			getterSrc = strings.Join([]string{getterSrc, query.Encode()}, "?")
 		}
 
 		r.Logger.Debugf("remote> downloading %s to %s", getterSrc, getterDst)

pkg/remote/remote_test.go

@@ -179,7 +179,7 @@ func TestRemote_SShGitHub_WithSshKey(t *testing.T) {
 				if wd != CacheDir() {
 					return fmt.Errorf("unexpected wd: %s", wd)
 				}
-				if src != "git::ssh://git@github.com/helmfile/helmfiles.git?ref=0.40.0&sshkey=ZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkJTU3dOY2xoVzQ2Vm9VR3dMQ3JscVRHYUdOVWdRVUVEUEptc1ZzdUViL2RBNUcrQk9YMWxGaUVMYU9HQ2F6bS9KQkR2V3Y2Y0ZDQUtVRjVocVJOUjdJPSA=" {
+				if src != "git::ssh://git@github.com/helmfile/helmfiles.git?ref=0.40.0&sshkey=ZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkJTU3dOY2xoVzQ2Vm9VR3dMQ3JscVRHYUdOVWdRVUVEUEptc1ZzdUViL2RBNUcrQk9YMWxGaUVMYU9HQ2F6bS9KQkR2V3Y2Y0ZDQUtVRjVocVJOUjdJPSA%3D" {
 					return fmt.Errorf("unexpected src: %s", src)
 				}
 
@@ -219,6 +219,73 @@ func TestRemote_SShGitHub_WithSshKey(t *testing.T) {
 	}
 }
 
+func TestRemote_SShGitHub_WithDisableCacheKey(t *testing.T) {
+	cleanfs := map[string]string{
+		CacheDir(): "",
+	}
+	cachefs := map[string]string{
+		filepath.Join(CacheDir(), "ssh_github_com_helmfile_helmfiles_git.ref=main/releases/kiam.yaml"): "foo: bar",
+	}
+
+	testcases := []struct {
+		name           string
+		files          map[string]string
+		expectCacheHit bool
+	}{
+		{name: "not expectCacheHit", files: cleanfs, expectCacheHit: false},
+		{name: "forceNoCacheHit", files: cachefs, expectCacheHit: false},
+	}
+
+	for _, tt := range testcases {
+		t.Run(tt.name, func(t *testing.T) {
+			testfs := testhelper.NewTestFs(tt.files)
+
+			hit := true
+
+			get := func(wd, src, dst string) error {
+				if wd != CacheDir() {
+					return fmt.Errorf("unexpected wd: %s", wd)
+				}
+				if src != "git::ssh://git@github.com/helmfile/helmfiles.git?ref=main" {
+					return fmt.Errorf("unexpected src: %s", src)
+				}
+
+				hit = false
+
+				return nil
+			}
+
+			getter := &testGetter{
+				get: get,
+			}
+			remote := &Remote{
+				Logger: helmexec.NewLogger(io.Discard, "debug"),
+				Home:   CacheDir(),
+				Getter: getter,
+				fs:     testfs.ToFileSystem(),
+			}
+
+			url := "git::ssh://git@github.com/helmfile/helmfiles.git@releases/kiam.yaml?ref=main&cache=false"
+			file, err := remote.Locate(url)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			expectedFile := filepath.Join(CacheDir(), "ssh_github_com_helmfile_helmfiles_git.ref=main/releases/kiam.yaml")
+			if file != expectedFile {
+				t.Errorf("unexpected file located: %s vs expected: %s", file, expectedFile)
+			}
+
+			if tt.expectCacheHit && !hit {
+				t.Errorf("unexpected result: unexpected cache miss")
+			}
+			if !tt.expectCacheHit && hit {
+				t.Errorf("unexpected result: unexpected cache hit")
+			}
+		})
+	}
+}
+
 func TestRemote_S3(t *testing.T) {
 	cleanfs := map[string]string{
 		CacheDir(): "",