feat: support most remote urls (#1061)

* feat: support most remote urls

This adds support for s3:// http:// https:// and most other go-getter style urls by

- Adding http downloader
- Adding s3 downloader that authenticates with aws sdk shared credentials
- Changing the parser to accept remote formats that do not include '::'
- Removing validation that required go-getter urls to contain '@' which is not required by go-getter spec

Resolves #831

Signed-off-by: joshwoodcock <ltdwoodcock@gmail.com>

* Fix nil region

Signed-off-by: Josh Woodcock <b.woodcock@partner.samsung.com>

* fix golangci

Signed-off-by: xiaomudk <xiaomudk@gmail.com>

* fix testcase error

Signed-off-by: xiaomudk <xiaomudk@gmail.com>

* optimize code

Signed-off-by: yxxhero <aiopsclub@163.com>

* fix handle error

Signed-off-by: zhuxixi179 <zhuxixi179@hellobike.com>

---------

Signed-off-by: joshwoodcock <ltdwoodcock@gmail.com>
Signed-off-by: Josh Woodcock <b.woodcock@partner.samsung.com>
Signed-off-by: xiaomudk <xiaomudk@gmail.com>
Signed-off-by: yxxhero <aiopsclub@163.com>
Signed-off-by: zhuxixi179 <zhuxixi179@hellobike.com>
Co-authored-by: joshwoodcock <ltdwoodcock@gmail.com>
Co-authored-by: Josh Woodcock <b.woodcock@partner.samsung.com>
Co-authored-by: yxxhero <aiopsclub@163.com>
Co-authored-by: zhuxixi179 <zhuxixi179@hellobike.com>

pkg/remote/remote_test.go

@@ -20,21 +20,18 @@ func TestRemote_HttpsGitHub(t *testing.T) {
 		filepath.Join(CacheDir(), "https_github_com_cloudposse_helmfiles_git.ref=0.40.0/releases/kiam.yaml"): "foo: bar",
 	}
 
-	type testcase struct {
+	testcases := []struct {
+		name           string
 		files          map[string]string
 		expectCacheHit bool
+	}{
+		{name: "not expectCacheHit", files: cleanfs, expectCacheHit: false},
+		{name: "expectCacheHit", files: cachefs, expectCacheHit: true},
 	}
 
-	testcases := []testcase{
-		{files: cleanfs, expectCacheHit: false},
-		{files: cachefs, expectCacheHit: true},
-	}
-
-	for i := range testcases {
-		testcase := testcases[i]
-
-		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			testfs := testhelper.NewTestFs(testcase.files)
+	for _, tt := range testcases {
+		t.Run(tt.name, func(t *testing.T) {
+			testfs := testhelper.NewTestFs(tt.files)
 
 			hit := true
 
@@ -78,10 +75,10 @@ func TestRemote_HttpsGitHub(t *testing.T) {
 				t.Errorf("unexpected file located: %s vs expected: %s", file, expectedFile)
 			}
 
-			if testcase.expectCacheHit && !hit {
+			if tt.expectCacheHit && !hit {
 				t.Errorf("unexpected result: unexpected cache miss")
 			}
-			if !testcase.expectCacheHit && hit {
+			if !tt.expectCacheHit && hit {
 				t.Errorf("unexpected result: unexpected cache hit")
 			}
 		})
@@ -93,24 +90,21 @@ func TestRemote_SShGitHub(t *testing.T) {
 		CacheDir(): "",
 	}
 	cachefs := map[string]string{
-		filepath.Join(CacheDir(), "ssh_github_com_cloudposse_helmfiles_git.ref=0.40.0/releases/kiam.yaml"): "foo: bar",
+		filepath.Join(CacheDir(), "ssh_github_com_helmfile_helmfiles_git.ref=0.40.0/releases/kiam.yaml"): "foo: bar",
 	}
 
-	type testcase struct {
+	testcases := []struct {
+		name           string
 		files          map[string]string
 		expectCacheHit bool
+	}{
+		{name: "not expectCacheHit", files: cleanfs, expectCacheHit: false},
+		{name: "expectCacheHit", files: cachefs, expectCacheHit: true},
 	}
 
-	testcases := []testcase{
-		{files: cleanfs, expectCacheHit: false},
-		{files: cachefs, expectCacheHit: true},
-	}
-
-	for i := range testcases {
-		testcase := testcases[i]
-
-		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			testfs := testhelper.NewTestFs(testcase.files)
+	for _, tt := range testcases {
+		t.Run(tt.name, func(t *testing.T) {
+			testfs := testhelper.NewTestFs(tt.files)
 
 			hit := true
 
@@ -118,7 +112,7 @@ func TestRemote_SShGitHub(t *testing.T) {
 				if wd != CacheDir() {
 					return fmt.Errorf("unexpected wd: %s", wd)
 				}
-				if src != "git::ssh://git@github.com/cloudposse/helmfiles.git?ref=0.40.0" {
+				if src != "git::ssh://git@github.com/helmfile/helmfiles.git?ref=0.40.0" {
 					return fmt.Errorf("unexpected src: %s", src)
 				}
 
@@ -137,21 +131,21 @@ func TestRemote_SShGitHub(t *testing.T) {
 				fs:     testfs.ToFileSystem(),
 			}
 
-			url := "git::ssh://git@github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0"
+			url := "git::ssh://git@github.com/helmfile/helmfiles.git@releases/kiam.yaml?ref=0.40.0"
 			file, err := remote.Locate(url)
 			if err != nil {
 				t.Fatalf("unexpected error: %v", err)
 			}
 
-			expectedFile := filepath.Join(CacheDir(), "ssh_github_com_cloudposse_helmfiles_git.ref=0.40.0/releases/kiam.yaml")
+			expectedFile := filepath.Join(CacheDir(), "ssh_github_com_helmfile_helmfiles_git.ref=0.40.0/releases/kiam.yaml")
 			if file != expectedFile {
 				t.Errorf("unexpected file located: %s vs expected: %s", file, expectedFile)
 			}
 
-			if testcase.expectCacheHit && !hit {
+			if tt.expectCacheHit && !hit {
 				t.Errorf("unexpected result: unexpected cache miss")
 			}
-			if !testcase.expectCacheHit && hit {
+			if !tt.expectCacheHit && hit {
 				t.Errorf("unexpected result: unexpected cache hit")
 			}
 		})
@@ -163,24 +157,21 @@ func TestRemote_SShGitHub_WithSshKey(t *testing.T) {
 		CacheDir(): "",
 	}
 	cachefs := map[string]string{
-		filepath.Join(CacheDir(), "ssh_github_com_cloudposse_helmfiles_git.ref=0.40.0_sshkey=redacted/releases/kiam.yaml"): "foo: bar",
+		filepath.Join(CacheDir(), "ssh_github_com_helmfile_helmfiles_git.ref=0.40.0_sshkey=redacted/releases/kiam.yaml"): "foo: bar",
 	}
 
-	type testcase struct {
+	testcases := []struct {
+		name           string
 		files          map[string]string
 		expectCacheHit bool
+	}{
+		{name: "not expectCacheHit", files: cleanfs, expectCacheHit: false},
+		{name: "expectCacheHit", files: cachefs, expectCacheHit: true},
 	}
 
-	testcases := []testcase{
-		{files: cleanfs, expectCacheHit: false},
-		{files: cachefs, expectCacheHit: true},
-	}
-
-	for i := range testcases {
-		testcase := testcases[i]
-
-		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			testfs := testhelper.NewTestFs(testcase.files)
+	for _, tt := range testcases {
+		t.Run(tt.name, func(t *testing.T) {
+			testfs := testhelper.NewTestFs(tt.files)
 
 			hit := true
 
@@ -188,7 +179,7 @@ func TestRemote_SShGitHub_WithSshKey(t *testing.T) {
 				if wd != CacheDir() {
 					return fmt.Errorf("unexpected wd: %s", wd)
 				}
-				if src != "git::ssh://git@github.com/cloudposse/helmfiles.git?ref=0.40.0&sshkey=ZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkJTU3dOY2xoVzQ2Vm9VR3dMQ3JscVRHYUdOVWdRVUVEUEptc1ZzdUViL2RBNUcrQk9YMWxGaUVMYU9HQ2F6bS9KQkR2V3Y2Y0ZDQUtVRjVocVJOUjdJPSA=" {
+				if src != "git::ssh://git@github.com/helmfile/helmfiles.git?ref=0.40.0&sshkey=ZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkJTU3dOY2xoVzQ2Vm9VR3dMQ3JscVRHYUdOVWdRVUVEUEptc1ZzdUViL2RBNUcrQk9YMWxGaUVMYU9HQ2F6bS9KQkR2V3Y2Y0ZDQUtVRjVocVJOUjdJPSA=" {
 					return fmt.Errorf("unexpected src: %s", src)
 				}
 
@@ -207,21 +198,90 @@ func TestRemote_SShGitHub_WithSshKey(t *testing.T) {
 				fs:     testfs.ToFileSystem(),
 			}
 
-			url := "git::ssh://git@github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0&sshkey=ZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkJTU3dOY2xoVzQ2Vm9VR3dMQ3JscVRHYUdOVWdRVUVEUEptc1ZzdUViL2RBNUcrQk9YMWxGaUVMYU9HQ2F6bS9KQkR2V3Y2Y0ZDQUtVRjVocVJOUjdJPSA="
+			url := "git::ssh://git@github.com/helmfile/helmfiles.git@releases/kiam.yaml?ref=0.40.0&sshkey=ZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkJTU3dOY2xoVzQ2Vm9VR3dMQ3JscVRHYUdOVWdRVUVEUEptc1ZzdUViL2RBNUcrQk9YMWxGaUVMYU9HQ2F6bS9KQkR2V3Y2Y0ZDQUtVRjVocVJOUjdJPSA="
 			file, err := remote.Locate(url)
 			if err != nil {
 				t.Fatalf("unexpected error: %v", err)
 			}
 
-			expectedFile := filepath.Join(CacheDir(), "ssh_github_com_cloudposse_helmfiles_git.ref=0.40.0_sshkey=redacted/releases/kiam.yaml")
+			expectedFile := filepath.Join(CacheDir(), "ssh_github_com_helmfile_helmfiles_git.ref=0.40.0_sshkey=redacted/releases/kiam.yaml")
 			if file != expectedFile {
 				t.Errorf("unexpected file located: %s vs expected: %s", file, expectedFile)
 			}
 
-			if testcase.expectCacheHit && !hit {
+			if tt.expectCacheHit && !hit {
 				t.Errorf("unexpected result: unexpected cache miss")
 			}
-			if !testcase.expectCacheHit && hit {
+			if !tt.expectCacheHit && hit {
+				t.Errorf("unexpected result: unexpected cache hit")
+			}
+		})
+	}
+}
+
+func TestRemote_S3(t *testing.T) {
+	cleanfs := map[string]string{
+		CacheDir(): "",
+	}
+	cachefs := map[string]string{
+		filepath.Join(CacheDir(), "s3_helm-s3-values-example/subdir/values.yaml"): "foo: bar",
+	}
+
+	testcases := []struct {
+		name           string
+		files          map[string]string
+		expectCacheHit bool
+	}{
+		{name: "not expectCacheHit", files: cleanfs, expectCacheHit: false},
+		{name: "expectCacheHit", files: cachefs, expectCacheHit: true},
+	}
+
+	for _, tt := range testcases {
+		t.Run(tt.name, func(t *testing.T) {
+			testfs := testhelper.NewTestFs(tt.files)
+
+			hit := true
+
+			get := func(wd, src, dst string) error {
+				if wd != CacheDir() {
+					return fmt.Errorf("unexpected wd: %s", wd)
+				}
+				if src != "s3://helm-s3-values-example/subdir/values.yaml" {
+					return fmt.Errorf("unexpected src: %s", src)
+				}
+
+				hit = false
+
+				return nil
+			}
+
+			getter := &testGetter{
+				get: get,
+			}
+			remote := &Remote{
+				Logger:     helmexec.NewLogger(io.Discard, "debug"),
+				Home:       CacheDir(),
+				Getter:     getter,
+				S3Getter:   getter,
+				HttpGetter: getter,
+				fs:         testfs.ToFileSystem(),
+			}
+
+			url := "s3://helm-s3-values-example/subdir/values.yaml"
+			file, err := remote.Locate(url)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			expectedFile := filepath.Join(CacheDir(), "s3_helm-s3-values-example/subdir/values.yaml")
+			if file != expectedFile {
+				t.Errorf("unexpected file located: %s vs expected: %s", file, expectedFile)
+			}
+
+			if tt.expectCacheHit && !hit {
+				t.Errorf("unexpected result: unexpected cache miss")
+			}
+			if !tt.expectCacheHit && hit {
 				t.Errorf("unexpected result: unexpected cache hit")
 			}
 		})
@@ -229,18 +289,23 @@ func TestRemote_SShGitHub_WithSshKey(t *testing.T) {
 }
 
 func TestParse(t *testing.T) {
-	type testcase struct {
-		input                            string
-		getter, scheme, dir, file, query string
-		err                              string
-	}
-
-	testcases := []testcase{
+	testcases := []struct {
+		name   string
+		input  string
+		getter string
+		scheme string
+		dir    string
+		file   string
+		query  string
+		err    string
+	}{
 		{
+			name:  "miss scheme",
 			input: "raw/incubator",
 			err:   "parse url: missing scheme - probably this is a local file path? raw/incubator",
 		},
 		{
+			name:   "git scheme",
 			input:  "git::https://github.com/stakater/Forecastle.git@deployments/kubernetes/chart/forecastle?ref=v1.0.54",
 			getter: "git",
 			scheme: "https",
@@ -248,20 +313,81 @@ func TestParse(t *testing.T) {
 			file:   "deployments/kubernetes/chart/forecastle",
 			query:  "ref=v1.0.54",
 		},
+		{
+			name:   "s3 scheme",
+			input:  "s3://helm-s3-values-example/values.yaml",
+			getter: "normal",
+			scheme: "s3",
+			dir:    "",
+			file:   "values.yaml",
+			query:  "",
+		},
+		{
+			name:   "s3 scheme with subdir",
+			input:  "s3://helm-s3-values-example/subdir/values.yaml",
+			getter: "normal",
+			scheme: "s3",
+			dir:    "subdir",
+			file:   "values.yaml",
+			query:  "",
+		},
+		{
+			name:   "http scheme",
+			input:  "http::http://helm-s3-values-example.s3.us-east-2.amazonaws.com/values.yaml",
+			getter: "http",
+			scheme: "http",
+			dir:    "",
+			file:   "values.yaml",
+			query:  "",
+		},
+		{
+			name:   "http scheme with subdir",
+			input:  "http::http://helm-s3-values-example.s3.us-east-2.amazonaws.com/subdir/values.yaml",
+			getter: "http",
+			scheme: "http",
+			dir:    "subdir",
+			file:   "values.yaml",
+			query:  "",
+		},
+		{
+			name:   "https scheme",
+			input:  "http::https://helm-s3-values-example.s3.us-east-2.amazonaws.com/values.yaml",
+			getter: "http",
+			scheme: "https",
+			dir:    "",
+			file:   "values.yaml",
+			query:  "",
+		},
+		{
+			name:   "http scheme normal",
+			input:  "http://helm-s3-values-example.s3.us-east-2.amazonaws.com/values.yaml",
+			getter: "normal",
+			scheme: "http",
+			dir:    "",
+			file:   "values.yaml",
+			query:  "",
+		},
+		{
+			name:   "https scheme normal",
+			input:  "https://helm-s3-values-example.s3.us-east-2.amazonaws.com/values.yaml",
+			getter: "normal",
+			scheme: "https",
+			dir:    "",
+			file:   "values.yaml",
+			query:  "",
+		},
 	}
 
-	for i := range testcases {
-		tc := testcases[i]
-
-		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			src, err := Parse(tc.input)
+	for _, tt := range testcases {
+		t.Run(tt.name, func(t *testing.T) {
+			src, err := Parse(tt.input)
 
 			var errMsg string
 			if err != nil {
 				errMsg = err.Error()
 			}
 
-			if diff := cmp.Diff(tc.err, errMsg); diff != "" {
+			if diff := cmp.Diff(tt.err, errMsg); diff != "" {
 				t.Fatalf("Unexpected error:\n%s", diff)
 			}
 
@@ -274,23 +400,23 @@ func TestParse(t *testing.T) {
 				query = src.RawQuery
 			}
 
-			if diff := cmp.Diff(tc.getter, getter); diff != "" {
+			if diff := cmp.Diff(tt.getter, getter); diff != "" {
 				t.Fatalf("Unexpected getter:\n%s", diff)
 			}
 
-			if diff := cmp.Diff(tc.scheme, scheme); diff != "" {
+			if diff := cmp.Diff(tt.scheme, scheme); diff != "" {
 				t.Fatalf("Unexpected scheme:\n%s", diff)
 			}
 
-			if diff := cmp.Diff(tc.file, file); diff != "" {
+			if diff := cmp.Diff(tt.file, file); diff != "" {
 				t.Fatalf("Unexpected file:\n%s", diff)
 			}
 
-			if diff := cmp.Diff(tc.dir, dir); diff != "" {
+			if diff := cmp.Diff(tt.dir, dir); diff != "" {
 				t.Fatalf("Unexpected dir:\n%s", diff)
 			}
 
-			if diff := cmp.Diff(tc.query, query); diff != "" {
+			if diff := cmp.Diff(tt.query, query); diff != "" {
 				t.Fatalf("Unexpected query:\n%s", diff)
 			}
 		})
@@ -313,24 +439,21 @@ func TestRemote_Fetch(t *testing.T) {
 		filepath.Join(CacheDir(), "https_github_com_helmfile_helmfile_git.ref=v0.151.0/README.md"): "foo: bar",
 	}
 
-	type testcase struct {
+	testcases := []struct {
+		name           string
 		files          map[string]string
 		expectCacheHit bool
 		cacheDirOpt    string
+	}{
+		{name: "not expectCacheHit", files: cleanfs, expectCacheHit: false, cacheDirOpt: ""},
+		{name: "expectCacheHit", files: cachefs, expectCacheHit: true, cacheDirOpt: ""},
+		{name: "not expectCacheHit with states", files: cleanfs, expectCacheHit: false, cacheDirOpt: "states"},
+		{name: "expectCacheHit with states", files: cachefs, expectCacheHit: true, cacheDirOpt: "states"},
 	}
 
-	testcases := []testcase{
-		{files: cleanfs, expectCacheHit: false, cacheDirOpt: ""},
-		{files: cachefs, expectCacheHit: true, cacheDirOpt: ""},
-		{files: cleanfs, expectCacheHit: false, cacheDirOpt: "states"},
-		{files: cachefs, expectCacheHit: true, cacheDirOpt: "states"},
-	}
-
-	for i := range testcases {
-		testcase := testcases[i]
-
-		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			testfs := testhelper.NewTestFs(testcase.files)
+	for _, tt := range testcases {
+		t.Run(tt.name, func(t *testing.T) {
+			testfs := testhelper.NewTestFs(tt.files)
 
 			hit := true
 
@@ -368,10 +491,10 @@ func TestRemote_Fetch(t *testing.T) {
 				t.Errorf("unexpected file located: %s vs expected: %s", file, expectedFile)
 			}
 
-			if testcase.expectCacheHit && !hit {
+			if tt.expectCacheHit && !hit {
 				t.Errorf("unexpected result: unexpected cache miss")
 			}
-			if !testcase.expectCacheHit && hit {
+			if !tt.expectCacheHit && hit {
 				t.Errorf("unexpected result: unexpected cache hit")
 			}
 		})