feat: remote state files (#648)

This change enhances helmfile to accept terraform-module-like URLs in nested state files a.k.a sub-helmfiles.

```yaml
helmfiles:
- # Terraform-module-like URL for importing a remote directory and use a file in it as a nested-state file
  # The nested-state file is locally checked-out along with the remote directory containing it.
  # Therefore all the local paths in the file are resolved relative to the file
  path: git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0
```

The URL isn't equivalent to terraform module sources. The difference is that we use `@` to distinguish between (1) the path to the repository and directory containing the state file and (2) the path to the state file being loaded. This distinction provides us enough fleibiity to instruct helmfile to check-out necessary and sufficient directory to make the state file works.

Under the hood, it uses [hashicorp/go-getter](https://github.com/hashicorp/go-getter), that is used for [terraform module sources](https://www.terraform.io/docs/modules/sources.html) as well.

Only the git provider without authentication like git-credentials helper is tested. But theoretically any go-getter providers should work. Please feel free to test the provider of your choice and contribute documentation or instruction to use it :)

Resolves #347

pkg/app/app.go

@@ -3,6 +3,7 @@ package app
 import (
 	"fmt"
 	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/remote"
 	"github.com/roboll/helmfile/pkg/state"
 	"io/ioutil"
 	"log"
@@ -42,6 +43,8 @@ type App struct {
 
 	getwd func() (string, error)
 	chdir func(string) error
+
+	remote *remote.Remote
 }
 
 func New(conf ConfigProvider) *App {
@@ -302,6 +305,14 @@ func (a *App) visitStates(fileOrDir string, defOpts LoadOpts, converge func(*sta
 				} else {
 					optsForNestedState.Selectors = m.Selectors
 				}
+
+				path, err := a.remote.Locate(m.Path)
+				if err != nil {
+					return appError(fmt.Sprintf("in .helmfiles[%d]", i), err)
+				}
+
+				m.Path = path
+
 				if err := a.visitStates(m.Path, optsForNestedState, converge); err != nil {
 					switch err.(type) {
 					case *NoMatchingHelmfileError:
@@ -373,6 +384,26 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge
 		opts.Environment.OverrideValues = envvals
 	}
 
+	var dir string
+	if a.directoryExistsAt(fileOrDir) {
+		dir = fileOrDir
+	} else {
+		dir = filepath.Dir(fileOrDir)
+	}
+
+	getter := &remote.GoGetter{Logger: a.Logger}
+
+	remote := &remote.Remote{
+		Logger:     a.Logger,
+		Home:       dir,
+		Getter:     getter,
+		ReadFile:   a.readFile,
+		DirExists:  a.directoryExistsAt,
+		FileExists: a.fileExistsAt,
+	}
+
+	a.remote = remote
+
 	err := a.visitStates(fileOrDir, opts, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
 		if len(st.Selectors) > 0 {
 			err := st.FilterReleases()
@@ -388,6 +419,7 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge
 		type Key struct {
 			TillerNamespace, Name string
 		}
+
 		releaseNameCounts := map[Key]int{}
 		for _, r := range st.Releases {
 			tillerNamespace := st.HelmDefaults.TillerNamespace

pkg/app/app_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/state"
+	"github.com/roboll/helmfile/pkg/testhelper"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -13,11 +14,11 @@ import (
 )
 
 func appWithFs(app *App, files map[string]string) *App {
-	fs := state.NewTestFs(files)
+	fs := testhelper.NewTestFs(files)
 	return injectFs(app, fs)
 }
 
-func injectFs(app *App, fs *state.TestFs) *App {
+func injectFs(app *App, fs *testhelper.TestFs) *App {
 	app.readFile = fs.ReadFile
 	app.glob = fs.Glob
 	app.abs = fs.Abs
@@ -52,7 +53,7 @@ releases:
   chart: stable/grafana
 `,
 	}
-	fs := state.NewTestFs(files)
+	fs := testhelper.NewTestFs(files)
 	fs.GlobFixtures["/path/to/helmfile.d/a*.yaml"] = []string{"/path/to/helmfile.d/a2.yaml", "/path/to/helmfile.d/a1.yaml"}
 	app := &App{
 		KubeContext: "default",
@@ -98,7 +99,7 @@ BAR: 2
 BAZ: 4
 `,
 	}
-	fs := state.NewTestFs(files)
+	fs := testhelper.NewTestFs(files)
 	fs.GlobFixtures["/path/to/env.*.yaml"] = []string{"/path/to/env.2.yaml", "/path/to/env.1.yaml"}
 	app := &App{
 		KubeContext: "default",
@@ -137,7 +138,7 @@ releases:
   chart: stable/zipkin
 `,
 	}
-	fs := state.NewTestFs(files)
+	fs := testhelper.NewTestFs(files)
 	app := &App{
 		KubeContext: "default",
 		Logger:      helmexec.NewLogger(os.Stderr, "debug"),
@@ -190,7 +191,7 @@ releases:
   chart: stable/zipkin
 `, testcase.handler, testcase.filePattern),
 			}
-			fs := state.NewTestFs(files)
+			fs := testhelper.NewTestFs(files)
 			app := &App{
 				KubeContext: "default",
 				Logger:      helmexec.NewLogger(os.Stderr, "debug"),
@@ -251,7 +252,7 @@ releases:
 	}
 
 	for _, testcase := range testcases {
-		fs := state.NewTestFs(files)
+		fs := testhelper.NewTestFs(files)
 		fs.GlobFixtures["/path/to/helmfile.d/a*.yaml"] = []string{"/path/to/helmfile.d/a2.yaml", "/path/to/helmfile.d/a1.yaml"}
 		app := &App{
 			KubeContext: "default",
@@ -1077,7 +1078,7 @@ releases:
     stage: post
   <<: *default
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   default:
@@ -1158,7 +1159,7 @@ releases:
     stage: post
   <<: *default
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   default:
@@ -1235,7 +1236,7 @@ releases:
 - name: myrelease0
   chart: mychart0
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   default:
@@ -1295,7 +1296,7 @@ releases:
 - name: myrelease0
   chart: mychart0
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   default:
@@ -1372,7 +1373,7 @@ releases:
     stage: post
   <<: *default
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   test:
@@ -1458,7 +1459,7 @@ releases:
   chart: mychart3
   <<: *default
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/yaml/templates.yaml": `templates:
   default: &default
@@ -1515,7 +1516,7 @@ releases:
 - name: {{ .Environment.Values.foo | quote }}
   chart: {{ .Environment.Values.bar | quote }}
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		statePath:         stateContent,
 		"/path/to/1.yaml": `bar: ["bar"]`,
 		"/path/to/2.yaml": `bar: ["BAR"]`,
@@ -1568,7 +1569,7 @@ releases:
 - name: {{ .Environment.Values.foo | quote }}
   chart: {{ .Environment.Values.bar | quote }}
 `
-		testFs := state.NewTestFs(map[string]string{
+		testFs := testhelper.NewTestFs(map[string]string{
 			statePath:         stateContent,
 			"/path/to/1.yaml": `bar: ["bar"]`,
 			"/path/to/2.yaml": `bar: ["BAR"]`,
@@ -1653,7 +1654,7 @@ releases:
 		tc := testcases[i]
 		statePath := "/path/to/helmfile.yaml"
 		stateContent := fmt.Sprintf(tc.state, tc.expr)
-		testFs := state.NewTestFs(map[string]string{
+		testFs := testhelper.NewTestFs(map[string]string{
 			statePath:         stateContent,
 			"/path/to/1.yaml": `foo: FOO`,
 			"/path/to/2.yaml": `bar: { "baz": "BAZ" }

pkg/app/two_pass_renderer_test.go

@@ -3,6 +3,7 @@ package app
 import (
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/state"
+	"github.com/roboll/helmfile/pkg/testhelper"
 	"os"
 	"strings"
 	"testing"
@@ -10,8 +11,8 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-func makeLoader(files map[string]string, env string) (*desiredStateLoader, *state.TestFs) {
-	testfs := state.NewTestFs(files)
+func makeLoader(files map[string]string, env string) (*desiredStateLoader, *testhelper.TestFs) {
+	testfs := testhelper.NewTestFs(files)
 	return &desiredStateLoader{
 		env:        env,
 		namespace:  "namespace",

pkg/remote/remote.go

@@ -0,0 +1,242 @@
+package remote
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/hashicorp/go-getter"
+	"github.com/hashicorp/go-getter/helper/url"
+	"go.uber.org/zap"
+	"gopkg.in/yaml.v2"
+	"path/filepath"
+	"strings"
+)
+
+const DefaultCacheDir = ".helmfile/cache"
+
+type Remote struct {
+	Logger *zap.SugaredLogger
+
+	// Home is the home directory for helmfile. Usually this points to $HOME of the user running helmfile.
+	// Helmfile saves fetched remote files into .helmfile/cache under home
+	Home string
+
+	// Getter is the underlying implementation of getter used for fetching remote files
+	Getter Getter
+
+	// ReadFile is the implementation of the file reader that reads a local file from the specified path.
+	// Inject any implementation of your choice, like an im-memory impl for testing, ioutil.ReadFile for the real-world use.
+	ReadFile   func(string) ([]byte, error)
+	DirExists  func(string) bool
+	FileExists func(string) bool
+}
+
+func (r *Remote) Unmarshal(src string, dst interface{}) error {
+	bytes, err := r.GetBytes(src)
+	if err != nil {
+		return err
+	}
+
+	strs := strings.Split(src, "/")
+	file := strs[len(strs)-1]
+	ext := filepath.Ext(file)
+
+	{
+		r.Logger.Debugf("unmarshalling %s", string(bytes))
+
+		var err error
+		switch ext {
+		case "json":
+			err = json.Unmarshal(bytes, dst)
+		default:
+			err = yaml.Unmarshal(bytes, dst)
+		}
+
+		r.Logger.Debugf("unmarshalled to %v", dst)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r *Remote) GetBytes(goGetterSrc string) ([]byte, error) {
+	f, err := r.Fetch(goGetterSrc)
+	if err != nil {
+		return nil, err
+	}
+
+	bytes, err := r.ReadFile(f)
+	if err != nil {
+		return nil, fmt.Errorf("read file: %v", err)
+	}
+
+	return bytes, nil
+}
+
+// Locate takes an URL to a remote file or a path to a local file.
+// If the argument was an URL, it fetches the remote directory contained within the URL,
+// and returns the path to the file in the fetched directory
+func (r *Remote) Locate(urlOrPath string) (string, error) {
+	fetched, err := r.Fetch(urlOrPath)
+	if err != nil {
+		switch err.(type) {
+		case InvalidURLError:
+			return urlOrPath, nil
+		}
+		return "", err
+	}
+	return fetched, nil
+}
+
+type InvalidURLError struct {
+	err string
+}
+
+func (e InvalidURLError) Error() string {
+	return e.err
+}
+
+type Source struct {
+	Getter, Scheme, Host, Dir, File, RawQuery string
+}
+
+func IsRemote(goGetterSrc string) bool {
+	if _, err := Parse(goGetterSrc); err != nil {
+		return false
+	}
+	return true
+}
+
+func Parse(goGetterSrc string) (*Source, error) {
+	items := strings.Split(goGetterSrc, "::")
+	var getter string
+	switch len(items) {
+	case 2:
+		getter = items[0]
+		goGetterSrc = items[1]
+	}
+
+	u, err := url.Parse(goGetterSrc)
+	if err != nil {
+		return nil, InvalidURLError{err: fmt.Sprintf("parse url: %v", err)}
+	}
+
+	if u.Scheme == "" {
+		return nil, InvalidURLError{err: fmt.Sprintf("parse url: missing scheme - probably this is a local file path? %s", goGetterSrc)}
+	}
+
+	pathComponents := strings.Split(u.Path, "@")
+	if len(pathComponents) != 2 {
+		return nil, fmt.Errorf("invalid src format: it must be `[<getter>::]<scheme>://<host>/<path/to/dir>@<path/to/file>?key1=val1&key2=val2: got %s", goGetterSrc)
+	}
+
+	return &Source{
+		Getter:   getter,
+		Scheme:   u.Scheme,
+		Host:     u.Host,
+		Dir:      pathComponents[0],
+		File:     pathComponents[1],
+		RawQuery: u.RawQuery,
+	}, nil
+}
+
+func (r *Remote) Fetch(goGetterSrc string) (string, error) {
+	u, err := Parse(goGetterSrc)
+	if err != nil {
+		return "", err
+	}
+
+	srcDir := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, u.Dir)
+	file := u.File
+
+	r.Logger.Debugf("getter: %s", u.Getter)
+	r.Logger.Debugf("scheme: %s", u.Scheme)
+	r.Logger.Debugf("host: %s", u.Host)
+	r.Logger.Debugf("dir: %s", u.Dir)
+	r.Logger.Debugf("file: %s", u.File)
+
+	// This should be shared across variant commands, so that they can share cache for the shared imports
+	cacheBaseDir := DefaultCacheDir
+
+	query := u.RawQuery
+
+	var cacheKey string
+	replacer := strings.NewReplacer(":", "", "//", "_", "/", "_", ".", "_")
+	dirKey := replacer.Replace(srcDir)
+	if len(query) > 0 {
+		paramsKey := strings.Replace(query, "&", "_", -1)
+		cacheKey = fmt.Sprintf("%s.%s", dirKey, paramsKey)
+	} else {
+		cacheKey = dirKey
+	}
+
+	cached := false
+
+	getterDst := filepath.Join(cacheBaseDir, cacheKey)
+
+	cacheDirPath := filepath.Join(r.Home, getterDst)
+
+	{
+		if r.FileExists(cacheDirPath) {
+			return "", fmt.Errorf("%s is not directory. please remove it so that variant could use it for dependency caching", getterDst)
+		}
+
+		if r.DirExists(cacheDirPath) {
+			cached = true
+		}
+	}
+
+	if !cached {
+		var getterSrc string
+
+		if len(query) == 0 {
+			getterSrc = srcDir
+		} else {
+			getterSrc = strings.Join([]string{srcDir, query}, "?")
+		}
+
+		if u.Getter != "" {
+			getterSrc = u.Getter + "::" + getterSrc
+		}
+
+		r.Logger.Debugf("downloading %s to %s", getterSrc, getterDst)
+
+		if err := r.Getter.Get(r.Home, getterSrc, getterDst); err != nil {
+			return "", err
+		}
+	}
+
+	return filepath.Join(cacheDirPath, file), nil
+}
+
+type Getter interface {
+	Get(wd, src, dst string) error
+}
+
+type GoGetter struct {
+	Logger *zap.SugaredLogger
+}
+
+func (g *GoGetter) Get(wd, src, dst string) error {
+	ctx := context.Background()
+
+	get := &getter.Client{
+		Ctx:     ctx,
+		Src:     src,
+		Dst:     dst,
+		Pwd:     wd,
+		Mode:    getter.ClientModeDir,
+		Options: []getter.ClientOption{},
+	}
+
+	g.Logger.Debugf("client: %+v", *get)
+
+	if err := get.Get(); err != nil {
+		return fmt.Errorf("get: %v", err)
+	}
+
+	return nil
+}

pkg/remote/remote_test.go

@@ -0,0 +1,94 @@
+package remote
+
+import (
+	"fmt"
+	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/testhelper"
+	"os"
+	"testing"
+)
+
+func TestRemote(t *testing.T) {
+	cleanfs := map[string]string{
+		"path/to/home": "",
+	}
+	cachefs := map[string]string{
+		"path/to/home/.helmfile/cache/https_github_com_cloudposse_helmfiles_git.ref=0.40.0/releases/kiam.yaml": "foo: bar",
+	}
+
+	type testcase struct {
+		files          map[string]string
+		expectCacheHit bool
+	}
+
+	testcases := []testcase{
+		{files: cleanfs, expectCacheHit: false},
+		{files: cachefs, expectCacheHit: true},
+	}
+
+	for i := range testcases {
+		testcase := testcases[i]
+
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			testfs := testhelper.NewTestFs(testcase.files)
+
+			hit := true
+
+			get := func(wd, src, dst string) error {
+				if wd != "path/to/home" {
+					return fmt.Errorf("unexpected wd: %s", wd)
+				}
+				if src != "git::https://github.com/cloudposse/helmfiles.git?ref=0.40.0" {
+					return fmt.Errorf("unexpected src: %s", src)
+				}
+
+				hit = false
+
+				return nil
+			}
+
+			getter := &testGetter{
+				get: get,
+			}
+			remote := &Remote{
+				Logger:     helmexec.NewLogger(os.Stderr, "debug"),
+				Home:       "path/to/home",
+				Getter:     getter,
+				ReadFile:   testfs.ReadFile,
+				FileExists: testfs.FileExistsAt,
+				DirExists:  testfs.DirectoryExistsAt,
+			}
+
+			// FYI, go-getter in the `dir` mode accepts URL like the below. So helmfile expects URLs similar to it:
+			//   go-getter -mode dir git::https://github.com/cloudposse/helmfiles.git?ref=0.40.0 gettertest1/b
+
+			// We use `@` to separate dir and the file path. This is a good idea borrowed from helm-git:
+			//   https://github.com/aslafy-z/helm-git
+
+			url := "git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0"
+			file, err := remote.Locate(url)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			if file != "path/to/home/.helmfile/cache/https_github_com_cloudposse_helmfiles_git.ref=0.40.0/releases/kiam.yaml" {
+				t.Errorf("unexpected file located: %s", file)
+			}
+
+			if testcase.expectCacheHit && !hit {
+				t.Errorf("unexpected result: unexpected cache miss")
+			}
+			if !testcase.expectCacheHit && hit {
+				t.Errorf("unexpected result: unexpected cache hit")
+			}
+		})
+	}
+}
+
+type testGetter struct {
+	get func(wd, src, dst string) error
+}
+
+func (t *testGetter) Get(wd, src, dst string) error {
+	return t.get(wd, src, dst)
+}

pkg/state/create_test.go

@@ -1,6 +1,7 @@
 package state
 
 import (
+	"github.com/roboll/helmfile/pkg/testhelper"
 	"go.uber.org/zap"
 	"io/ioutil"
 	"path/filepath"
@@ -98,7 +99,7 @@ bar: {{ readFile "bar.txt" }}
 
 	expectedValues := `env: production`
 
-	testFs := NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		fooYamlFile: string(fooYamlContent),
 		barYamlFile: string(barYamlContent),
 		barTextFile: string(barTextContent),

pkg/state/state.go

@@ -6,6 +6,7 @@ import (
 	"github.com/roboll/helmfile/pkg/environment"
 	"github.com/roboll/helmfile/pkg/event"
 	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/remote"
 	"github.com/roboll/helmfile/pkg/tmpl"
 	"io/ioutil"
 	"os"
@@ -1262,6 +1263,11 @@ func (st *HelmState) storage() *Storage {
 func (st *HelmState) ExpandedHelmfiles() ([]SubHelmfileSpec, error) {
 	helmfiles := []SubHelmfileSpec{}
 	for _, hf := range st.Helmfiles {
+		if remote.IsRemote(hf.Path) {
+			helmfiles = append(helmfiles, hf)
+			continue
+		}
+
 		matches, err := st.storage().ExpandPaths(hf.Path)
 		if err != nil {
 			return nil, err

pkg/state/state_test.go

@@ -2,6 +2,7 @@ package state
 
 import (
 	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/testhelper"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -16,7 +17,7 @@ import (
 
 var logger = helmexec.NewLogger(os.Stdout, "warn")
 
-func injectFs(st *HelmState, fs *TestFs) *HelmState {
+func injectFs(st *HelmState, fs *testhelper.TestFs) *HelmState {
 	st.glob = fs.Glob
 	st.readFile = fs.ReadFile
 	st.fileExists = fs.FileExists
@@ -1035,7 +1036,7 @@ func TestHelmState_SyncReleases_MissingValuesFileForUndesiredRelease(t *testing.
 				Releases: []ReleaseSpec{tt.release},
 				logger:   logger,
 			}
-			fs := NewTestFs(map[string]string{})
+			fs := testhelper.NewTestFs(map[string]string{})
 			state = injectFs(state, fs)
 			helm := &mockHelmExec{
 				lists: map[listKey]string{},
@@ -1434,7 +1435,7 @@ func TestHelmState_SyncReleasesCleanup(t *testing.T) {
 					return nil
 				},
 			}
-			testfs := NewTestFs(map[string]string{
+			testfs := testhelper.NewTestFs(map[string]string{
 				"/path/to/someFile": `foo: FOO`,
 			})
 			state = injectFs(state, testfs)
@@ -1517,7 +1518,7 @@ func TestHelmState_DiffReleasesCleanup(t *testing.T) {
 					return nil
 				},
 			}
-			testfs := NewTestFs(map[string]string{
+			testfs := testhelper.NewTestFs(map[string]string{
 				"/path/to/someFile": `foo: bar
 `,
 			})

pkg/testhelper/testfs.go

@@ -1,7 +1,8 @@
-package state
+package testhelper
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 )
@@ -20,8 +21,10 @@ type TestFs struct {
 func NewTestFs(files map[string]string) *TestFs {
 	dirs := map[string]bool{}
 	for abs, _ := range files {
-		d := filepath.Dir(abs)
-		dirs[d] = true
+		for d := filepath.Dir(abs); !dirs[d]; d = filepath.Dir(d) {
+			dirs[d] = true
+			fmt.Fprintf(os.Stderr, "testfs: recognized dir: %s\n", d)
+		}
 	}
 	return &TestFs{
 		Cwd:   "/path/to",