feat: remote state files (#648)

This change enhances helmfile to accept terraform-module-like URLs in nested state files a.k.a sub-helmfiles.

```yaml
helmfiles:
- # Terraform-module-like URL for importing a remote directory and use a file in it as a nested-state file
  # The nested-state file is locally checked-out along with the remote directory containing it.
  # Therefore all the local paths in the file are resolved relative to the file
  path: git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0
```

The URL isn't equivalent to terraform module sources. The difference is that we use `@` to distinguish between (1) the path to the repository and directory containing the state file and (2) the path to the state file being loaded. This distinction provides us enough fleibiity to instruct helmfile to check-out necessary and sufficient directory to make the state file works.

Under the hood, it uses [hashicorp/go-getter](https://github.com/hashicorp/go-getter), that is used for [terraform module sources](https://www.terraform.io/docs/modules/sources.html) as well.

Only the git provider without authentication like git-credentials helper is tested. But theoretically any go-getter providers should work. Please feel free to test the provider of your choice and contribute documentation or instruction to use it :)

Resolves #347

pkg/app/app.go

@@ -3,6 +3,7 @@ package app
 import (
 	"fmt"
 	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/remote"
 	"github.com/roboll/helmfile/pkg/state"
 	"io/ioutil"
 	"log"
@@ -42,6 +43,8 @@ type App struct {
 
 	getwd func() (string, error)
 	chdir func(string) error
+
+	remote *remote.Remote
 }
 
 func New(conf ConfigProvider) *App {
@@ -302,6 +305,14 @@ func (a *App) visitStates(fileOrDir string, defOpts LoadOpts, converge func(*sta
 				} else {
 					optsForNestedState.Selectors = m.Selectors
 				}
+
+				path, err := a.remote.Locate(m.Path)
+				if err != nil {
+					return appError(fmt.Sprintf("in .helmfiles[%d]", i), err)
+				}
+
+				m.Path = path
+
 				if err := a.visitStates(m.Path, optsForNestedState, converge); err != nil {
 					switch err.(type) {
 					case *NoMatchingHelmfileError:
@@ -373,6 +384,26 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge
 		opts.Environment.OverrideValues = envvals
 	}
 
+	var dir string
+	if a.directoryExistsAt(fileOrDir) {
+		dir = fileOrDir
+	} else {
+		dir = filepath.Dir(fileOrDir)
+	}
+
+	getter := &remote.GoGetter{Logger: a.Logger}
+
+	remote := &remote.Remote{
+		Logger:     a.Logger,
+		Home:       dir,
+		Getter:     getter,
+		ReadFile:   a.readFile,
+		DirExists:  a.directoryExistsAt,
+		FileExists: a.fileExistsAt,
+	}
+
+	a.remote = remote
+
 	err := a.visitStates(fileOrDir, opts, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
 		if len(st.Selectors) > 0 {
 			err := st.FilterReleases()
@@ -388,6 +419,7 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge
 		type Key struct {
 			TillerNamespace, Name string
 		}
+
 		releaseNameCounts := map[Key]int{}
 		for _, r := range st.Releases {
 			tillerNamespace := st.HelmDefaults.TillerNamespace

pkg/app/app_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/state"
+	"github.com/roboll/helmfile/pkg/testhelper"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -13,11 +14,11 @@ import (
 )
 
 func appWithFs(app *App, files map[string]string) *App {
-	fs := state.NewTestFs(files)
+	fs := testhelper.NewTestFs(files)
 	return injectFs(app, fs)
 }
 
-func injectFs(app *App, fs *state.TestFs) *App {
+func injectFs(app *App, fs *testhelper.TestFs) *App {
 	app.readFile = fs.ReadFile
 	app.glob = fs.Glob
 	app.abs = fs.Abs
@@ -52,7 +53,7 @@ releases:
   chart: stable/grafana
 `,
 	}
-	fs := state.NewTestFs(files)
+	fs := testhelper.NewTestFs(files)
 	fs.GlobFixtures["/path/to/helmfile.d/a*.yaml"] = []string{"/path/to/helmfile.d/a2.yaml", "/path/to/helmfile.d/a1.yaml"}
 	app := &App{
 		KubeContext: "default",
@@ -98,7 +99,7 @@ BAR: 2
 BAZ: 4
 `,
 	}
-	fs := state.NewTestFs(files)
+	fs := testhelper.NewTestFs(files)
 	fs.GlobFixtures["/path/to/env.*.yaml"] = []string{"/path/to/env.2.yaml", "/path/to/env.1.yaml"}
 	app := &App{
 		KubeContext: "default",
@@ -137,7 +138,7 @@ releases:
   chart: stable/zipkin
 `,
 	}
-	fs := state.NewTestFs(files)
+	fs := testhelper.NewTestFs(files)
 	app := &App{
 		KubeContext: "default",
 		Logger:      helmexec.NewLogger(os.Stderr, "debug"),
@@ -190,7 +191,7 @@ releases:
   chart: stable/zipkin
 `, testcase.handler, testcase.filePattern),
 			}
-			fs := state.NewTestFs(files)
+			fs := testhelper.NewTestFs(files)
 			app := &App{
 				KubeContext: "default",
 				Logger:      helmexec.NewLogger(os.Stderr, "debug"),
@@ -251,7 +252,7 @@ releases:
 	}
 
 	for _, testcase := range testcases {
-		fs := state.NewTestFs(files)
+		fs := testhelper.NewTestFs(files)
 		fs.GlobFixtures["/path/to/helmfile.d/a*.yaml"] = []string{"/path/to/helmfile.d/a2.yaml", "/path/to/helmfile.d/a1.yaml"}
 		app := &App{
 			KubeContext: "default",
@@ -1077,7 +1078,7 @@ releases:
     stage: post
   <<: *default
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   default:
@@ -1158,7 +1159,7 @@ releases:
     stage: post
   <<: *default
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   default:
@@ -1235,7 +1236,7 @@ releases:
 - name: myrelease0
   chart: mychart0
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   default:
@@ -1295,7 +1296,7 @@ releases:
 - name: myrelease0
   chart: mychart0
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   default:
@@ -1372,7 +1373,7 @@ releases:
     stage: post
   <<: *default
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   test:
@@ -1458,7 +1459,7 @@ releases:
   chart: mychart3
   <<: *default
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/yaml/templates.yaml": `templates:
   default: &default
@@ -1515,7 +1516,7 @@ releases:
 - name: {{ .Environment.Values.foo | quote }}
   chart: {{ .Environment.Values.bar | quote }}
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		statePath:         stateContent,
 		"/path/to/1.yaml": `bar: ["bar"]`,
 		"/path/to/2.yaml": `bar: ["BAR"]`,
@@ -1568,7 +1569,7 @@ releases:
 - name: {{ .Environment.Values.foo | quote }}
   chart: {{ .Environment.Values.bar | quote }}
 `
-		testFs := state.NewTestFs(map[string]string{
+		testFs := testhelper.NewTestFs(map[string]string{
 			statePath:         stateContent,
 			"/path/to/1.yaml": `bar: ["bar"]`,
 			"/path/to/2.yaml": `bar: ["BAR"]`,
@@ -1653,7 +1654,7 @@ releases:
 		tc := testcases[i]
 		statePath := "/path/to/helmfile.yaml"
 		stateContent := fmt.Sprintf(tc.state, tc.expr)
-		testFs := state.NewTestFs(map[string]string{
+		testFs := testhelper.NewTestFs(map[string]string{
 			statePath:         stateContent,
 			"/path/to/1.yaml": `foo: FOO`,
 			"/path/to/2.yaml": `bar: { "baz": "BAZ" }

pkg/app/two_pass_renderer_test.go

@@ -3,6 +3,7 @@ package app
 import (
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/state"
+	"github.com/roboll/helmfile/pkg/testhelper"
 	"os"
 	"strings"
 	"testing"
@@ -10,8 +11,8 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-func makeLoader(files map[string]string, env string) (*desiredStateLoader, *state.TestFs) {
-	testfs := state.NewTestFs(files)
+func makeLoader(files map[string]string, env string) (*desiredStateLoader, *testhelper.TestFs) {
+	testfs := testhelper.NewTestFs(files)
 	return &desiredStateLoader{
 		env:        env,
 		namespace:  "namespace",