Add the ability to load a remote environment values file (#1296)

Enables the user to specify a remote path for an environment values file, e.g.,

```yaml
environments:
  cluster-azure-us-west:
    values:
      - git::https://git.company.org/helmfiles/global/azure.yaml?ref=master
      - git::https://git.company.org/helmfiles/global/us-west.yaml?ref=master
  cluster-gcp-europe-west:
    values:
      - git::https://git.company.org/helmfiles/global/gcp.yaml?ref=master
      - git::https://git.company.org/helmfiles/global/europe-west.yaml?ref=master

releases:
  - ...
```

This is particularly useful when you co-locate helmfiles within your project repo but want to reuse the definitions in a global repo.

go.sum

@@ -650,8 +650,10 @@ github.com/ory/dockertest v3.3.4+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnh
 github.com/otiai10/copy v1.1.1 h1:PH7IFlRQ6Fv9vYmuXbDRLdgTHoP1w483kPNUP2bskpo=
 github.com/otiai10/copy v1.1.1/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc=
 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
 github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso=
 github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0=
@@ -1050,6 +1052,7 @@ google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/
 google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40=
 google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.21.0 h1:zS+Q/CJJnVlXpXQVIz+lH0ZT2lBuT2ac7XD8Y/3w6hY=
+google.golang.org/api v0.26.0 h1:VJZ8h6E8ip82FRpQl848c5vAadxlTXrUh8RzQzSRm08=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

pkg/app/app.go

@@ -487,6 +487,7 @@ func (a *App) loadDesiredStateFromYaml(file string, opts ...LoadOpts) (*state.He
 		namespace:  a.Namespace,
 		logger:     a.Logger,
 		abs:        a.abs,
+		remote:     a.remote,
 
 		overrideKubeContext: a.OverrideKubeContext,
 		overrideHelmBinary:  a.OverrideHelmBinary,
@@ -791,18 +792,7 @@ func (a *App) visitStatesWithSelectorsAndRemoteSupport(fileOrDir string, converg
 		return err
 	}
 
-	getter := &remote.GoGetter{Logger: a.Logger}
+	a.remote = remote.NewRemote(a.Logger, dir, a.readFile, a.directoryExistsAt, a.fileExistsAt)
-
-	remote := &remote.Remote{
-		Logger:     a.Logger,
-		Home:       dir,
-		Getter:     getter,
-		ReadFile:   a.readFile,
-		DirExists:  a.directoryExistsAt,
-		FileExists: a.fileExistsAt,
-	}
-
-	a.remote = remote
 
 	return a.visitStates(fileOrDir, opts, converge)
 }

pkg/app/app_test.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
+	"github.com/roboll/helmfile/pkg/remote"
 	"io"
 	"log"
 	"os"
@@ -1505,11 +1506,13 @@ helmDefaults:
 		readFile:            testFs.ReadFile,
 		glob:                testFs.Glob,
 		abs:                 testFs.Abs,
+		directoryExistsAt:   testFs.DirectoryExistsAt,
 		fileExistsAt:        testFs.FileExistsAt,
 		fileExists:          testFs.FileExists,
 		Env:                 "default",
 		Logger:              helmexec.NewLogger(os.Stderr, "debug"),
 	}
+	app.remote = remote.NewRemote(app.Logger, "", app.readFile, app.directoryExistsAt, app.fileExistsAt)
 
 	expectNoCallsToHelm(app)
 
@@ -1597,6 +1600,7 @@ helmDefaults:
 		Env:                "default",
 		Logger:             helmexec.NewLogger(os.Stderr, "debug"),
 	}
+	app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
 
 	expectNoCallsToHelm(app)
 
@@ -1675,6 +1679,7 @@ foo: FOO
 		Env:                "default",
 		Logger:             helmexec.NewLogger(os.Stderr, "debug"),
 	}
+	app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
 
 	expectNoCallsToHelm(app)
 
@@ -1740,6 +1745,7 @@ foo: FOO
 		Env:                "default",
 		Logger:             helmexec.NewLogger(os.Stderr, "debug"),
 	}
+	app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
 
 	expectNoCallsToHelm(app)
 
@@ -1823,6 +1829,7 @@ helmDefaults:
 		Env:                "test",
 		Logger:             helmexec.NewLogger(os.Stderr, "debug"),
 	}
+	app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
 
 	expectNoCallsToHelm(app)
 
@@ -1897,6 +1904,7 @@ releases:
 		Env:                "default",
 		Logger:             helmexec.NewLogger(os.Stderr, "debug"),
 	}
+	app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
 
 	expectNoCallsToHelm(app)
 
@@ -1955,6 +1963,7 @@ releases:
 		Logger:             helmexec.NewLogger(os.Stderr, "debug"),
 	}
 
+	app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
 	expectNoCallsToHelm(app)
 
 	st, err := app.loadDesiredStateFromYaml(statePath, LoadOpts{Reverse: true})
@@ -2010,6 +2019,8 @@ releases:
 			Env:                "default",
 			Logger:             helmexec.NewLogger(os.Stderr, "debug"),
 		}
+		app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
+
 		opts := LoadOpts{
 			CalleePath: statePath,
 			Environment: state.SubhelmfileEnvironmentSpec{
@@ -2122,6 +2133,7 @@ services:
 			Env:                "default",
 			Logger:             helmexec.NewLogger(os.Stderr, "debug"),
 		}
+		app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
 
 		expectNoCallsToHelm(app)
 

pkg/app/desired_state_file_loader.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/remote"
 	"path/filepath"
 	"sort"
 
@@ -32,6 +33,7 @@ type desiredStateLoader struct {
 	glob       func(string) ([]string, error)
 	getHelm    func(*state.HelmState) helmexec.Interface
 
+	remote      *remote.Remote
 	logger      *zap.SugaredLogger
 	valsRuntime vals.Evaluator
 }
@@ -46,7 +48,7 @@ func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, e
 			return nil, fmt.Errorf("bug: opts.CalleePath was nil: f=%s, opts=%v", f, opts)
 		}
 		storage := state.NewStorage(opts.CalleePath, ld.logger, ld.glob)
-		envld := state.NewEnvironmentValuesLoader(storage, ld.readFile, ld.logger)
+		envld := state.NewEnvironmentValuesLoader(storage, ld.readFile, ld.logger, ld.remote)
 		handler := state.MissingFileHandlerError
 		vals, err := envld.LoadEnvironmentValues(&handler, args)
 		if err != nil {
@@ -147,7 +149,7 @@ func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *e
 }
 
 func (a *desiredStateLoader) underlying() *state.StateCreator {
-	c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.valsRuntime, a.getHelm, a.overrideHelmBinary)
+	c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.valsRuntime, a.getHelm, a.overrideHelmBinary, a.remote)
 	c.LoadFile = a.loadFile
 	return c
 }

pkg/app/two_pass_renderer_test.go

@@ -1,6 +1,7 @@
 package app
 
 import (
+	"github.com/roboll/helmfile/pkg/remote"
 	"os"
 	"strings"
 	"testing"
@@ -14,6 +15,8 @@ import (
 
 func makeLoader(files map[string]string, env string) (*desiredStateLoader, *testhelper.TestFs) {
 	testfs := testhelper.NewTestFs(files)
+	logger := helmexec.NewLogger(os.Stdout, "debug")
+	r := remote.NewRemote(logger, testfs.Cwd, testfs.ReadFile, testfs.DirectoryExistsAt, testfs.FileExistsAt)
 	return &desiredStateLoader{
 		env:        env,
 		namespace:  "namespace",
@@ -22,6 +25,7 @@ func makeLoader(files map[string]string, env string) (*desiredStateLoader, *test
 		fileExists: testfs.FileExists,
 		abs:        testfs.Abs,
 		glob:       testfs.Glob,
+		remote:     r,
 	}, testfs
 }
 

pkg/remote/remote.go

@@ -260,3 +260,15 @@ func (g *GoGetter) Get(wd, src, dst string) error {
 
 	return nil
 }
+
+func NewRemote(logger *zap.SugaredLogger, homeDir string, readFile func(string) ([]byte, error), dirExists func(string) bool, fileExists func(string) bool) *Remote {
+	remote := &Remote{
+		Logger:     logger,
+		Home:       homeDir,
+		Getter:     &GoGetter{Logger: logger},
+		ReadFile:   readFile,
+		DirExists:  dirExists,
+		FileExists: fileExists,
+	}
+	return remote
+}

pkg/state/create.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/remote"
 	"io"
 	"os"
 
@@ -52,9 +53,11 @@ type StateCreator struct {
 	getHelm func(*HelmState) helmexec.Interface
 
 	overrideHelmBinary string
+
+	remote *remote.Remote
 }
 
-func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), valsRuntime vals.Evaluator, getHelm func(*HelmState) helmexec.Interface, overrideHelmBinary string) *StateCreator {
+func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), valsRuntime vals.Evaluator, getHelm func(*HelmState) helmexec.Interface, overrideHelmBinary string, remote *remote.Remote) *StateCreator {
 	return &StateCreator{
 		logger:      logger,
 		readFile:    readFile,
@@ -66,6 +69,8 @@ func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error)
 		getHelm:     getHelm,
 
 		overrideHelmBinary: overrideHelmBinary,
+
+		remote: remote,
 	}
 }
 
@@ -139,7 +144,7 @@ func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *envi
 		return nil, &StateLoadError{fmt.Sprintf("failed to read %s", state.FilePath), err}
 	}
 
-	e.Defaults, err = state.loadValuesEntries(nil, state.DefaultValues)
+	e.Defaults, err = state.loadValuesEntries(nil, state.DefaultValues, c.remote)
 	if err != nil {
 		return nil, err
 	}
@@ -203,7 +208,7 @@ func (c *StateCreator) loadEnvValues(st *HelmState, name string, failOnMissingEn
 	envSpec, ok := st.Environments[name]
 	if ok {
 		var err error
-		envVals, err = st.loadValuesEntries(envSpec.MissingFileHandler, envSpec.Values)
+		envVals, err = st.loadValuesEntries(envSpec.MissingFileHandler, envSpec.Values, c.remote)
 		if err != nil {
 			return nil, err
 		}
@@ -321,11 +326,11 @@ func (c *StateCreator) scatterGatherEnvSecretFiles(st *HelmState, envSecretFiles
 	return nil
 }
 
-func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []interface{}) (map[string]interface{}, error) {
+func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []interface{}, remote *remote.Remote) (map[string]interface{}, error) {
 	envVals := map[string]interface{}{}
 
 	valuesEntries := append([]interface{}{}, entries...)
-	ld := NewEnvironmentValuesLoader(st.storage(), st.readFile, st.logger)
+	ld := NewEnvironmentValuesLoader(st.storage(), st.readFile, st.logger, remote)
 	var err error
 	envVals, err = ld.LoadEnvironmentValues(missingFileHandler, valuesEntries)
 	if err != nil {

pkg/state/create_test.go

@@ -1,6 +1,7 @@
 package state
 
 import (
+	"github.com/roboll/helmfile/pkg/remote"
 	"io/ioutil"
 	"path/filepath"
 	"reflect"
@@ -108,7 +109,9 @@ bar: {{ readFile "bar.txt" }}
 	})
 	testFs.Cwd = "/example/path/to"
 
-	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil, nil, "").ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", true, nil)
+	r := remote.NewRemote(logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
+	state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil, nil, "", r).
+		ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", true, nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}

pkg/state/envvals_loader.go

@@ -5,6 +5,7 @@ import (
 	"github.com/imdario/mergo"
 	"github.com/roboll/helmfile/pkg/environment"
 	"github.com/roboll/helmfile/pkg/maputil"
+	"github.com/roboll/helmfile/pkg/remote"
 	"github.com/roboll/helmfile/pkg/tmpl"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
@@ -17,13 +18,16 @@ type EnvironmentValuesLoader struct {
 	readFile func(string) ([]byte, error)
 
 	logger *zap.SugaredLogger
+
+	remote *remote.Remote
 }
 
-func NewEnvironmentValuesLoader(storage *Storage, readFile func(string) ([]byte, error), logger *zap.SugaredLogger) *EnvironmentValuesLoader {
+func NewEnvironmentValuesLoader(storage *Storage, readFile func(string) ([]byte, error), logger *zap.SugaredLogger, remote *remote.Remote) *EnvironmentValuesLoader {
 	return &EnvironmentValuesLoader{
 		storage:  storage,
 		readFile: readFile,
 		logger:   logger,
+		remote:   remote,
 	}
 }
 
@@ -36,6 +40,11 @@ func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *str
 		switch strOrMap := entry.(type) {
 		case string:
 			urlOrPath := strOrMap
+			localPath, err := ld.remote.Locate(urlOrPath)
+			if err == nil {
+				urlOrPath = localPath
+			}
+
 			files, skipped, err := ld.storage.resolveFile(missingFileHandler, "environment values", urlOrPath)
 			if err != nil {
 				return nil, err

pkg/state/envvals_loader_test.go

@@ -2,6 +2,7 @@ package state
 
 import (
 	"github.com/google/go-cmp/cmp"
+	"github.com/roboll/helmfile/pkg/remote"
 	"go.uber.org/zap"
 	"io/ioutil"
 	"path/filepath"
@@ -23,7 +24,10 @@ func newLoader() *EnvironmentValuesLoader {
 		logger:   sugar,
 	}
 
-	return NewEnvironmentValuesLoader(storage, ioutil.ReadFile, sugar)
+	readFile := func(s string) ([]byte, error) { return []byte{}, nil }
+	dirExists := func(d string) bool { return false }
+	fileExists := func(f string) bool { return false }
+	return NewEnvironmentValuesLoader(storage, ioutil.ReadFile, sugar, remote.NewRemote(sugar, "/tmp", readFile, dirExists, fileExists))
 }
 
 // See https://github.com/roboll/helmfile/pull/1169