public_git
/
helmfile
mirror of https://github.com/helmfile/helmfile.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add the ability to load a remote environment values file (#1296) 

Enables the user to specify a remote path for an environment values file, e.g.,

```yaml
environments:
  cluster-azure-us-west:
    values:
      - git::https://git.company.org/helmfiles/global/azure.yaml?ref=master
      - git::https://git.company.org/helmfiles/global/us-west.yaml?ref=master
  cluster-gcp-europe-west:
    values:
      - git::https://git.company.org/helmfiles/global/gcp.yaml?ref=master
      - git::https://git.company.org/helmfiles/global/europe-west.yaml?ref=master

releases:
  - ...
```

This is particularly useful when you co-locate helmfiles within your project repo but want to reuse the definitions in a global repo.
This commit is contained in:
Kevin J. Qiu 2020-06-10 21:04:01 -04:00 committed by GitHub
parent ca844aa044
commit 0f86cc9b87
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 66 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
go.sum
View File

@ -650,8 +650,10 @@ github.com/ory/dockertest v3.3.4+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnh
github.com/otiai10/copy v1.1.1 h1:PH7IFlRQ6Fv9vYmuXbDRLdgTHoP1w483kPNUP2bskpo=
github.com/otiai10/copy v1.1.1/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=
github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc=
github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso=
github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0=
@ -1050,6 +1052,7 @@ google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/
google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40=
google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.21.0 h1:zS+Q/CJJnVlXpXQVIz+lH0ZT2lBuT2ac7XD8Y/3w6hY=
google.golang.org/api v0.26.0 h1:VJZ8h6E8ip82FRpQl848c5vAadxlTXrUh8RzQzSRm08=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

14
pkg/app/app.go
View File

@ -487,6 +487,7 @@ func (a *App) loadDesiredStateFromYaml(file string, opts ...LoadOpts) (*state.He
namespace: a.Namespace,
logger: a.Logger,
abs: a.abs,
remote: a.remote,
overrideKubeContext: a.OverrideKubeContext,
overrideHelmBinary: a.OverrideHelmBinary,
@ -791,18 +792,7 @@ func (a *App) visitStatesWithSelectorsAndRemoteSupport(fileOrDir string, converg
return err
}
getter := &remote.GoGetter{Logger: a.Logger}
remote := &remote.Remote{
Logger: a.Logger,
Home: dir,
Getter: getter,
ReadFile: a.readFile,
DirExists: a.directoryExistsAt,
FileExists: a.fileExistsAt,
}
a.remote = remote
a.remote = remote.NewRemote(a.Logger, dir, a.readFile, a.directoryExistsAt, a.fileExistsAt)
return a.visitStates(fileOrDir, opts, converge)
}

12
pkg/app/app_test.go
View File

@ -4,6 +4,7 @@ import (
"bufio"
"bytes"
"fmt"
"github.com/roboll/helmfile/pkg/remote"
"io"
"log"
"os"
@ -1505,11 +1506,13 @@ helmDefaults:
readFile: testFs.ReadFile,
glob: testFs.Glob,
abs: testFs.Abs,
directoryExistsAt: testFs.DirectoryExistsAt,
fileExistsAt: testFs.FileExistsAt,
fileExists: testFs.FileExists,
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
app.remote = remote.NewRemote(app.Logger, "", app.readFile, app.directoryExistsAt, app.fileExistsAt)
expectNoCallsToHelm(app)
@ -1597,6 +1600,7 @@ helmDefaults:
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
expectNoCallsToHelm(app)
@ -1675,6 +1679,7 @@ foo: FOO
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
expectNoCallsToHelm(app)
@ -1740,6 +1745,7 @@ foo: FOO
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
expectNoCallsToHelm(app)
@ -1823,6 +1829,7 @@ helmDefaults:
Env: "test",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
expectNoCallsToHelm(app)
@ -1897,6 +1904,7 @@ releases:
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
expectNoCallsToHelm(app)
@ -1955,6 +1963,7 @@ releases:
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
expectNoCallsToHelm(app)
st, err := app.loadDesiredStateFromYaml(statePath, LoadOpts{Reverse: true})
@ -2010,6 +2019,8 @@ releases:
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
opts := LoadOpts{
CalleePath: statePath,
Environment: state.SubhelmfileEnvironmentSpec{
@ -2122,6 +2133,7 @@ services:
Env: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
}
app.remote = remote.NewRemote(app.Logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
expectNoCallsToHelm(app)

6
pkg/app/desired_state_file_loader.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"github.com/roboll/helmfile/pkg/helmexec"
"github.com/roboll/helmfile/pkg/remote"
"path/filepath"
"sort"
@ -32,6 +33,7 @@ type desiredStateLoader struct {
glob func(string) ([]string, error)
getHelm func(*state.HelmState) helmexec.Interface
remote *remote.Remote
logger *zap.SugaredLogger
valsRuntime vals.Evaluator
}
@ -46,7 +48,7 @@ func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, e
return nil, fmt.Errorf("bug: opts.CalleePath was nil: f=%s, opts=%v", f, opts)
}
storage := state.NewStorage(opts.CalleePath, ld.logger, ld.glob)
envld := state.NewEnvironmentValuesLoader(storage, ld.readFile, ld.logger)
envld := state.NewEnvironmentValuesLoader(storage, ld.readFile, ld.logger, ld.remote)
handler := state.MissingFileHandlerError
vals, err := envld.LoadEnvironmentValues(&handler, args)
if err != nil {
@ -147,7 +149,7 @@ func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *e
}
func (a *desiredStateLoader) underlying() *state.StateCreator {
c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.valsRuntime, a.getHelm, a.overrideHelmBinary)
c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.valsRuntime, a.getHelm, a.overrideHelmBinary, a.remote)
c.LoadFile = a.loadFile
return c
}

4
pkg/app/two_pass_renderer_test.go
View File

@ -1,6 +1,7 @@
package app
import (
"github.com/roboll/helmfile/pkg/remote"
"os"
"strings"
"testing"
@ -14,6 +15,8 @@ import (
func makeLoader(files map[string]string, env string) (*desiredStateLoader, *testhelper.TestFs) {
testfs := testhelper.NewTestFs(files)
logger := helmexec.NewLogger(os.Stdout, "debug")
r := remote.NewRemote(logger, testfs.Cwd, testfs.ReadFile, testfs.DirectoryExistsAt, testfs.FileExistsAt)
return &desiredStateLoader{
env: env,
namespace: "namespace",
@ -22,6 +25,7 @@ func makeLoader(files map[string]string, env string) (*desiredStateLoader, *test
fileExists: testfs.FileExists,
abs: testfs.Abs,
glob: testfs.Glob,
remote: r,
}, testfs
}

12
pkg/remote/remote.go
View File

@ -260,3 +260,15 @@ func (g *GoGetter) Get(wd, src, dst string) error {
return nil
}
func NewRemote(logger *zap.SugaredLogger, homeDir string, readFile func(string) ([]byte, error), dirExists func(string) bool, fileExists func(string) bool) *Remote {
remote := &Remote{
Logger: logger,
Home: homeDir,
Getter: &GoGetter{Logger: logger},
ReadFile: readFile,
DirExists: dirExists,
FileExists: fileExists,
}
return remote
}

15
pkg/state/create.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"github.com/roboll/helmfile/pkg/helmexec"
"github.com/roboll/helmfile/pkg/remote"
"io"
"os"
@ -52,9 +53,11 @@ type StateCreator struct {
getHelm func(*HelmState) helmexec.Interface
overrideHelmBinary string
remote *remote.Remote
}
func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), valsRuntime vals.Evaluator, getHelm func(*HelmState) helmexec.Interface, overrideHelmBinary string) *StateCreator {
func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), valsRuntime vals.Evaluator, getHelm func(*HelmState) helmexec.Interface, overrideHelmBinary string, remote *remote.Remote) *StateCreator {
return &StateCreator{
logger: logger,
readFile: readFile,
@ -66,6 +69,8 @@ func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error)
getHelm: getHelm,
overrideHelmBinary: overrideHelmBinary,
remote: remote,
}
}
@ -139,7 +144,7 @@ func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *envi
return nil, &StateLoadError{fmt.Sprintf("failed to read %s", state.FilePath), err}
}
e.Defaults, err = state.loadValuesEntries(nil, state.DefaultValues)
e.Defaults, err = state.loadValuesEntries(nil, state.DefaultValues, c.remote)
if err != nil {
return nil, err
}
@ -203,7 +208,7 @@ func (c *StateCreator) loadEnvValues(st *HelmState, name string, failOnMissingEn
envSpec, ok := st.Environments[name]
if ok {
var err error
envVals, err = st.loadValuesEntries(envSpec.MissingFileHandler, envSpec.Values)
envVals, err = st.loadValuesEntries(envSpec.MissingFileHandler, envSpec.Values, c.remote)
if err != nil {
return nil, err
}
@ -321,11 +326,11 @@ func (c *StateCreator) scatterGatherEnvSecretFiles(st *HelmState, envSecretFiles
return nil
}
func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []interface{}) (map[string]interface{}, error) {
func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []interface{}, remote *remote.Remote) (map[string]interface{}, error) {
envVals := map[string]interface{}{}
valuesEntries := append([]interface{}{}, entries...)
ld := NewEnvironmentValuesLoader(st.storage(), st.readFile, st.logger)
ld := NewEnvironmentValuesLoader(st.storage(), st.readFile, st.logger, remote)
var err error
envVals, err = ld.LoadEnvironmentValues(missingFileHandler, valuesEntries)
if err != nil {

5
pkg/state/create_test.go
View File

@ -1,6 +1,7 @@
package state
import (
"github.com/roboll/helmfile/pkg/remote"
"io/ioutil"
"path/filepath"
"reflect"
@ -108,7 +109,9 @@ bar: {{ readFile "bar.txt" }}
})
testFs.Cwd = "/example/path/to"
state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil, nil, "").ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", true, nil)
r := remote.NewRemote(logger, testFs.Cwd, testFs.ReadFile, testFs.DirectoryExistsAt, testFs.FileExistsAt)
state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil, nil, "", r).
ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", true, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}

11
pkg/state/envvals_loader.go
View File

@ -5,6 +5,7 @@ import (
"github.com/imdario/mergo"
"github.com/roboll/helmfile/pkg/environment"
"github.com/roboll/helmfile/pkg/maputil"
"github.com/roboll/helmfile/pkg/remote"
"github.com/roboll/helmfile/pkg/tmpl"
"go.uber.org/zap"
"gopkg.in/yaml.v2"
@ -17,13 +18,16 @@ type EnvironmentValuesLoader struct {
readFile func(string) ([]byte, error)
logger *zap.SugaredLogger
remote *remote.Remote
}
func NewEnvironmentValuesLoader(storage *Storage, readFile func(string) ([]byte, error), logger *zap.SugaredLogger) *EnvironmentValuesLoader {
func NewEnvironmentValuesLoader(storage *Storage, readFile func(string) ([]byte, error), logger *zap.SugaredLogger, remote *remote.Remote) *EnvironmentValuesLoader {
return &EnvironmentValuesLoader{
storage: storage,
readFile: readFile,
logger: logger,
remote: remote,
}
}
@ -36,6 +40,11 @@ func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *str
switch strOrMap := entry.(type) {
case string:
urlOrPath := strOrMap
localPath, err := ld.remote.Locate(urlOrPath)
if err == nil {
urlOrPath = localPath
}
files, skipped, err := ld.storage.resolveFile(missingFileHandler, "environment values", urlOrPath)
if err != nil {
return nil, err

6
pkg/state/envvals_loader_test.go
View File

@ -2,6 +2,7 @@ package state
import (
"github.com/google/go-cmp/cmp"
"github.com/roboll/helmfile/pkg/remote"
"go.uber.org/zap"
"io/ioutil"
"path/filepath"
@ -23,7 +24,10 @@ func newLoader() *EnvironmentValuesLoader {
logger: sugar,
}
return NewEnvironmentValuesLoader(storage, ioutil.ReadFile, sugar)
readFile := func(s string) ([]byte, error) { return []byte{}, nil }
dirExists := func(d string) bool { return false }
fileExists := func(f string) bool { return false }
return NewEnvironmentValuesLoader(storage, ioutil.ReadFile, sugar, remote.NewRemote(sugar, "/tmp", readFile, dirExists, fileExists))
}
// See https://github.com/roboll/helmfile/pull/1169