From a584aeab2e2ae31e0d68e93d50867d1597c38b50 Mon Sep 17 00:00:00 2001 From: Travis Groth Date: Wed, 14 Aug 2019 20:27:55 -0400 Subject: [PATCH] Share helmexec from State Creation (#804) Closes #444 and #782 This is the final PR to fully cache and parallelize helm secret decryption. It threads the shared helmexec.Interface into the StateCreator and HelmState structs to be used during environment secret decryption. This should effectively cache secrets for the duration of a helmfile run, regardless of where they are first decrypted. --- pkg/app/app.go | 8 +++++--- pkg/app/desired_state_file_loader.go | 13 ++++++++----- pkg/state/create.go | 13 +++++++------ pkg/state/create_test.go | 7 ++++--- pkg/state/state.go | 1 + 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/pkg/app/app.go b/pkg/app/app.go index 72c7a88c..aa6154de 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -2,9 +2,6 @@ package app import ( "fmt" - "github.com/roboll/helmfile/pkg/helmexec" - "github.com/roboll/helmfile/pkg/remote" - "github.com/roboll/helmfile/pkg/state" "io/ioutil" "log" "os" @@ -12,6 +9,10 @@ import ( "strings" "syscall" + "github.com/roboll/helmfile/pkg/helmexec" + "github.com/roboll/helmfile/pkg/remote" + "github.com/roboll/helmfile/pkg/state" + "go.uber.org/zap" "path/filepath" @@ -239,6 +240,7 @@ func (a *App) loadDesiredStateFromYaml(file string, opts ...LoadOpts) (*state.He Reverse: a.Reverse, KubeContext: a.KubeContext, glob: a.glob, + helm: a.helmExecer, } var op LoadOpts diff --git a/pkg/app/desired_state_file_loader.go b/pkg/app/desired_state_file_loader.go index 2af933d7..6264aad8 100644 --- a/pkg/app/desired_state_file_loader.go +++ b/pkg/app/desired_state_file_loader.go @@ -4,12 +4,14 @@ import ( "bytes" "errors" "fmt" - "github.com/imdario/mergo" - "github.com/roboll/helmfile/pkg/environment" - "github.com/roboll/helmfile/pkg/state" - "go.uber.org/zap" "path/filepath" "sort" + + "github.com/imdario/mergo" + "github.com/roboll/helmfile/pkg/environment" + "github.com/roboll/helmfile/pkg/helmexec" + "github.com/roboll/helmfile/pkg/state" + "go.uber.org/zap" ) type desiredStateLoader struct { @@ -25,6 +27,7 @@ type desiredStateLoader struct { glob func(string) ([]string, error) logger *zap.SugaredLogger + helm helmexec.Interface } func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, error) { @@ -125,7 +128,7 @@ func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *e } func (a *desiredStateLoader) underlying() *state.StateCreator { - c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob) + c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.helm) c.LoadFile = a.loadFile return c } diff --git a/pkg/state/create.go b/pkg/state/create.go index be3ef4ad..d798c61e 100644 --- a/pkg/state/create.go +++ b/pkg/state/create.go @@ -38,13 +38,14 @@ type StateCreator struct { fileExists func(string) (bool, error) abs func(string) (string, error) glob func(string) ([]string, error) + helm helmexec.Interface Strict bool LoadFile func(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*HelmState, error) } -func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error)) *StateCreator { +func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), helm helmexec.Interface) *StateCreator { return &StateCreator{ logger: logger, readFile: readFile, @@ -52,6 +53,7 @@ func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error) abs: abs, glob: glob, Strict: true, + helm: helm, } } @@ -61,6 +63,7 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState, state.FilePath = file state.basePath = baseDir + state.helm = c.helm decoder := yaml.NewDecoder(bytes.NewReader(content)) if !c.Strict { @@ -186,9 +189,6 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, } if len(envSpec.Secrets) > 0 { - helm := helmexec.New(st.logger, "", &helmexec.ShellRunner{ - Logger: st.logger, - }) var envSecretFiles []string for _, urlOrPath := range envSpec.Secrets { @@ -202,7 +202,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, envSecretFiles = append(envSecretFiles, resolved...) } - if err = st.scatterGatherEnvSecretFiles(envSecretFiles, helm, envVals, readFile); err != nil { + if err = st.scatterGatherEnvSecretFiles(envSecretFiles, envVals, readFile); err != nil { return nil, err } } @@ -225,7 +225,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, return newEnv, nil } -func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, helm helmexec.Interface, envVals map[string]interface{}, readFile func(string) ([]byte, error)) error { +func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, envVals map[string]interface{}, readFile func(string) ([]byte, error)) error { var errs []error inputs := envSecretFiles @@ -239,6 +239,7 @@ func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, helm h secrets := make(chan string, inputsSize) results := make(chan secretResult, inputsSize) + helm := st.helm st.scatterGather(0, inputsSize, func() { diff --git a/pkg/state/create_test.go b/pkg/state/create_test.go index 76863a3c..3a8ca395 100644 --- a/pkg/state/create_test.go +++ b/pkg/state/create_test.go @@ -1,13 +1,14 @@ package state import ( - "github.com/roboll/helmfile/pkg/testhelper" - "go.uber.org/zap" "io/ioutil" "path/filepath" "reflect" "testing" + "github.com/roboll/helmfile/pkg/testhelper" + "go.uber.org/zap" + . "gotest.tools/assert" "gotest.tools/assert/cmp" ) @@ -107,7 +108,7 @@ bar: {{ readFile "bar.txt" }} }) testFs.Cwd = "/example/path/to" - state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", false, nil) + state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", false, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/state/state.go b/pkg/state/state.go index a6ae762e..bc01822d 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -62,6 +62,7 @@ type HelmState struct { tempDir func(string, string) (string, error) runner helmexec.Runner + helm helmexec.Interface } // SubHelmfileSpec defines the subhelmfile path and options