From 0fad9f0544f018984f557fb0d0ada6a5bb2efdd7 Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Fri, 11 Sep 2020 22:19:36 +0900 Subject: [PATCH] Add experimental write-values command for writing values files only (#1469) Ref #1460 --- main.go | 34 ++++++++++ pkg/app/app.go | 77 ++++++++++++++++++++++ pkg/app/config.go | 7 ++ pkg/state/state.go | 160 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 278 insertions(+) diff --git a/main.go b/main.go index 96603f5e..938c4124 100644 --- a/main.go +++ b/main.go @@ -257,6 +257,36 @@ func main() { return run.Template(c) }), }, + { + Name: "write-values", + Usage: "write values files for releases. Similar to `helmfile template`, write values files instead of manifests.", + Flags: []cli.Flag{ + cli.StringSliceFlag{ + Name: "set", + Usage: "additional values to be merged into the command", + }, + cli.StringSliceFlag{ + Name: "values", + Usage: "additional value files to be merged into the command", + }, + cli.StringFlag{ + Name: "output-file-template", + Usage: "go text template for generating the output file. Default: {{ .State.BaseName }}-{{ .State.AbsPathSHA1 }}/{{ .Release.Name}}.yaml", + }, + cli.IntFlag{ + Name: "concurrency", + Value: 0, + Usage: "maximum number of concurrent downloads of release charts", + }, + cli.BoolFlag{ + Name: "skip-deps", + Usage: "skip running `helm repo update` and `helm dependency build`", + }, + }, + Action: action(func(run *app.App, c configImpl) error { + return run.WriteValues(c) + }), + }, { Name: "lint", Usage: "lint charts from state file (helm lint)", @@ -574,6 +604,10 @@ func (c configImpl) OutputDirTemplate() string { return c.c.String("output-dir-template") } +func (c configImpl) OutputFileTemplate() string { + return c.c.String("output-file-template") +} + func (c configImpl) Validate() bool { return c.c.Bool("validate") } diff --git a/pkg/app/app.go b/pkg/app/app.go index 352e4c2c..b60bc647 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -236,6 +236,25 @@ func (a *App) Template(c TemplateConfigProvider) error { }, SetFilter(true)) } +func (a *App) WriteValues(c WriteValuesConfigProvider) error { + return a.ForEachState(func(run *Run) (ok bool, errs []error) { + // `helm template` in helm v2 does not support local chart. + // So, we set forceDownload=true for helm v2 only + prepErr := run.withPreparedCharts("write-values", state.ChartPrepareOptions{ + ForceDownload: !run.helm.IsHelm3(), + SkipRepos: c.SkipDeps(), + }, func() { + ok, errs = a.writeValues(run, c) + }) + + if prepErr != nil { + errs = append(errs, prepErr) + } + + return + }, SetFilter(true)) +} + func (a *App) Lint(c LintConfigProvider) error { return a.ForEachState(func(run *Run) (_ bool, errs []error) { // `helm lint` on helm v2 and v3 does not support remote charts, that we need to set `forceDownload=true` here @@ -1418,6 +1437,64 @@ func (a *App) template(r *Run, c TemplateConfigProvider) (bool, []error) { return true, errs } +func (a *App) writeValues(r *Run, c WriteValuesConfigProvider) (bool, []error) { + st := r.state + helm := r.helm + + allReleases := st.GetReleasesWithOverrides() + + toRender, err := a.getSelectedReleases(r) + if err != nil { + return false, []error{err} + } + if len(toRender) == 0 { + return false, nil + } + + // Do build deps and prepare only on selected releases so that we won't waste time + // on running various helm commands on unnecessary releases + st.Releases = toRender + + releasesToWrite := map[string]state.ReleaseSpec{} + for _, r := range toRender { + id := state.ReleaseToID(&r) + if r.Installed != nil && !*r.Installed { + continue + } + releasesToWrite[id] = r + } + + var errs []error + + // Traverse DAG of all the releases so that we don't suffer from false-positive missing dependencies + st.Releases = allReleases + + if len(releasesToWrite) > 0 { + _, writeErrs := withDAG(st, helm, a.Logger, false, a.Wrap(func(subst *state.HelmState, helm helmexec.Interface) []error { + var rs []state.ReleaseSpec + + for _, r := range subst.Releases { + if r2, ok := releasesToWrite[state.ReleaseToID(&r)]; ok { + rs = append(rs, r2) + } + } + + subst.Releases = rs + + opts := &state.WriteValuesOpts{ + Set: c.Set(), + OutputFileTemplate: c.OutputFileTemplate(), + } + return subst.WriteReleasesValues(helm, c.Values(), opts) + })) + + if writeErrs != nil && len(writeErrs) > 0 { + errs = append(errs, writeErrs...) + } + } + return true, errs +} + func fileExistsAt(path string) bool { fileInfo, err := os.Stat(path) return err == nil && fileInfo.Mode().IsRegular() diff --git a/pkg/app/config.go b/pkg/app/config.go index 4a44542f..10cba2c1 100644 --- a/pkg/app/config.go +++ b/pkg/app/config.go @@ -137,6 +137,13 @@ type TemplateConfigProvider interface { concurrencyConfig } +type WriteValuesConfigProvider interface { + Values() []string + Set() []string + OutputFileTemplate() string + SkipDeps() bool +} + type StatusesConfigProvider interface { Args() string diff --git a/pkg/state/state.go b/pkg/state/state.go index 256fa558..4eddbd04 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "github.com/imdario/mergo" "golang.org/x/sync/errgroup" "io" "io/ioutil" @@ -1225,6 +1226,103 @@ func (st *HelmState) TemplateReleases(helm helmexec.Interface, outputDir string, return nil } +type WriteValuesOpts struct { + Set []string + OutputFileTemplate string +} + +type WriteValuesOpt interface{ Apply(*WriteValuesOpts) } + +func (o *WriteValuesOpts) Apply(opts *WriteValuesOpts) { + *opts = *o +} + +// WriteReleasesValues writes values files for releases +func (st *HelmState) WriteReleasesValues(helm helmexec.Interface, additionalValues []string, opt ...WriteValuesOpt) []error { + opts := &WriteValuesOpts{} + for _, o := range opt { + o.Apply(opts) + } + + for i := range st.Releases { + release := &st.Releases[i] + + if !release.Desired() { + continue + } + + st.ApplyOverrides(release) + + generatedFiles, err := st.generateValuesFiles(helm, release, i) + if err != nil { + return []error{err} + } + + defer func() { + st.removeFiles(generatedFiles) + }() + + for _, value := range additionalValues { + valfile, err := filepath.Abs(value) + if err != nil { + return []error{err} + } + + if _, err := os.Stat(valfile); os.IsNotExist(err) { + return []error{err} + } + generatedFiles = append(generatedFiles, valfile) + } + + outputValuesFile, err := st.GenerateOutputFilePath(release, opts.OutputFileTemplate) + if err != nil { + return []error{err} + } + + if err := os.MkdirAll(filepath.Dir(outputValuesFile), 0755); err != nil { + return []error{err} + } + + st.logger.Infof("Writing values file %s", outputValuesFile) + + merged := map[string]interface{}{} + + for _, f := range generatedFiles { + src := map[string]interface{}{} + + srcBytes, err := st.readFile(f) + if err != nil { + return []error{fmt.Errorf("reading %s: %w", f, err)} + } + + if err := yaml.Unmarshal(srcBytes, &src); err != nil { + return []error{fmt.Errorf("unmarshalling yaml %s: %w", f, err)} + } + + if err := mergo.Merge(&merged, &src, mergo.WithOverride, mergo.WithOverwriteWithEmptyValue); err != nil { + return []error{fmt.Errorf("merging %s: %w", f, err)} + } + } + + var buf bytes.Buffer + + y := yaml.NewEncoder(&buf) + if err := y.Encode(merged); err != nil { + return []error{err} + } + + if err := ioutil.WriteFile(outputValuesFile, buf.Bytes(), 0644); err != nil { + return []error{fmt.Errorf("writing values file %s: %w", outputValuesFile, err)} + } + + if _, err := st.TriggerCleanupEvent(release, "write-values"); err != nil { + st.logger.Warnf("warn: %v\n", err) + } + } + + return nil +} + type LintOpts struct { Set []string } @@ -2580,6 +2678,68 @@ func (st *HelmState) GenerateOutputDir(outputDir string, release *ReleaseSpec, o return buf.String(), nil } +func (st *HelmState) GenerateOutputFilePath(release *ReleaseSpec, outputFileTemplate string) (string, error) { + // get absolute path of state file to generate a hash + // use this hash to write helm output in a specific directory by state file and release name + // ie. in a directory named stateFileName-stateFileHash-releaseName + stateAbsPath, err := filepath.Abs(st.FilePath) + if err != nil { + return stateAbsPath, err + } + + hasher := sha1.New() + io.WriteString(hasher, stateAbsPath) + + var stateFileExtension = filepath.Ext(st.FilePath) + var stateFileName = st.FilePath[0 : len(st.FilePath)-len(stateFileExtension)] + + sha1sum := hex.EncodeToString(hasher.Sum(nil))[:8] + + var sb strings.Builder + sb.WriteString(stateFileName) + sb.WriteString("-") + sb.WriteString(sha1sum) + sb.WriteString("-") + sb.WriteString(release.Name) + + if outputFileTemplate == "" { + outputFileTemplate = filepath.Join("{{ .State.BaseName }}-{{ .State.AbsPathSHA1 }}", "{{ .Release.Name}}.yaml") + } + + t, err := template.New("output-file").Parse(outputFileTemplate) + if err != nil { + return "", fmt.Errorf("parsing output-file templmate") + } + + buf := &bytes.Buffer{} + + type state struct { + BaseName string + Path string + AbsPath string + AbsPathSHA1 string + } + + data := struct { + State state + Release *ReleaseSpec + }{ + State: state{ + BaseName: stateFileName, + Path: st.FilePath, + AbsPath: stateAbsPath, + AbsPathSHA1: sha1sum, + }, + Release: release, + } + + if err := t.Execute(buf, data); err != nil { + return "", fmt.Errorf("executing output-file template: %w", err) + } + + return buf.String(), nil +} + func (st *HelmState) ToYaml() (string, error) { if result, err := yaml.Marshal(st); err != nil { return "", err