From 5b7cbe54e9863d800aec478f81f6bcd370be5fe5 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 8 Jan 2026 14:28:25 -0500 Subject: [PATCH] Add --sequential-helmfiles flag for optional sequential processing (#2350) * Initial plan * Fix helmfile.d parallel processing to respect alphabetical order Changed multiple helmfile.d file processing from parallel to sequential to match the documented behavior. Files are now processed in alphabetical order as stated in the documentation, ensuring predictable deployment order. This fixes issues where deployments would break because parallel processing did not respect the intended ordering (e.g., databases before applications). - Modified visitStatesWithContext to process files sequentially - Removed unused processStateFileParallel function - Updated test snapshots to reflect deterministic ordering Co-authored-by: yxxhero <11087727+yxxhero@users.noreply.github.com> * Add --sequential-helmfiles flag for opt-in sequential processing Based on feedback, added a flag to control processing mode instead of forcing sequential behavior. Parallel processing is now the default for performance, with an opt-in --sequential-helmfiles flag for when order matters. Changes: - Added SequentialHelmfiles field to GlobalOptions and App - Added --sequential-helmfiles CLI flag - Modified visitStatesWithContext to check flag and choose parallel vs sequential - Updated documentation to clarify parallel is default, sequential is opt-in - Restored original parallel processing as default behavior This gives users control over the behavior based on their needs. Co-authored-by: yxxhero <11087727+yxxhero@users.noreply.github.com> * Add documentation and improve code readability - Added documentation to processStateFileParallel function - Extracted shouldProcessInParallel variable for clarity - All tests pass, security scan clean Co-authored-by: yxxhero <11087727+yxxhero@users.noreply.github.com> * Fix lint issue: remove trailing whitespace Removed trailing whitespace from line 983 in pkg/app/app.go to fix formatting lint error. Co-authored-by: yxxhero <11087727+yxxhero@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: yxxhero <11087727+yxxhero@users.noreply.github.com> --- cmd/root.go | 1 + docs/index.md | 13 ++++++++++--- pkg/app/app.go | 13 +++++++++++-- pkg/app/config.go | 1 + pkg/config/global.go | 7 +++++++ 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index eedf844d..d702718a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -144,6 +144,7 @@ The name of a release can be used as a label: "--selector name=myrelease"`) fs.BoolVar(&globalOptions.EnableLiveOutput, "enable-live-output", globalOptions.EnableLiveOutput, `Show live output from the Helm binary Stdout/Stderr into Helmfile own Stdout/Stderr. It only applies for the Helm CLI commands, Stdout/Stderr for Hooks are still displayed only when it's execution finishes.`) fs.BoolVarP(&globalOptions.Interactive, "interactive", "i", false, "Request confirmation before attempting to modify clusters") + fs.BoolVar(&globalOptions.SequentialHelmfiles, "sequential-helmfiles", false, "Process helmfile.d files sequentially in alphabetical order instead of in parallel. Useful when file order matters for dependencies.") // avoid 'pflag: help requested' error (#251) fs.BoolP("help", "h", false, "help for helmfile") } diff --git a/docs/index.md b/docs/index.md index 42877ea3..84948bd6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1338,12 +1338,19 @@ And there are two ways to organize your files. The default helmfile directory is `helmfile.d`, that is, in case helmfile is unable to locate `helmfile.yaml`, it tries to locate `helmfile.d/*.yaml`. -All the yaml files under the specified directory are processed in the alphabetical order. For example, you can use a `-.yaml` naming convention to control the sync order. +By default, multiple files in `helmfile.d` are processed in **parallel** for better performance. If you need files to be processed **sequentially in alphabetical order** (e.g., for dependency ordering where databases must be deployed before applications), use the `--sequential-helmfiles` flag. + +For example, you can use a `-.yaml` naming convention to control the sync order when using `--sequential-helmfiles`: * `helmfile.d`/ * `00-database.yaml` - * `00-backend.yaml` - * `01-frontend.yaml` + * `01-backend.yaml` + * `02-frontend.yaml` + +```bash +# Process files sequentially in alphabetical order +helmfile --sequential-helmfiles sync +``` ### Glob patterns diff --git a/pkg/app/app.go b/pkg/app/app.go index 6dffe8ed..798e46a1 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -38,6 +38,7 @@ type App struct { EnforcePluginVerification bool HelmOCIPlainHTTP bool DisableKubeVersionAutoDetection bool + SequentialHelmfiles bool Logger *zap.SugaredLogger Kubeconfig string @@ -86,6 +87,7 @@ func New(conf ConfigProvider) *App { DisableForceUpdate: conf.DisableForceUpdate(), EnforcePluginVerification: conf.EnforcePluginVerification(), HelmOCIPlainHTTP: conf.HelmOCIPlainHTTP(), + SequentialHelmfiles: conf.SequentialHelmfiles(), Logger: conf.Logger(), Kubeconfig: conf.Kubeconfig(), Env: conf.Env(), @@ -855,6 +857,9 @@ func (a *App) visitStates(fileOrDir string, defOpts LoadOpts, converge func(*sta return a.visitStatesWithContext(fileOrDir, defOpts, converge, nil) } +// processStateFileParallel processes a single helmfile state file in a goroutine. +// It is used for parallel processing of multiple helmfile.d files. +// Results are communicated via errChan (errors) and matchChan (whether file had matching releases). func (a *App) processStateFileParallel(relPath string, defOpts LoadOpts, converge func(*state.HelmState) (bool, []error), sharedCtx *Context, errChan chan error, matchChan chan bool) { var file string var dir string @@ -974,7 +979,11 @@ func (a *App) visitStatesWithContext(fileOrDir string, defOpts LoadOpts, converg desiredStateFiles, err := a.findDesiredStateFiles(fileOrDir, defOpts) - if len(desiredStateFiles) > 1 { + // Process files in parallel if we have multiple files and parallel mode is enabled + shouldProcessInParallel := len(desiredStateFiles) > 1 && !a.SequentialHelmfiles + + if shouldProcessInParallel { + // Parallel processing for multiple files (default behavior) var wg sync.WaitGroup errChan := make(chan error, len(desiredStateFiles)) matchChan := make(chan bool, len(desiredStateFiles)) @@ -1002,7 +1011,7 @@ func (a *App) visitStatesWithContext(fileOrDir string, defOpts LoadOpts, converg noMatchInHelmfiles = false } } else { - // Sequential processing for single file + // Sequential processing for single file or when --sequential-helmfiles is set err = a.visitStateFiles(fileOrDir, defOpts, func(f, d string) (retErr error) { opts := defOpts.DeepCopy() diff --git a/pkg/app/config.go b/pkg/app/config.go index f8a070c1..ea22c3d5 100644 --- a/pkg/app/config.go +++ b/pkg/app/config.go @@ -13,6 +13,7 @@ type ConfigProvider interface { HelmOCIPlainHTTP() bool SkipDeps() bool SkipRefresh() bool + SequentialHelmfiles() bool FileOrDir() string KubeContext() string diff --git a/pkg/config/global.go b/pkg/config/global.go index 0cf58f96..f17ea2d9 100644 --- a/pkg/config/global.go +++ b/pkg/config/global.go @@ -74,6 +74,8 @@ type GlobalOptions struct { Args string // LogOutput is the writer to use for writing logs. LogOutput io.Writer + // SequentialHelmfiles is true if helmfile.d files should be processed sequentially instead of in parallel. + SequentialHelmfiles bool } // Logger returns the logger to use. @@ -205,6 +207,11 @@ func (g *GlobalImpl) HelmOCIPlainHTTP() bool { return g.GlobalOptions.HelmOCIPlainHTTP } +// SequentialHelmfiles returns whether to process helmfile.d files sequentially +func (g *GlobalImpl) SequentialHelmfiles() bool { + return g.GlobalOptions.SequentialHelmfiles +} + // Logger returns the logger func (g *GlobalImpl) Logger() *zap.SugaredLogger { return g.logger