feat: add --repo-retries for helm repo and registry login commands (#2683)

* feat: add --repo-retries for retrying helm repo and registry login commands Add a configurable retry mechanism for chart repository operations to handle unstable networks (corporate proxies, slow internal registries). Closes #1894 - New --repo-retries N flag and HELMFILE_REPO_RETRIES env var (default 0 = opt-in, backward compatible) - Retry applies to helm repo add, helm repo update (incl. ACR), and helm registry login with exponential backoff (1s, 2s, 4s, ..., capped 30s) - Single retryRepoOp helper; per-attempt args/buffer are local to avoid state leaking across retries - Tests cover succeed-after-retry, exhausted-retries, disabled-by-default, and regression guards for password-buffer and args-accumulation Signed-off-by: yxxhero <aiopsclub@163.com> * fix: address PR review (overflow guard, cancellable sleep, flag-override, docs) Address Copilot review feedback on #2683: - Cap backoff shift exponent at 5 to prevent time.Duration overflow on large --repo-retries values - Make retry sleep context-aware (sleepCtx) so Ctrl+C aborts the retry loop promptly via the ShellRunner context - Log a concise exit status instead of the verbose ExitError dump, and clarify the retry-counter wording ('retry N/M') - Use -1 sentinel as the CLI default so --repo-retries=0 can explicitly disable retries even when HELMFILE_REPO_RETRIES is set - Align help text and docs: retry applies 'on failure' (not just transient errors), document the 0-disables behavior - Add tests for overflow guard, cancellable sleep, and flag-zero-disables Signed-off-by: yxxhero <aiopsclub@163.com> * fix: abort retries on canceled context, hide sentinel default, align comment Address follow-up Copilot review on #2683: - Fix tight-loop bug: sleepCtx now returns whether it completed vs was interrupted by context cancellation, and retryRepoOp aborts the retry loop on interruption so Ctrl+C no longer spins into rapid helm calls - Hide the -1 sentinel from --help by overriding the displayed default to 0 (pflag DefValue), matching the documented default while keeping the flag-override semantics - Correct HelmExecOptions.RepoRetry comment: 'on failure' not 'transient network errors', matching the actual retry behavior - Add Test_Retry_AbortsOnCanceledContext covering the no-tight-loop path Signed-off-by: yxxhero <aiopsclub@163.com> * fix: copy args per retry in RegistryLogin, make cancel test deterministic Address follow-up Copilot review on #2683: - RegistryLogin: pass a per-attempt copy of args to execStdIn so its internal append (for helm.extra) can't alias the shared slice across retries - Test_Retry_AbortsOnCanceledContext: cancel the context deterministically inside the op closure after the first attempt, replacing the flaky time.Sleep(20ms) goroutine Signed-off-by: yxxhero <aiopsclub@163.com> * fix: return error on unknown managed repo type instead of silent skip Address Copilot review on #2683: AddRepo logged an error for an unknown managed type but returned nil, silently succeeding while skipping the repo add. Now returns an error so misconfigurations fail loudly. Signed-off-by: yxxhero <aiopsclub@163.com> --------- Signed-off-by: yxxhero <aiopsclub@163.com>
2026-10-09 20:56:06 +02:00 · 2026-07-23 18:10:04 +08:00
parent f64f3ec197
commit 7bebfab71a
9 changed files with 425 additions and 41 deletions
@@ -3,6 +3,7 @@ package helmexec
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/url"
@@ -12,6 +13,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 	"unicode"

 	"github.com/Masterminds/semver/v3"
@@ -38,6 +40,9 @@ type HelmExecOptions struct {
 	DisableForceUpdate        bool // If true, do not force helm repos to update when executing "helm repo add" (Helm 3)
 	EnforcePluginVerification bool // If true, fail plugin installation if verification is not supported
 	HelmOCIPlainHTTP          bool // If true, use plain HTTP for OCI registries
+	// RepoRetry is the number of times to retry helm repo and registry login
+	// operations on failure, with exponential backoff. 0 disables retries.
+	RepoRetry int
 }

 type execer struct {
@@ -272,11 +277,91 @@ func (helm *execer) SetDisableForceUpdate(forceUpdate bool) {
 	helm.options.DisableForceUpdate = forceUpdate
 }

-func (helm *execer) AddRepo(name, repository, cafile, certfile, keyfile, username, password string, managed string, passCredentials, skipTLSVerify bool) error {
-	var args []string
+// repoRetryBaseBackoff is the base unit for exponential backoff between repo
+// operation retries (doubles each attempt, capped at 30x). Exposed as a
+// package-level variable so tests can shrink it to avoid real sleeps.
+var repoRetryBaseBackoff = time.Second
+
+// retryRepoOp runs op, retrying up to helm.options.RepoRetry times on failure
+// with exponential backoff (1x, 2x, 4x, ..., capped at 30x the base unit). It
+// returns the output from the (last) attempt. A zero/negative RepoRetry
+// disables retrying — op runs exactly once.
+func (helm *execer) retryRepoOp(name string, op func() ([]byte, error)) ([]byte, error) {
+	maxRetries := helm.options.RepoRetry
 	var out []byte
 	var err error
+	for attempt := 0; ; attempt++ {
+		out, err = op()
+		if err == nil || maxRetries <= 0 || attempt >= maxRetries {
+			return out, err
+		}
+		// Cap the shift exponent at 5 (2^5 = 32x already exceeds the 30x cap)
+		// so very large --repo-retries values can't overflow time.Duration.
+		shift := attempt
+		if shift > 5 {
+			shift = 5
+		}
+		backoff := repoRetryBaseBackoff * time.Duration(1<<shift)
+		if backoff > 30*repoRetryBaseBackoff {
+			backoff = 30 * repoRetryBaseBackoff
+		}
+		helm.logger.Warnf("repo operation %q failed (%s); retry %d/%d in %v",
+			name, conciseError(err), attempt+1, maxRetries, backoff)
+		// sleepCtx returns false if interrupted by context cancellation; in that
+		// case stop retrying so a canceled context (Ctrl+C) doesn't spin into a
+		// tight loop of rapid helm invocations.
+		if !helm.sleepCtx(backoff) {
+			return out, err
+		}
+	}
+}

+// conciseError returns a short reason for logging. For a helm ExitError it
+// reports just the exit status, avoiding the very verbose PATH/ARGS/OUTPUT
+// dump that Error() produces.
+func conciseError(err error) string {
+	var ee ExitError
+	if errors.As(err, &ee) {
+		return fmt.Sprintf("exit status %d", ee.ExitStatus())
+	}
+	return err.Error()
+}
+
+// sleepCtx sleeps for d, returning early if the runner's context is canceled,
+// so an interrupt (Ctrl+C) aborts the retry loop promptly rather than blocking
+// until the full backoff elapses. It returns true if the full duration elapsed,
+// or false if it was interrupted by context cancellation. Falls back to
+// time.Sleep (returning true) for runners without a context (e.g. the test
+// mockRunner).
+func (helm *execer) sleepCtx(d time.Duration) bool {
+	ctx := helm.runnerContext()
+	if ctx == nil {
+		time.Sleep(d)
+		return true
+	}
+	timer := time.NewTimer(d)
+	defer timer.Stop()
+	select {
+	case <-timer.C:
+		return true
+	case <-ctx.Done():
+		return false
+	}
+}
+
+// runnerContext returns the ShellRunner's context if the runner is a
+// ShellRunner (pointer or value), else nil.
+func (helm *execer) runnerContext() context.Context {
+	switch r := helm.runner.(type) {
+	case *ShellRunner:
+		return r.Ctx
+	case ShellRunner:
+		return r.Ctx
+	}
+	return nil
+}
+
+func (helm *execer) AddRepo(name, repository, cafile, certfile, keyfile, username, password string, managed string, passCredentials, skipTLSVerify bool) error {
 	if name == "" && repository != "" {
 		helm.logger.Infof("empty field name\n")
 		return fmt.Errorf("empty field name")
@@ -288,46 +373,52 @@ func (helm *execer) AddRepo(name, repository, cafile, certfile, keyfile, usernam
 		helm.extra = savedExtra
 	}()

+	var out []byte
+	var err error
 	switch managed {
 	case "acr":
 		helm.logger.Infof("Adding repo %v (acr)", name)
-		out, err = helm.azcli(name)
+		out, err = helm.retryRepoOp(fmt.Sprintf("add %s (acr)", name), func() ([]byte, error) {
+			return helm.azcli(name)
+		})
 	case "":
-		args = append(args, "repo", "add", name, repository)
-
-		// --force-update is needed for both Helm 3.3.2+ and Helm 4
-		// to ensure repository indexes are updated when a repository already exists
-		// See https://github.com/helm/helm/pull/8777
-		if !helm.options.DisableForceUpdate && (helm.IsHelm4() || helm.IsVersionAtLeast("3.3.2")) {
-			args = append(args, "--force-update")
-		}
-
-		if certfile != "" && keyfile != "" {
-			args = append(args, "--cert-file", certfile, "--key-file", keyfile)
-		}
-		if cafile != "" {
-			args = append(args, "--ca-file", cafile)
-		}
-
-		if passCredentials {
-			args = append(args, "--pass-credentials")
-		}
-		if skipTLSVerify {
-			args = append(args, "--insecure-skip-tls-verify")
-		}
 		helm.logger.Infof("Adding repo %v %v", name, repository)
-		if username != "" && password != "" {
-			args = append(args, "--username", username, "--password-stdin")
-			buffer := bytes.Buffer{}
-			fmt.Fprintf(&buffer, "%s\n", password)
-			out, err = helm.execStdIn(args, map[string]string{}, &buffer)
-		} else {
-			out, err = helm.exec(args, map[string]string{}, nil)
-		}
+		out, err = helm.retryRepoOp(fmt.Sprintf("add %s", name), func() ([]byte, error) {
+			// args is local to each attempt, so the username/password append
+			// below can't accumulate across retries.
+			args := []string{"repo", "add", name, repository}
+
+			// --force-update is needed for both Helm 3.3.2+ and Helm 4
+			// to ensure repository indexes are updated when a repository already exists
+			// See https://github.com/helm/helm/pull/8777
+			if !helm.options.DisableForceUpdate && (helm.IsHelm4() || helm.IsVersionAtLeast("3.3.2")) {
+				args = append(args, "--force-update")
+			}
+
+			if certfile != "" && keyfile != "" {
+				args = append(args, "--cert-file", certfile, "--key-file", keyfile)
+			}
+			if cafile != "" {
+				args = append(args, "--ca-file", cafile)
+			}
+
+			if passCredentials {
+				args = append(args, "--pass-credentials")
+			}
+			if skipTLSVerify {
+				args = append(args, "--insecure-skip-tls-verify")
+			}
+			if username != "" && password != "" {
+				args = append(args, "--username", username, "--password-stdin")
+				buffer := bytes.Buffer{}
+				fmt.Fprintf(&buffer, "%s\n", password)
+				return helm.execStdIn(args, map[string]string{}, &buffer)
+			}
+			return helm.exec(args, map[string]string{}, nil)
+		})
 	default:
 		helm.logger.Errorf("ERROR: unknown type '%v' for repository %v", managed, name)
-		out = nil
-		err = nil
+		err = fmt.Errorf("unknown managed type %q for repository %v", managed, name)
 	}

 	helm.info(out)
@@ -341,7 +432,9 @@ func (helm *execer) UpdateRepo() error {
 	defer func() {
 		helm.extra = savedExtra
 	}()
-	out, err := helm.exec([]string{"repo", "update"}, map[string]string{}, nil)
+	out, err := helm.retryRepoOp("update", func() ([]byte, error) {
+		return helm.exec([]string{"repo", "update"}, map[string]string{}, nil)
+	})
 	helm.info(out)
 	return err
 }
@@ -373,11 +466,15 @@ func (helm *execer) RegistryLogin(repository, username, password, caFile, certFi
 	}

 	args = append(args, "--username", username, "--password-stdin")
-	buffer := bytes.Buffer{}
-	fmt.Fprintf(&buffer, "%s\n", password)

 	helm.logger.Info("Logging in to registry")
-	out, err := helm.execStdIn(args, map[string]string{"HELM_EXPERIMENTAL_OCI": "1"}, &buffer)
+	out, err := helm.retryRepoOp(fmt.Sprintf("registry login %s", repository), func() ([]byte, error) {
+		buffer := bytes.Buffer{}
+		fmt.Fprintf(&buffer, "%s\n", password)
+		// Copy args so execStdIn's internal append (for helm.extra) can't alias
+		// the shared slice across retries.
+		return helm.execStdIn(append([]string{}, args...), map[string]string{"HELM_EXPERIMENTAL_OCI": "1"}, &buffer)
+	})
 	helm.info(out)
 	return err
 }