mirror of
https://github.com/helmfile/helmfile.git
synced 2026-09-30 10:31:24 +02:00
fix: clean up chartify temp directories after helm operations (#1799) Chartify creates temporary output directories (e.g. /tmp/chartify<random>/) during chart preparation for commands like build, template, and diff. These directories were never tracked for cleanup, causing disk space to accumulate over time with thousands of orphaned chartify* folders. The existing clean() closure in PrepareChartify only removed generated values files, not the chartify output directory itself. The chartified chart must survive until all helm operations complete, so it could not be removed during chart preparation. This change: - Adds chartifyTempDirTracker to HelmState (pointer-based to avoid copy-lock issues from HelmState being copied in several places) - Tracks chartify output dirs via addChartifyTempDir() after c.Chartify() succeeds in processChartification - Cleans them up via CleanupChartifyTempDirs() in WithPreparedCharts after all helm operations complete - Also removes empty parent temp directories (e.g. /tmp/chartify<random>/) Fixes #1799 Signed-off-by: yxxhero <aiopsclub@163.com>
6054 lines
201 KiB
Go
6054 lines
201 KiB
Go
package state
|
|
|
|
import (
|
|
"bytes"
|
|
gocontext "context"
|
|
"crypto/sha1"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"text/template"
|
|
"time"
|
|
|
|
"dario.cat/mergo"
|
|
"github.com/Masterminds/semver/v3"
|
|
"github.com/gofrs/flock"
|
|
"github.com/helmfile/chartify"
|
|
"github.com/helmfile/vals"
|
|
"github.com/tatsushid/go-prettytable"
|
|
"go.uber.org/zap"
|
|
helmchart "helm.sh/helm/v3/pkg/chart"
|
|
cliv3 "helm.sh/helm/v3/pkg/cli"
|
|
cliv4 "helm.sh/helm/v4/pkg/cli"
|
|
|
|
"github.com/helmfile/helmfile/pkg/agent/llm"
|
|
"github.com/helmfile/helmfile/pkg/argparser"
|
|
"github.com/helmfile/helmfile/pkg/environment"
|
|
"github.com/helmfile/helmfile/pkg/envvar"
|
|
"github.com/helmfile/helmfile/pkg/event"
|
|
"github.com/helmfile/helmfile/pkg/filesystem"
|
|
"github.com/helmfile/helmfile/pkg/helmexec"
|
|
"github.com/helmfile/helmfile/pkg/kubedog"
|
|
"github.com/helmfile/helmfile/pkg/maputil"
|
|
"github.com/helmfile/helmfile/pkg/remote"
|
|
"github.com/helmfile/helmfile/pkg/tmpl"
|
|
"github.com/helmfile/helmfile/pkg/yaml"
|
|
)
|
|
|
|
const (
|
|
// EmptyTimeout represents the `--timeout` value passed to helm commands not being specified via helmfile flags.
|
|
// This is used by an interim solution to make the urfave/cli command report to the helmfile internal about that the
|
|
// --timeout flag is missingl
|
|
EmptyTimeout = -1
|
|
|
|
// Valid enum for updateStrategy values
|
|
UpdateStrategyReinstallIfForbidden = "reinstallIfForbidden"
|
|
|
|
// Valid values for environment mergeStrategy.
|
|
// MergeStrategyOverride (default) makes later values files override earlier ones.
|
|
// MergeStrategyFallback flips the precedence: earlier files win and later files only fill gaps.
|
|
MergeStrategyOverride = "override"
|
|
MergeStrategyFallback = "fallback"
|
|
)
|
|
|
|
// ReleaseSetSpec is release set spec
|
|
type ReleaseSetSpec struct {
|
|
DefaultHelmBinary string `yaml:"helmBinary,omitempty"`
|
|
DefaultKustomizeBinary string `yaml:"kustomizeBinary,omitempty"`
|
|
|
|
// DefaultValues is the default values to be overrode by environment values and command-line overrides
|
|
DefaultValues []any `yaml:"values,omitempty"`
|
|
|
|
Environments map[string]EnvironmentSpec `yaml:"environments,omitempty"`
|
|
|
|
Bases []string `yaml:"bases,omitempty"`
|
|
HelmDefaults HelmSpec `yaml:"helmDefaults,omitempty"`
|
|
Helmfiles []SubHelmfileSpec `yaml:"helmfiles,omitempty"`
|
|
|
|
OverrideKubeContext string `yaml:"kubeContext,omitempty"`
|
|
OverrideNamespace string `yaml:"namespace,omitempty"`
|
|
OverrideChart string `yaml:"chart,omitempty"`
|
|
Repositories []RepositorySpec `yaml:"repositories,omitempty"`
|
|
CommonLabels map[string]string `yaml:"commonLabels,omitempty"`
|
|
Releases []ReleaseSpec `yaml:"releases,omitempty"`
|
|
OrginReleases []ReleaseSpec `yaml:"-"`
|
|
Selectors []string `yaml:"-"`
|
|
|
|
// LLM is the optional OpenAI-compatible configuration used by AI-assisted
|
|
// subcommands such as `helmfile doctor`. When absent (the default),
|
|
// those commands degrade to their non-AI equivalents (e.g. plain `diff`).
|
|
// This field is read at app layer; the state layer treats it as inert.
|
|
LLM llm.Config `yaml:"llm,omitempty"`
|
|
|
|
// Capabilities.APIVersions
|
|
ApiVersions []string `yaml:"apiVersions,omitempty"`
|
|
|
|
// Capabilities.KubeVersion
|
|
KubeVersion string `yaml:"kubeVersion,omitempty"`
|
|
|
|
// Hooks is a list of extension points paired with operations, that are executed in specific points of the lifecycle of releases defined in helmfile
|
|
Hooks []event.Hook `yaml:"hooks,omitempty"`
|
|
|
|
Templates map[string]TemplateSpec `yaml:"templates"`
|
|
|
|
// DefaultInherit is a list of template names that all releases inherit by default.
|
|
// Each release will automatically inherit these templates unless it already explicitly
|
|
// inherits from the same template.
|
|
DefaultInherit DefaultInherits `yaml:"defaultInherit,omitempty"`
|
|
|
|
Env environment.Environment `yaml:"-"`
|
|
|
|
// If set to "Error", return an error when a subhelmfile points to a
|
|
// non-existent path. The default behavior is to print a warning. Note the
|
|
// differing default compared to other MissingFileHandlers.
|
|
MissingFileHandler *string `yaml:"missingFileHandler,omitempty"`
|
|
// MissingFileHandlerConfig is composed of various settings for the MissingFileHandler
|
|
MissingFileHandlerConfig *MissingFileHandlerConfig `yaml:"missingFileHandlerConfig,omitempty"`
|
|
|
|
LockFile string `yaml:"lockFilePath,omitempty"`
|
|
}
|
|
|
|
type MissingFileHandlerConfig struct {
|
|
// IgnoreMissingGitBranch is set to true in order to let the missing file handler
|
|
// treat missing git branch errors like `pathspec 'develop' did not match any file(s) known to git` safe
|
|
// and ignored when the handler is set to Warn or Info.
|
|
IgnoreMissingGitBranch bool `yaml:"ignoreMissingGitBranch,omitempty"`
|
|
}
|
|
|
|
// helmStateAlias is helm state alias
|
|
type helmStateAlias HelmState
|
|
|
|
func (hs *HelmState) UnmarshalYAML(unmarshal func(any) error) error {
|
|
helmStateInfo := make(map[string]any)
|
|
if err := unmarshal(&helmStateInfo); err != nil {
|
|
return err
|
|
}
|
|
|
|
return unmarshal((*helmStateAlias)(hs))
|
|
}
|
|
|
|
// HelmState structure for the helmfile
|
|
type HelmState struct {
|
|
basePath string
|
|
FilePath string
|
|
|
|
ReleaseSetSpec `yaml:",inline"`
|
|
|
|
logger *zap.SugaredLogger
|
|
fs *filesystem.FileSystem
|
|
tempDir func(string, string) (string, error)
|
|
|
|
valsRuntime vals.Evaluator
|
|
|
|
kubeconfig string
|
|
|
|
// chartifyTempDirs tracks temporary directories created by chartify during
|
|
// chart preparation. These directories contain the chartified charts and must
|
|
// survive until all helm operations complete, after which they are cleaned up
|
|
// via CleanupChartifyTempDirs. It is a pointer so that copying HelmState
|
|
// (which happens in several places) does not copy the embedded mutex.
|
|
// See issue #1799.
|
|
chartifyTempDirs *chartifyTempDirTracker
|
|
|
|
// RenderedValues is the helmfile-wide values that is `.Values`
|
|
// which is accessible from within the whole helmfile go template.
|
|
// Note that this is usually computed by DesiredStateLoader from ReleaseSetSpec.Env
|
|
RenderedValues map[string]any
|
|
}
|
|
|
|
// chartifyTempDirTracker holds the set of chartify output directories to be
|
|
// cleaned up after helm operations complete. The mutex guards concurrent access
|
|
// from chart-preparation workers. See issue #1799.
|
|
type chartifyTempDirTracker struct {
|
|
mu sync.Mutex
|
|
dirs []string
|
|
}
|
|
|
|
func (st *HelmState) SetKubeconfig(kubeconfig string) {
|
|
st.kubeconfig = kubeconfig
|
|
}
|
|
|
|
// SubHelmfileSpec defines the subhelmfile path and options
|
|
type SubHelmfileSpec struct {
|
|
//path or glob pattern for the sub helmfiles
|
|
Path string `yaml:"path,omitempty"`
|
|
//chosen selectors for the sub helmfiles
|
|
Selectors []string `yaml:"selectors,omitempty"`
|
|
//do the sub helmfiles inherits from parent selectors
|
|
SelectorsInherited bool `yaml:"selectorsInherited,omitempty"`
|
|
|
|
Environment SubhelmfileEnvironmentSpec
|
|
}
|
|
|
|
// SubhelmfileEnvironmentSpec is the environment spec for a subhelmfile
|
|
type SubhelmfileEnvironmentSpec struct {
|
|
OverrideValues []any `yaml:"values,omitempty"`
|
|
OverrideCLISetValues []any `yaml:"-"` // CLI --state-values-set values only, merged element-by-element
|
|
}
|
|
|
|
// HelmSpec to defines helmDefault values
|
|
type HelmSpec struct {
|
|
KubeContext string `yaml:"kubeContext,omitempty"`
|
|
Args []string `yaml:"args,omitempty"`
|
|
DiffArgs []string `yaml:"diffArgs,omitempty"`
|
|
SyncArgs []string `yaml:"syncArgs,omitempty"`
|
|
Verify bool `yaml:"verify"`
|
|
Keyring string `yaml:"keyring,omitempty"`
|
|
// EnableDNS, when set to true, enable DNS lookups when rendering templates
|
|
EnableDNS bool `yaml:"enableDNS"`
|
|
// Propagate '--skip-schema-validation' to helmv3 template and helm install
|
|
SkipSchemaValidation *bool `yaml:"skipSchemaValidation,omitempty"`
|
|
// Devel, when set to true, use development versions, too. Equivalent to version '>0.0.0-0'
|
|
Devel bool `yaml:"devel"`
|
|
// Wait, if set to true, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment are in a ready state before marking the release as successful
|
|
Wait bool `yaml:"wait"`
|
|
// WaitRetries, if set and --wait enabled, will retry any failed check on resource state, except if HTTP status code < 500 is received, subject to the specified number of retries
|
|
// DEPRECATED: This field is ignored as the --wait-retries flag was removed from Helm. Preserved for backward compatibility.
|
|
WaitRetries int `yaml:"waitRetries"`
|
|
// WaitForJobs, if set and --wait enabled, will wait until all Jobs have been completed before marking the release as successful. It will wait for as long as --timeout
|
|
WaitForJobs bool `yaml:"waitForJobs"`
|
|
// Timeout is the time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks, and waits on pod/pvc/svc/deployment readiness) (default 300)
|
|
Timeout int `yaml:"timeout"`
|
|
// RecreatePods, when set to true, instruct helmfile to perform pods restart for the resource if applicable
|
|
RecreatePods bool `yaml:"recreatePods"`
|
|
// Force, when set to true, forces resource update through delete/recreate if needed
|
|
Force bool `yaml:"force"`
|
|
// Atomic, when set to true, restore previous state in case of a failed install/upgrade attempt
|
|
Atomic bool `yaml:"atomic"`
|
|
// CleanupOnFail, when set to true, the --cleanup-on-fail helm flag is passed to the upgrade command
|
|
CleanupOnFail bool `yaml:"cleanupOnFail,omitempty"`
|
|
// ForceConflicts, when set to true, force server-side apply changes against conflicts (Helm 4 only)
|
|
ForceConflicts bool `yaml:"forceConflicts"`
|
|
// HistoryMax, limit the maximum number of revisions saved per release. Use 0 for no limit (default 10)
|
|
HistoryMax *int `yaml:"historyMax,omitempty"`
|
|
// CreateNamespace, when set to true (default), --create-namespace is passed to helm on install/upgrade
|
|
CreateNamespace *bool `yaml:"createNamespace,omitempty"`
|
|
// SkipCRDs passes the --skip-crds flag to helm upgrade --install to ensure any CRDs contained in the crds/
|
|
// subdirectory of a helm chart are not automatically applied with every helmfile sync or apply
|
|
SkipCRDs bool `yaml:"skipCRDs,omitempty"`
|
|
// SkipDeps disables running `helm dependency up` and `helm dependency build` on this release's chart.
|
|
// This is relevant only when your release uses a local chart or a directory containing K8s manifests or a Kustomization
|
|
// as a Helm chart.
|
|
SkipDeps bool `yaml:"skipDeps"`
|
|
// SkipRefresh disables running `helm dependency up`
|
|
SkipRefresh bool `yaml:"skipRefresh"`
|
|
// on helm upgrade/diff, reuse values currently set in the release and merge them with the ones defined within helmfile
|
|
ReuseValues bool `yaml:"reuseValues"`
|
|
// Propagate '--post-renderer' to helmv3 template and helm install
|
|
PostRenderer *string `yaml:"postRenderer,omitempty"`
|
|
// Propagate '--post-renderer-args' to helmv3 template and helm install
|
|
PostRendererArgs []string `yaml:"postRendererArgs,omitempty"`
|
|
// Cascade '--cascade' to helmv3 delete, available values: background, foreground, or orphan, default: background
|
|
Cascade *string `yaml:"cascade,omitempty"`
|
|
// SuppressOutputLineRegex is a list of regexes to suppress output lines
|
|
SuppressOutputLineRegex []string `yaml:"suppressOutputLineRegex,omitempty"`
|
|
// DisableAutoDetectedKubeVersionForDiff controls whether auto-detected kubeVersion should be passed
|
|
// to helm diff. When false (default), auto-detected kubeVersion is passed to fix issue #2275.
|
|
// Set to true to only pass explicit kubeVersion from helmfile.yaml, preventing helm-diff from
|
|
// normalizing server-side defaults which could hide real changes (e.g., ipFamilyPolicy, ipFamilies).
|
|
DisableAutoDetectedKubeVersionForDiff *bool `yaml:"disableAutoDetectedKubeVersionForDiff,omitempty"`
|
|
|
|
DisableValidation *bool `yaml:"disableValidation,omitempty"`
|
|
DisableOpenAPIValidation *bool `yaml:"disableOpenAPIValidation,omitempty"`
|
|
// InsecureSkipTLSVerify is true if the TLS verification should be skipped when fetching remote chart
|
|
InsecureSkipTLSVerify bool `yaml:"insecureSkipTLSVerify,omitempty"`
|
|
// PlainHttp is true if the remote charte should be fetched using HTTP and not HTTPS
|
|
PlainHttp bool `yaml:"plainHttp,omitempty"`
|
|
// Wait, if set to true, will wait until all resources are deleted before mark delete command as successful
|
|
DeleteWait bool `yaml:"deleteWait,omitempty"`
|
|
// Timeout is the time in seconds to wait for helmfile delete command (default 300)
|
|
DeleteTimeout int `yaml:"deleteTimeout,omitempty"`
|
|
// SyncReleaseLabels is true if the release labels should be synced with the helmfile labels
|
|
SyncReleaseLabels *bool `yaml:"syncReleaseLabels,omitempty"`
|
|
// TakeOwnership is true if the helmfile should take ownership of the release
|
|
TakeOwnership *bool `yaml:"takeOwnership,omitempty"`
|
|
// ServerSide controls the helm 4 --server-side flag for upgrade. Must be "true", "false", or "auto".
|
|
ServerSide *string `yaml:"serverSide,omitempty"`
|
|
// TrackMode specifies whether to use 'helm' or 'kubedog' for tracking resources
|
|
TrackMode string `yaml:"trackMode,omitempty"`
|
|
// HelmStuckGrace, when > 0, enables the helmfile-side safety-valve
|
|
// helm-killer for releases using --track-mode kubedog. See
|
|
// ReleaseSpec.HelmStuckGrace for semantics.
|
|
HelmStuckGrace int `yaml:"helmStuckGrace,omitempty"`
|
|
}
|
|
|
|
// RepositorySpec that defines values for a helm repo
|
|
type RepositorySpec struct {
|
|
Name string `yaml:"name,omitempty"`
|
|
URL string `yaml:"url,omitempty"`
|
|
CaFile string `yaml:"caFile,omitempty"`
|
|
CertFile string `yaml:"certFile,omitempty"`
|
|
KeyFile string `yaml:"keyFile,omitempty"`
|
|
Username string `yaml:"username,omitempty"`
|
|
Password string `yaml:"password,omitempty"`
|
|
RegistryConfig string `yaml:"registryConfig,omitempty"`
|
|
Managed string `yaml:"managed,omitempty"`
|
|
OCI bool `yaml:"oci,omitempty"`
|
|
Verify bool `yaml:"verify,omitempty"`
|
|
Keyring string `yaml:"keyring,omitempty"`
|
|
PassCredentials bool `yaml:"passCredentials,omitempty"`
|
|
SkipTLSVerify bool `yaml:"skipTLSVerify,omitempty"`
|
|
PlainHttp bool `yaml:"plainHttp,omitempty"`
|
|
}
|
|
|
|
type Inherit struct {
|
|
Template string `yaml:"template,omitempty"`
|
|
Except []string `yaml:"except,omitempty"`
|
|
}
|
|
|
|
type Inherits []Inherit
|
|
|
|
// ReleaseSpec defines the configuration for a Helm release managed by helmfile.
|
|
type ReleaseSpec struct {
|
|
// Chart is the name of the chart being installed to create this release
|
|
Chart string `yaml:"chart,omitempty"`
|
|
|
|
// ChartPath is the downloaded and modified version of the remote Chart specified by the Chart field.
|
|
// This field is empty when the release is going to use the remote chart as-is, without any modifications(e.g. chartify).
|
|
ChartPath string `yaml:"chartPath,omitempty"`
|
|
|
|
// Directory is an alias to Chart which may be of more fit when you want to use a local/remote directory containing
|
|
// K8s manifests or Kustomization as a chart
|
|
Directory string `yaml:"directory,omitempty"`
|
|
// Version is the semver version or version constraint for the chart
|
|
Version string `yaml:"version,omitempty"`
|
|
// Verify enables signature verification on fetched chart.
|
|
// Beware some (or many?) chart repositories and charts don't seem to support it.
|
|
Verify *bool `yaml:"verify,omitempty"`
|
|
Keyring string `yaml:"keyring,omitempty"`
|
|
// EnableDNS, when set to true, enable DNS lookups when rendering templates
|
|
EnableDNS *bool `yaml:"enableDNS,omitempty"`
|
|
// Devel, when set to true, use development versions, too. Equivalent to version '>0.0.0-0'
|
|
Devel *bool `yaml:"devel,omitempty"`
|
|
// Wait, if set to true, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment are in a ready state before marking the release as successful
|
|
Wait *bool `yaml:"wait,omitempty"`
|
|
// WaitRetries, if set and --wait enabled, will retry any failed check on resource state, except if HTTP status code < 500 is received, subject to the specified number of retries
|
|
// DEPRECATED: This field is ignored as the --wait-retries flag was removed from Helm. Preserved for backward compatibility.
|
|
WaitRetries *int `yaml:"waitRetries,omitempty"`
|
|
// WaitForJobs, if set and --wait enabled, will wait until all Jobs have been completed before marking the release as successful. It will wait for as long as --timeout
|
|
WaitForJobs *bool `yaml:"waitForJobs,omitempty"`
|
|
// Timeout is the time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks, and waits on pod/pvc/svc/deployment readiness) (default 300)
|
|
Timeout *int `yaml:"timeout,omitempty"`
|
|
// RecreatePods, when set to true, instruct helmfile to perform pods restart for the resource if applicable
|
|
RecreatePods *bool `yaml:"recreatePods,omitempty"`
|
|
// Force, when set to true, forces resource update through delete/recreate if needed
|
|
Force *bool `yaml:"force,omitempty"`
|
|
// Installed, when set to true, `delete --purge` the release
|
|
Installed *bool `yaml:"installed,omitempty"`
|
|
// UpdateStrategy, when set, indicate the strategy to use to update the release
|
|
UpdateStrategy string `yaml:"updateStrategy,omitempty"`
|
|
// Atomic, when set to true, restore previous state in case of a failed install/upgrade attempt
|
|
Atomic *bool `yaml:"atomic,omitempty"`
|
|
// CleanupOnFail, when set to true, the --cleanup-on-fail helm flag is passed to the upgrade command
|
|
CleanupOnFail *bool `yaml:"cleanupOnFail,omitempty"`
|
|
// ForceConflicts, when set to true, force server-side apply changes against conflicts (Helm 4 only)
|
|
ForceConflicts *bool `yaml:"forceConflicts,omitempty"`
|
|
// HistoryMax, limit the maximum number of revisions saved per release. Use 0 for no limit (default 10)
|
|
HistoryMax *int `yaml:"historyMax,omitempty"`
|
|
// Condition, when set, evaluate the mapping specified in this string to a boolean which decides whether or not to process the release
|
|
Condition string `yaml:"condition,omitempty"`
|
|
// CreateNamespace, when set to true (default), --create-namespace is passed to helm on install
|
|
CreateNamespace *bool `yaml:"createNamespace,omitempty"`
|
|
// ReuseValues, on helm upgrade/diff, reuse values currently set in the release and merge them with the ones defined within helmfile
|
|
ReuseValues *bool `yaml:"reuseValues,omitempty"`
|
|
|
|
// DisableOpenAPIValidation is rarely used to bypass OpenAPI validations only that is used for e.g.
|
|
// work-around against broken CRs
|
|
// See also:
|
|
// - https://github.com/helm/helm/pull/6819
|
|
// - https://github.com/roboll/helmfile/issues/1167
|
|
DisableOpenAPIValidation *bool `yaml:"disableOpenAPIValidation,omitempty"`
|
|
|
|
// DisableValidation is rarely used to bypass the whole validation of manifests against the Kubernetes cluster
|
|
// so that `helm diff` can be run containing a chart that installs both CRD and CRs on first install.
|
|
// FYI, such diff without `--disable-validation` fails on first install because the K8s cluster doesn't have CRDs registered yet.
|
|
DisableValidation *bool `yaml:"disableValidation,omitempty"`
|
|
|
|
// DisableValidationOnInstall disables the K8s API validation while running helm-diff on the release being newly installed on helmfile-apply.
|
|
// It is useful when any release contains custom resources for CRDs that is not yet installed onto the cluster.
|
|
DisableValidationOnInstall *bool `yaml:"disableValidationOnInstall,omitempty"`
|
|
|
|
// MissingFileHandler is set to either "Error" or "Warn". "Error" instructs helmfile to fail when unable to find a values or secrets file. When "Warn", it prints the file and continues.
|
|
// The default value for MissingFileHandler is "Error".
|
|
MissingFileHandler *string `yaml:"missingFileHandler,omitempty"`
|
|
|
|
// MissingFileHandlerConfig is composed of various settings for the MissingFileHandler
|
|
MissingFileHandlerConfig *MissingFileHandlerConfig `yaml:"missingFileHandlerConfig,omitempty"`
|
|
|
|
// Needs is the [KUBECONTEXT/][NS/]NAME representations of releases that this release depends on.
|
|
Needs []string `yaml:"needs,omitempty"`
|
|
|
|
// Hooks is a list of extension points paired with operations, that are executed in specific points of the lifecycle of releases defined in helmfile
|
|
Hooks []event.Hook `yaml:"hooks,omitempty"`
|
|
|
|
// UnitTests is a list of test file or directory paths for helm-unittest integration.
|
|
// When specified, `helmfile unittest` will run `helm unittest` with the merged values and these test paths.
|
|
UnitTests []string `yaml:"unitTests,omitempty"`
|
|
|
|
// Name is the name of this release
|
|
Name string `yaml:"name,omitempty"`
|
|
|
|
// Description is the description for this release that will be passed to helm upgrade with --description flag
|
|
Description string `yaml:"description,omitempty"`
|
|
Namespace string `yaml:"namespace,omitempty"`
|
|
Labels map[string]string `yaml:"labels,omitempty"`
|
|
Values []any `yaml:"values,omitempty"`
|
|
Secrets []any `yaml:"secrets,omitempty"`
|
|
SetValues []SetValue `yaml:"set,omitempty"`
|
|
SetStringValues []SetValue `yaml:"setString,omitempty"`
|
|
duration time.Duration
|
|
|
|
ValuesTemplate []any `yaml:"valuesTemplate,omitempty"`
|
|
SetValuesTemplate []SetValue `yaml:"setTemplate,omitempty"`
|
|
|
|
// Capabilities.APIVersions
|
|
ApiVersions []string `yaml:"apiVersions,omitempty"`
|
|
|
|
// Capabilities.KubeVersion
|
|
KubeVersion string `yaml:"kubeVersion,omitempty"`
|
|
|
|
// The 'env' section is not really necessary any longer, as 'set' would now provide the same functionality
|
|
EnvValues []SetValue `yaml:"env,omitempty"`
|
|
|
|
ValuesPathPrefix string `yaml:"valuesPathPrefix,omitempty"`
|
|
|
|
KubeContext string `yaml:"kubeContext,omitempty"`
|
|
|
|
// InsecureSkipTLSVerify is true if the TLS verification should be skipped when fetching remote chart.
|
|
InsecureSkipTLSVerify bool `yaml:"insecureSkipTLSVerify,omitempty"`
|
|
|
|
// PlainHttp is true if the remote charte should be fetched using HTTP and not HTTPS
|
|
PlainHttp bool `yaml:"plainHttp,omitempty"`
|
|
|
|
// These values are used in templating
|
|
VerifyTemplate *string `yaml:"verifyTemplate,omitempty"`
|
|
WaitTemplate *string `yaml:"waitTemplate,omitempty"`
|
|
InstalledTemplate *string `yaml:"installedTemplate,omitempty"`
|
|
|
|
// These settings requires helm-x integration to work
|
|
Dependencies []Dependency `yaml:"dependencies,omitempty"`
|
|
JSONPatches []any `yaml:"jsonPatches,omitempty"`
|
|
StrategicMergePatches []any `yaml:"strategicMergePatches,omitempty"`
|
|
|
|
// Transformers is the list of Kustomize transformers
|
|
//
|
|
// Each item can be a path to a YAML or go template file, or an embedded transformer declaration as a YAML hash.
|
|
// It's often used to add common labels and annotations to your resources.
|
|
// See https://github.com/kubernetes-sigs/kustomize/blob/master/examples/configureBuiltinPlugin.md#configuring-the-builtin-plugins-instead for more information.
|
|
Transformers []any `yaml:"transformers,omitempty"`
|
|
Adopt []string `yaml:"adopt,omitempty"`
|
|
|
|
//version of the chart that has really been installed cause desired version may be fuzzy (~2.0.0)
|
|
installedVersion string
|
|
|
|
// ForceGoGetter forces the use of go-getter for fetching remote directory as maniefsts/chart/kustomization
|
|
// by parsing the url from `chart` field of the release.
|
|
// This is handy when getting the go-getter url parsing error when it doesn't work as expected.
|
|
// Without this, any error in url parsing result in silently falling-back to normal process of treating `chart:` as the regular
|
|
// helm chart name.
|
|
ForceGoGetter bool `yaml:"forceGoGetter,omitempty"`
|
|
|
|
// ForceNamespace is an experimental feature to set metadata.namespace in every K8s resource rendered by the chart,
|
|
// regardless of the template, even when it doesn't have `namespace: {{ .Namespace | quote }}`.
|
|
// This is only needed when you can't FIX your chart to have `namespace: {{ .Namespace }}` AND you're using `helmfile template`.
|
|
// In standard use-cases, `Namespace` should be sufficient.
|
|
// Use this only when you know what you want to do!
|
|
ForceNamespace string `yaml:"forceNamespace,omitempty"`
|
|
|
|
// SkipDeps disables running `helm dependency up` and `helm dependency build` on this release's chart.
|
|
// This is relevant only when your release uses a local chart or a directory containing K8s manifests or a Kustomization
|
|
// as a Helm chart.
|
|
SkipDeps *bool `yaml:"skipDeps,omitempty"`
|
|
|
|
// SkipRefresh disables running `helm dependency up`
|
|
SkipRefresh *bool `yaml:"skipRefresh,omitempty"`
|
|
|
|
// Propagate '--post-renderer' to helmv3 template and helm install
|
|
PostRenderer *string `yaml:"postRenderer,omitempty"`
|
|
|
|
// Propagate '--skip-schema-validation' to helmv3 template and helm install
|
|
SkipSchemaValidation *bool `yaml:"skipSchemaValidation,omitempty"`
|
|
|
|
// Propagate '--post-renderer-args' to helmv3 template and helm install
|
|
PostRendererArgs []string `yaml:"postRendererArgs,omitempty"`
|
|
|
|
// Cascade '--cascade' to helmv3 delete, available values: background, foreground, or orphan, default: background
|
|
Cascade *string `yaml:"cascade,omitempty"`
|
|
|
|
// SuppressOutputLineRegex is a list of regexes to suppress output lines
|
|
SuppressOutputLineRegex []string `yaml:"suppressOutputLineRegex,omitempty"`
|
|
// DisableAutoDetectedKubeVersionForDiff controls whether auto-detected kubeVersion should be passed
|
|
// to helm diff for this release. See HelmSpec.DisableAutoDetectedKubeVersionForDiff for details.
|
|
DisableAutoDetectedKubeVersionForDiff *bool `yaml:"disableAutoDetectedKubeVersionForDiff,omitempty"`
|
|
|
|
// Inherit is used to inherit a release template from a release or another release template
|
|
Inherit Inherits `yaml:"inherit,omitempty"`
|
|
|
|
// SuppressDiff skip the helm diff output. Useful for charts which produces large not helpful diff.
|
|
SuppressDiff *bool `yaml:"suppressDiff,omitempty"`
|
|
|
|
// --wait flag for destroy/delete, if set to true, will wait until all resources are deleted before mark delete command as successful
|
|
DeleteWait *bool `yaml:"deleteWait,omitempty"`
|
|
// Timeout is the time in seconds to wait for helmfile delete command (default 300)
|
|
DeleteTimeout *int `yaml:"deleteTimeout,omitempty"`
|
|
// SyncReleaseLabels is true if the release labels should be synced with the helmfile labels
|
|
SyncReleaseLabels *bool `yaml:"syncReleaseLabels,omitempty"`
|
|
// TakeOwnership is true if release should take ownership of resources
|
|
TakeOwnership *bool `yaml:"takeOwnership,omitempty"`
|
|
// ServerSide controls the helm 4 --server-side flag for this release. Must be "true", "false", or "auto".
|
|
ServerSide *string `yaml:"serverSide,omitempty"`
|
|
// TrackMode specifies whether to use 'helm' or 'kubedog' for tracking resources
|
|
TrackMode string `yaml:"trackMode,omitempty"`
|
|
// TrackTimeout specifies timeout for kubedog tracking (in seconds)
|
|
TrackTimeout *int `yaml:"trackTimeout,omitempty"`
|
|
// TrackLogs enables log streaming with kubedog
|
|
TrackLogs *bool `yaml:"trackLogs,omitempty"`
|
|
// TrackFailedLogs streams logs only for pods that enter a failed state
|
|
// (CrashLoopBackOff, Error, etc.). Pods that succeed produce no output.
|
|
// Has no effect when TrackLogs is true (full streaming wins).
|
|
TrackFailedLogs *bool `yaml:"trackFailedLogs,omitempty"`
|
|
// HelmStuckGrace, when > 0, enables the safety-valve helm-killer for
|
|
// kubedog tracking: if the cluster confirms every tracked resource has
|
|
// converged but the helm subprocess is still running, helmfile waits
|
|
// this many seconds before sending SIGINT to helm. Targets the helm v4
|
|
// hook waiter wedge that --track-timeout would otherwise resolve only
|
|
// after hours. Zero (or absent) disables the killer.
|
|
HelmStuckGrace *int `yaml:"helmStuckGrace,omitempty"`
|
|
// TrackKinds is a whitelist of resource kinds to track
|
|
TrackKinds []string `yaml:"trackKinds,omitempty"`
|
|
// SkipKinds is a blacklist of resource kinds to skip tracking
|
|
SkipKinds []string `yaml:"skipKinds,omitempty"`
|
|
// TrackResources is a whitelist of specific resources to track
|
|
TrackResources []TrackResourceSpec `yaml:"trackResources,omitempty"`
|
|
// KubedogQPS specifies the QPS (queries per second) for kubedog kubernetes client
|
|
KubedogQPS *float32 `yaml:"kubedogQPS,omitempty"`
|
|
// KubedogBurst specifies the burst for kubedog kubernetes client
|
|
KubedogBurst *int `yaml:"kubedogBurst,omitempty"`
|
|
// TrackFailOnError controls whether kubedog tracking failures cause a non-zero exit code
|
|
TrackFailOnError *bool `yaml:"trackFailOnError,omitempty"`
|
|
}
|
|
|
|
// TrackResourceSpec specifies a resource to track
|
|
type TrackResourceSpec struct {
|
|
Kind string `yaml:"kind,omitempty"`
|
|
Name string `yaml:"name,omitempty"`
|
|
Namespace string `yaml:"namespace,omitempty"`
|
|
}
|
|
|
|
func (r *Inherits) UnmarshalYAML(unmarshal func(any) error) error {
|
|
var v0151 []Inherit
|
|
if err := unmarshal(&v0151); err != nil {
|
|
var v0150 Inherit
|
|
if err := unmarshal(&v0150); err != nil {
|
|
return err
|
|
}
|
|
fmt.Fprintf(os.Stderr, "releases[].inherit of map(%+v) has been deprecated and will be removed in v0.152.0. Wrap it into an array: %v\n", v0150, err)
|
|
*r = []Inherit{v0150}
|
|
return nil
|
|
}
|
|
*r = v0151
|
|
return nil
|
|
}
|
|
|
|
type DefaultInherits []string
|
|
|
|
func (r *DefaultInherits) UnmarshalYAML(unmarshal func(any) error) error {
|
|
var list []string
|
|
if err := unmarshal(&list); err == nil {
|
|
*r = normalizeDefaultInherits(list)
|
|
return nil
|
|
}
|
|
|
|
var single string
|
|
if err := unmarshal(&single); err != nil {
|
|
return err
|
|
}
|
|
*r = normalizeDefaultInherits([]string{single})
|
|
return nil
|
|
}
|
|
|
|
// normalizeDefaultInherits trims names, drops empty entries, and returns nil for an empty result.
|
|
func normalizeDefaultInherits(in []string) []string {
|
|
if len(in) == 0 {
|
|
return nil
|
|
}
|
|
|
|
out := make([]string, 0, len(in))
|
|
for _, name := range in {
|
|
name = strings.TrimSpace(name)
|
|
if name == "" {
|
|
continue
|
|
}
|
|
out = append(out, name)
|
|
}
|
|
|
|
if len(out) == 0 {
|
|
return nil
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
// ChartPathOrName returns ChartPath if it is non-empty, and returns Chart otherwise.
|
|
// This is useful to redirect helm commands like `helm template`, `helm dependency update`, `helm diff`, and `helm upgrade --install` to
|
|
// our modified version of the chart, in case the user configured Helmfile to do modify the chart before being passed to Helm.
|
|
func (r ReleaseSpec) ChartPathOrName() string {
|
|
if r.ChartPath != "" {
|
|
return r.ChartPath
|
|
}
|
|
return r.Chart
|
|
}
|
|
|
|
type Release struct {
|
|
ReleaseSpec
|
|
|
|
Filtered bool
|
|
}
|
|
|
|
// SetValue are the key values to set on a helm release
|
|
type SetValue struct {
|
|
Name string `yaml:"name,omitempty"`
|
|
Value string `yaml:"value,omitempty"`
|
|
File string `yaml:"file,omitempty"`
|
|
Values []any `yaml:"values,omitempty"`
|
|
}
|
|
|
|
// AffectedReleases hold the list of released that where updated, deleted, or in error
|
|
type AffectedReleases struct {
|
|
Upgraded []*ReleaseSpec
|
|
Reinstalled []*ReleaseSpec
|
|
Deleted []*ReleaseSpec
|
|
Failed []*ReleaseSpec
|
|
DeleteFailed []*ReleaseSpec
|
|
}
|
|
|
|
// DefaultEnv is the default environment to use for helm commands
|
|
const DefaultEnv = "default"
|
|
|
|
// MissingFileHandlerError is the error returned when a file is missing
|
|
const MissingFileHandlerError = "Error"
|
|
|
|
// MissingFileHandlerInfo is the info returned when a file is missing
|
|
const MissingFileHandlerInfo = "Info"
|
|
|
|
// MissingFileHandlerWarn is the warning returned when a file is missing
|
|
const MissingFileHandlerWarn = "Warn"
|
|
|
|
// MissingFileHandlerDebug is the debug returned when a file is missing
|
|
const MissingFileHandlerDebug = "Debug"
|
|
|
|
var DefaultFetchOutputDirTemplate = filepath.Join(
|
|
"{{ .OutputDir }}{{ if .Release.Namespace }}",
|
|
"{{ .Release.Namespace }}{{ end }}{{ if .Release.KubeContext }}",
|
|
"{{ .Release.KubeContext }}{{ end }}",
|
|
"{{ .Release.Name }}",
|
|
"{{ .ChartName }}",
|
|
"{{ or .Release.Version \"latest\" }}",
|
|
)
|
|
|
|
func (st *HelmState) reformat(spec *ReleaseSpec) []string {
|
|
var needs []string
|
|
releaseInstalledInfo := make(map[string]bool)
|
|
for _, r := range st.OrginReleases {
|
|
kubecontext := r.KubeContext
|
|
namespace := r.Namespace
|
|
|
|
if st.OverrideKubeContext != "" {
|
|
kubecontext = st.OverrideKubeContext
|
|
}
|
|
if st.OverrideNamespace != "" {
|
|
namespace = st.OverrideNamespace
|
|
}
|
|
|
|
releaseInstalledInfo[fmt.Sprintf("%s/%s/%s", kubecontext, namespace, r.Name)] = r.Desired()
|
|
}
|
|
|
|
// Since the representation differs between needs and id,
|
|
// correct it by prepending Namespace and KubeContext.
|
|
for i := 0; i < len(spec.Needs); i++ {
|
|
n := spec.Needs[i]
|
|
|
|
var kubecontext, ns, name string
|
|
|
|
components := strings.Split(n, "/")
|
|
|
|
name = components[len(components)-1]
|
|
|
|
if len(components) > 1 {
|
|
ns = components[len(components)-2]
|
|
} else {
|
|
ns = spec.Namespace
|
|
}
|
|
|
|
if len(components) > 2 {
|
|
// Join all ID components except last 2 (namespace and name) into kubecontext.
|
|
// Should be safe because resource names do not contain slashes.
|
|
kubecontext = strings.Join(components[:len(components)-2], "/")
|
|
} else {
|
|
kubecontext = spec.KubeContext
|
|
}
|
|
|
|
if spec.Desired() && !releaseInstalledInfo[fmt.Sprintf("%s/%s/%s", kubecontext, ns, name)] {
|
|
st.logger.Warnf("WARNING: %s", fmt.Sprintf("release %s needs %s, but %s is not installed due to installed: false. Either mark %s as installed or remove %s from %s's needs", spec.Name, name, name, name, name, spec.Name))
|
|
}
|
|
|
|
var componentsAfterOverride []string
|
|
|
|
if kubecontext != "" {
|
|
componentsAfterOverride = append(componentsAfterOverride, kubecontext)
|
|
}
|
|
|
|
// This is intentionally `kubecontext != "" || ns != ""`, but "ns != ""
|
|
// To avoid conflating kubecontext=,namespace=foo,name=bar and kubecontext=foo,namespace=,name=bar
|
|
// as they are both `foo/bar`, we explicitly differentiate each with `foo//bar` and `foo/bar`.
|
|
// Note that `foo//bar` is not always a equivalent to `foo/default/bar` as the default namespace is depedent on
|
|
// the user's kubeconfig.
|
|
if kubecontext != "" || ns != "" {
|
|
componentsAfterOverride = append(componentsAfterOverride, ns)
|
|
}
|
|
|
|
componentsAfterOverride = append(componentsAfterOverride, name)
|
|
|
|
needs = append(needs, strings.Join(componentsAfterOverride, "/"))
|
|
}
|
|
return needs
|
|
}
|
|
|
|
func (st *HelmState) ApplyOverrides(spec *ReleaseSpec) {
|
|
if st.OverrideKubeContext != "" {
|
|
spec.KubeContext = st.OverrideKubeContext
|
|
}
|
|
if st.OverrideNamespace != "" {
|
|
spec.Namespace = st.OverrideNamespace
|
|
}
|
|
|
|
spec.Needs = st.reformat(spec)
|
|
}
|
|
|
|
type RepoUpdater interface {
|
|
IsHelm3() bool
|
|
IsHelm4() bool
|
|
AddRepo(name, repository, cafile, certfile, keyfile, username, password string, managed string, passCredentials, skipTLSVerify bool) error
|
|
UpdateRepo() error
|
|
RegistryLogin(name, username, password, caFile, certFile, keyFile string, skipTLSVerify bool) error
|
|
}
|
|
|
|
func (st *HelmState) SyncRepos(helm RepoUpdater, shouldSkip map[string]bool) ([]string, error) {
|
|
var updated []string
|
|
|
|
for _, repo := range st.Repositories {
|
|
if shouldSkip[repo.Name] {
|
|
continue
|
|
}
|
|
username, password := gatherUsernamePassword(repo.Name, repo.Username, repo.Password)
|
|
var err error
|
|
if repo.OCI {
|
|
// For OCI registries, we only need the registry host for login, not the full URL with chart path.
|
|
// e.g., "123456789012.dkr.ecr.us-east-1.amazonaws.com/charts" -> "123456789012.dkr.ecr.us-east-1.amazonaws.com"
|
|
registryHost := extractRegistryHost(repo.URL)
|
|
err = helm.RegistryLogin(registryHost, username, password, repo.CaFile, repo.CertFile, repo.KeyFile, repo.SkipTLSVerify)
|
|
} else {
|
|
err = helm.AddRepo(repo.Name, repo.URL, repo.CaFile, repo.CertFile, repo.KeyFile, username, password, repo.Managed, repo.PassCredentials, repo.SkipTLSVerify)
|
|
}
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
updated = append(updated, repo.Name)
|
|
}
|
|
|
|
return updated, nil
|
|
}
|
|
|
|
func gatherUsernamePassword(repoName string, username string, password string) (string, string) {
|
|
var user, pass string
|
|
|
|
replacedRepoName := strings.ToUpper(strings.ReplaceAll(repoName, "-", "_"))
|
|
if username != "" {
|
|
user = username
|
|
} else if u := os.Getenv(fmt.Sprintf("%s_USERNAME", replacedRepoName)); u != "" {
|
|
user = u
|
|
}
|
|
|
|
if password != "" {
|
|
pass = password
|
|
} else if p := os.Getenv(fmt.Sprintf("%s_PASSWORD", replacedRepoName)); p != "" {
|
|
pass = p
|
|
}
|
|
|
|
return user, pass
|
|
}
|
|
|
|
// extractRegistryHost extracts just the registry host from an OCI repository URL.
|
|
// For "helm registry login", we only need the registry host, not the full path including chart location.
|
|
// Examples:
|
|
// - "123456789012.dkr.ecr.us-west-2.amazonaws.com/charts" -> "123456789012.dkr.ecr.us-west-2.amazonaws.com"
|
|
// - "ghcr.io/deliveryhero/helm-charts" -> "ghcr.io"
|
|
// - "registry.example.com:5000/charts" -> "registry.example.com:5000"
|
|
//
|
|
// Note: This function does not handle URLs with query parameters or fragments, as these
|
|
// are not typically used in OCI registry URLs.
|
|
func extractRegistryHost(url string) string {
|
|
// Remove any protocol prefix if present
|
|
url = strings.TrimPrefix(url, "oci://")
|
|
url = strings.TrimPrefix(url, "https://")
|
|
url = strings.TrimPrefix(url, "http://")
|
|
|
|
// Find the first slash that separates the host from the path
|
|
if idx := strings.Index(url, "/"); idx != -1 {
|
|
return url[:idx]
|
|
}
|
|
// No path, return as-is
|
|
return url
|
|
}
|
|
|
|
type syncResult struct {
|
|
errors []*ReleaseError
|
|
}
|
|
|
|
type syncPrepareResult struct {
|
|
release *ReleaseSpec
|
|
flags []string
|
|
errors []*ReleaseError
|
|
files []string
|
|
}
|
|
|
|
// SyncReleases wrapper for executing helm upgrade on the releases
|
|
func (st *HelmState) prepareSyncReleases(helm helmexec.Interface, additionalValues []string, concurrency int, opt ...SyncOpt) ([]syncPrepareResult, []error) {
|
|
opts := &SyncOpts{}
|
|
for _, o := range opt {
|
|
o.Apply(opts)
|
|
}
|
|
|
|
releases := []*ReleaseSpec{}
|
|
for i := range st.Releases {
|
|
releases = append(releases, &st.Releases[i])
|
|
}
|
|
|
|
numReleases := len(releases)
|
|
jobs := make(chan *ReleaseSpec, numReleases)
|
|
results := make(chan syncPrepareResult, numReleases)
|
|
|
|
res := []syncPrepareResult{}
|
|
errs := []error{}
|
|
|
|
mut := sync.Mutex{}
|
|
|
|
st.scatterGather(
|
|
concurrency,
|
|
numReleases,
|
|
func() {
|
|
for i := 0; i < numReleases; i++ {
|
|
jobs <- releases[i]
|
|
}
|
|
close(jobs)
|
|
},
|
|
func(workerIndex int) {
|
|
for release := range jobs {
|
|
errs := []*ReleaseError{}
|
|
st.ApplyOverrides(release)
|
|
|
|
// If `installed: false`, the only potential operation on this release would be uninstalling.
|
|
// We skip generating values files in that case, because for an uninstall with `helm delete`, we don't need to those.
|
|
// The values files are for `helm upgrade -f values.yaml` calls that happens when the release has `installed: true`.
|
|
// This logic addresses:
|
|
// - https://github.com/roboll/helmfile/issues/519
|
|
// - https://github.com/roboll/helmfile/issues/616
|
|
if !release.Desired() {
|
|
results <- syncPrepareResult{release: release, flags: []string{}, errors: []*ReleaseError{}}
|
|
continue
|
|
}
|
|
|
|
// TODO We need a long-term fix for this :)
|
|
// See https://github.com/roboll/helmfile/issues/737
|
|
mut.Lock()
|
|
flags, files, flagsErr := st.flagsForUpgrade(helm, release, workerIndex, opts)
|
|
mut.Unlock()
|
|
if flagsErr != nil {
|
|
results <- syncPrepareResult{errors: []*ReleaseError{newReleaseFailedError(release, flagsErr)}, files: files}
|
|
continue
|
|
}
|
|
|
|
for _, value := range additionalValues {
|
|
valfile, err := filepath.Abs(value)
|
|
if err != nil {
|
|
errs = append(errs, newReleaseFailedError(release, err))
|
|
}
|
|
|
|
ok, err := st.fs.FileExists(valfile)
|
|
if err != nil {
|
|
errs = append(errs, newReleaseFailedError(release, err))
|
|
} else if !ok {
|
|
errs = append(errs, newReleaseFailedError(release, fmt.Errorf("file does not exist: %s", valfile)))
|
|
}
|
|
flags = append(flags, "--values", valfile)
|
|
}
|
|
|
|
if opts.Set != nil {
|
|
for _, s := range opts.Set {
|
|
flags = append(flags, "--set", s)
|
|
}
|
|
}
|
|
|
|
if st.HelmDefaults.SkipCRDs || opts.SkipCRDs {
|
|
flags = append(flags, "--skip-crds")
|
|
}
|
|
|
|
flags = st.appendValuesControlModeFlag(flags, opts.ReuseValues, opts.ResetValues, release)
|
|
|
|
if len(errs) > 0 {
|
|
results <- syncPrepareResult{errors: errs, files: files}
|
|
continue
|
|
}
|
|
|
|
results <- syncPrepareResult{release: release, flags: flags, errors: []*ReleaseError{}, files: files}
|
|
}
|
|
},
|
|
func() {
|
|
for i := 0; i < numReleases; {
|
|
r := <-results
|
|
for _, e := range r.errors {
|
|
errs = append(errs, e)
|
|
}
|
|
res = append(res, r)
|
|
i++
|
|
}
|
|
},
|
|
)
|
|
|
|
return res, errs
|
|
}
|
|
|
|
func (st *HelmState) isReleaseInstalled(context helmexec.HelmContext, helm helmexec.Interface, release ReleaseSpec) (bool, error) {
|
|
if os.Getenv(envvar.UseHelmStatusToCheckReleaseExistence) != "" {
|
|
st.logger.Debugf("Checking release existence using `helm status` for release %s", release.Name)
|
|
|
|
flags := st.kubeConnectionFlags(&release)
|
|
if release.Namespace != "" {
|
|
flags = append(flags, "--namespace", release.Namespace)
|
|
}
|
|
err := helm.ReleaseStatus(context, release.Name, flags...)
|
|
if err != nil && strings.Contains(err.Error(), "Error: release: not found") {
|
|
return false, nil
|
|
}
|
|
return true, err
|
|
}
|
|
|
|
out, err := st.listReleases(context, helm, &release)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return out != "", nil
|
|
}
|
|
|
|
func (st *HelmState) DetectReleasesToBeDeletedForSync(helm helmexec.Interface, releases []ReleaseSpec) ([]ReleaseSpec, error) {
|
|
deleted := []ReleaseSpec{}
|
|
for i := range releases {
|
|
release := releases[i]
|
|
|
|
if !release.Desired() {
|
|
installed, err := st.isReleaseInstalled(st.createHelmContext(&release, 0), helm, release)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if installed {
|
|
// Otherwise `release` messed up(https://github.com/roboll/helmfile/issues/554)
|
|
r := release
|
|
deleted = append(deleted, r)
|
|
}
|
|
}
|
|
}
|
|
return deleted, nil
|
|
}
|
|
|
|
func (st *HelmState) DetectReleasesToBeDeleted(helm helmexec.Interface, releases []ReleaseSpec) ([]ReleaseSpec, error) {
|
|
detected := []ReleaseSpec{}
|
|
for i := range releases {
|
|
release := releases[i]
|
|
|
|
installed, err := st.isReleaseInstalled(st.createHelmContext(&release, 0), helm, release)
|
|
if err != nil {
|
|
return nil, err
|
|
} else if installed {
|
|
// Otherwise `release` messed up(https://github.com/roboll/helmfile/issues/554)
|
|
r := release
|
|
detected = append(detected, r)
|
|
}
|
|
}
|
|
return detected, nil
|
|
}
|
|
|
|
type SyncOpts struct {
|
|
Set []string
|
|
SkipCleanup bool
|
|
SkipCRDs bool
|
|
Wait bool
|
|
WaitRetries int
|
|
WaitForJobs bool
|
|
Timeout int
|
|
SyncReleaseLabels bool
|
|
ReuseValues bool
|
|
ResetValues bool
|
|
PostRenderer string
|
|
SkipSchemaValidation bool
|
|
PostRendererArgs []string
|
|
SyncArgs string
|
|
HideNotes bool
|
|
TakeOwnership bool
|
|
ServerSide string
|
|
TrackMode string
|
|
TrackTimeout int
|
|
TrackLogs bool
|
|
TrackFailedLogs bool
|
|
HelmStuckGrace int
|
|
TrackFailOnError bool
|
|
Description string
|
|
Color bool
|
|
NoColor bool
|
|
}
|
|
|
|
type SyncOpt interface{ Apply(*SyncOpts) }
|
|
|
|
func (o *SyncOpts) Apply(opts *SyncOpts) {
|
|
*opts = *o
|
|
}
|
|
|
|
func ReleaseToID(r *ReleaseSpec) string {
|
|
var id string
|
|
|
|
kc := r.KubeContext
|
|
if kc != "" {
|
|
id += kc + "/"
|
|
}
|
|
|
|
ns := r.Namespace
|
|
|
|
if ns != "" {
|
|
id += ns + "/"
|
|
}
|
|
|
|
if kc != "" {
|
|
if ns == "" {
|
|
// This is intentional to avoid conflating kc=,ns=foo,name=bar and kc=foo,ns=,name=bar.
|
|
// Before https://github.com/roboll/helmfile/pull/1823 they were both `foo/bar` which turned out to break `needs` in many ways.
|
|
//
|
|
// We now explicitly differentiate each with `foo//bar` and `foo/bar`.
|
|
// Note that `foo//bar` is not always a equivalent to `foo/default/bar` as the default namespace is depedent on
|
|
// the user's kubeconfig.
|
|
// That's why we use `foo//bar` even if it looked unintuitive.
|
|
id += "/"
|
|
}
|
|
}
|
|
|
|
id += r.Name
|
|
|
|
return id
|
|
}
|
|
|
|
func (st *HelmState) appendDeleteWaitFlags(args []string, release *ReleaseSpec) []string {
|
|
if release.DeleteWait != nil && *release.DeleteWait || release.DeleteWait == nil && st.HelmDefaults.DeleteWait {
|
|
args = append(args, "--wait")
|
|
timeout := st.HelmDefaults.DeleteTimeout
|
|
if release.DeleteTimeout != nil {
|
|
timeout = *release.DeleteTimeout
|
|
}
|
|
if timeout != 0 {
|
|
duration := strconv.Itoa(timeout)
|
|
duration += "s"
|
|
args = append(args, "--timeout", duration)
|
|
}
|
|
}
|
|
return args
|
|
}
|
|
|
|
// DeleteReleasesForSync deletes releases that are marked for deletion
|
|
func (st *HelmState) DeleteReleasesForSync(affectedReleases *AffectedReleases, helm helmexec.Interface, workerLimit int, cascade string) []error {
|
|
errs := []error{}
|
|
|
|
releases := st.Releases
|
|
|
|
jobQueue := make(chan *ReleaseSpec, len(releases))
|
|
results := make(chan syncResult, len(releases))
|
|
if workerLimit == 0 {
|
|
workerLimit = len(releases)
|
|
}
|
|
|
|
m := new(sync.Mutex)
|
|
|
|
st.scatterGather(
|
|
workerLimit,
|
|
len(releases),
|
|
func() {
|
|
for i := 0; i < len(releases); i++ {
|
|
jobQueue <- &releases[i]
|
|
}
|
|
close(jobQueue)
|
|
},
|
|
func(workerIndex int) {
|
|
for release := range jobQueue {
|
|
var relErr *ReleaseError
|
|
context := st.createHelmContext(release, workerIndex)
|
|
|
|
if _, err := st.triggerPresyncEvent(release, "sync"); err != nil {
|
|
relErr = newReleaseFailedError(release, err)
|
|
} else {
|
|
var args []string
|
|
if release.Namespace != "" {
|
|
args = append(args, "--namespace", release.Namespace)
|
|
}
|
|
args = st.appendDeleteWaitFlags(args, release)
|
|
args = st.appendConnectionFlags(args, release)
|
|
deletionFlags := st.appendCascadeFlags(args, helm, release, cascade)
|
|
|
|
m.Lock()
|
|
start := time.Now()
|
|
if _, err := st.triggerReleaseEvent("preuninstall", nil, release, "sync"); err != nil {
|
|
affectedReleases.DeleteFailed = append(affectedReleases.Failed, release)
|
|
relErr = newReleaseFailedError(release, err)
|
|
} else if err := helm.DeleteRelease(context, release.Name, deletionFlags...); err != nil {
|
|
affectedReleases.DeleteFailed = append(affectedReleases.Failed, release)
|
|
relErr = newReleaseFailedError(release, err)
|
|
} else if _, err := st.triggerReleaseEvent("postuninstall", nil, release, "sync"); err != nil {
|
|
affectedReleases.DeleteFailed = append(affectedReleases.Failed, release)
|
|
relErr = newReleaseFailedError(release, err)
|
|
} else {
|
|
affectedReleases.Deleted = append(affectedReleases.Deleted, release)
|
|
}
|
|
release.duration = time.Since(start)
|
|
m.Unlock()
|
|
}
|
|
|
|
if _, err := st.triggerPostsyncEvent(release, relErr, "sync"); err != nil {
|
|
st.logger.Warnf("warn: %v\n", err)
|
|
}
|
|
|
|
if _, err := st.TriggerCleanupEvent(release, "sync"); err != nil {
|
|
st.logger.Warnf("warn: %v\n", err)
|
|
}
|
|
|
|
if relErr == nil {
|
|
results <- syncResult{}
|
|
} else {
|
|
results <- syncResult{errors: []*ReleaseError{relErr}}
|
|
}
|
|
}
|
|
},
|
|
func() {
|
|
for i := 0; i < len(releases); {
|
|
res := <-results
|
|
if len(res.errors) > 0 {
|
|
for _, e := range res.errors {
|
|
errs = append(errs, e)
|
|
}
|
|
}
|
|
i++
|
|
}
|
|
},
|
|
)
|
|
if len(errs) > 0 {
|
|
return errs
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SyncReleases wrapper for executing helm upgrade on the releases
|
|
func (st *HelmState) SyncReleases(affectedReleases *AffectedReleases, helm helmexec.Interface, additionalValues []string, workerLimit int, opt ...SyncOpt) []error {
|
|
opts := &SyncOpts{}
|
|
for _, o := range opt {
|
|
o.Apply(opts)
|
|
}
|
|
|
|
preps, prepErrs := st.prepareSyncReleases(helm, additionalValues, workerLimit, opts)
|
|
|
|
if !opts.SkipCleanup {
|
|
defer func() {
|
|
for _, p := range preps {
|
|
st.removeFiles(p.files)
|
|
}
|
|
}()
|
|
}
|
|
|
|
if len(prepErrs) > 0 {
|
|
return prepErrs
|
|
}
|
|
|
|
errs := []error{}
|
|
jobQueue := make(chan *syncPrepareResult, len(preps))
|
|
results := make(chan syncResult, len(preps))
|
|
if workerLimit == 0 {
|
|
workerLimit = len(preps)
|
|
}
|
|
|
|
m := new(sync.Mutex)
|
|
|
|
st.scatterGather(
|
|
workerLimit,
|
|
len(preps),
|
|
func() {
|
|
for i := 0; i < len(preps); i++ {
|
|
jobQueue <- &preps[i]
|
|
}
|
|
close(jobQueue)
|
|
},
|
|
func(workerIndex int) {
|
|
for prep := range jobQueue {
|
|
release := prep.release
|
|
flags := prep.flags
|
|
// Use ChartPath directly if already set (normalized during chart preparation),
|
|
// otherwise normalize the original chart name.
|
|
chart := release.ChartPathOrName()
|
|
if release.ChartPath == "" {
|
|
chart = normalizeChart(st.basePath, chart)
|
|
}
|
|
var relErr *ReleaseError
|
|
context := st.createHelmContext(release, workerIndex)
|
|
|
|
start := time.Now()
|
|
if _, err := st.triggerPresyncEvent(release, "sync"); err != nil {
|
|
relErr = newReleaseFailedError(release, err)
|
|
} else if !release.Desired() {
|
|
installed, err := st.isReleaseInstalled(context, helm, *release)
|
|
if err != nil {
|
|
relErr = newReleaseFailedError(release, err)
|
|
} else if installed {
|
|
var args []string
|
|
deletionFlags := st.appendConnectionFlags(args, release)
|
|
m.Lock()
|
|
if _, err := st.triggerReleaseEvent("preuninstall", nil, release, "sync"); err != nil {
|
|
affectedReleases.Failed = append(affectedReleases.Failed, release)
|
|
relErr = newReleaseFailedError(release, err)
|
|
} else if err := helm.DeleteRelease(context, release.Name, deletionFlags...); err != nil {
|
|
affectedReleases.Failed = append(affectedReleases.Failed, release)
|
|
relErr = newReleaseFailedError(release, err)
|
|
} else if _, err := st.triggerReleaseEvent("postuninstall", nil, release, "sync"); err != nil {
|
|
affectedReleases.Failed = append(affectedReleases.Failed, release)
|
|
relErr = newReleaseFailedError(release, err)
|
|
} else {
|
|
affectedReleases.Deleted = append(affectedReleases.Deleted, release)
|
|
}
|
|
m.Unlock()
|
|
}
|
|
} else if release.UpdateStrategy == UpdateStrategyReinstallIfForbidden {
|
|
relErr = st.performSyncOrReinstallOfRelease(affectedReleases, helm, context, release, chart, m, flags...)
|
|
if relErr == nil {
|
|
relErr = st.trackReleaseIfEnabled(gocontext.Background(), release, helm, opts)
|
|
}
|
|
} else {
|
|
trackHandle, trackStarted := st.startBackgroundKubedogTracking(gocontext.Background(), release, helm, opts)
|
|
// trackHandle.Helm is a logger-scoped helm clone that
|
|
// captures output to an in-memory buffer while tracking is
|
|
// active. When tracking isn't running it's the original
|
|
// helm — either path is safe to call directly.
|
|
releaseHelm := trackHandle.Helm
|
|
helmErr := st.withChartOperationLock(release, chart, func() error {
|
|
return releaseHelm.SyncRelease(context, release.Name, chart, release.Namespace, flags...)
|
|
})
|
|
if helmErr != nil && trackStarted && trackHandle.WasHelmKilled() {
|
|
// The kubedog safety valve sent SIGINT to helm because
|
|
// the cluster confirmed convergence while helm was
|
|
// wedged on its hook waiter. Treat the resulting helm
|
|
// error as success and proceed via the post-helm-done
|
|
// path.
|
|
helmErr = nil
|
|
}
|
|
if helmErr != nil {
|
|
if trackStarted {
|
|
trackHandle.Cancel()
|
|
_ = trackHandle.Wait()
|
|
} else {
|
|
trackHandle.FlushBufferedHelmOutput()
|
|
}
|
|
m.Lock()
|
|
affectedReleases.Failed = append(affectedReleases.Failed, release)
|
|
m.Unlock()
|
|
relErr = newReleaseFailedError(release, helmErr)
|
|
} else {
|
|
if trackStarted {
|
|
trackHandle.NotifyHelmDone()
|
|
}
|
|
m.Lock()
|
|
affectedReleases.Upgraded = append(affectedReleases.Upgraded, release)
|
|
m.Unlock()
|
|
// Skip the deployed-version probe when helm was killed
|
|
// by the safety valve — releaseHelm's context is now
|
|
// canceled so the probe would just fail noisily, and
|
|
// the release secret is likely in pending-install
|
|
// state anyway.
|
|
if !(trackStarted && trackHandle.WasHelmKilled()) {
|
|
installedVersion, err := st.getDeployedVersion(context, releaseHelm, release)
|
|
if err != nil { // err is not really impacting so just log it
|
|
st.logger.Debugf("getting deployed release version failed: %v", err)
|
|
} else {
|
|
release.installedVersion = installedVersion
|
|
}
|
|
}
|
|
|
|
var trackErr *ReleaseError
|
|
if trackStarted {
|
|
trackErr = trackHandle.Wait()
|
|
} else {
|
|
trackErr = st.trackReleaseIfEnabled(gocontext.Background(), release, helm, opts)
|
|
}
|
|
if trackErr != nil {
|
|
m.Lock()
|
|
affectedReleases.Failed = append(affectedReleases.Failed, release)
|
|
m.Unlock()
|
|
relErr = trackErr
|
|
}
|
|
}
|
|
}
|
|
|
|
if _, err := st.triggerPostsyncEvent(release, relErr, "sync"); err != nil {
|
|
if relErr == nil {
|
|
relErr = newReleaseFailedError(release, err)
|
|
} else {
|
|
st.logger.Warnf("warn: %v\n", err)
|
|
}
|
|
}
|
|
|
|
if _, err := st.TriggerCleanupEvent(release, "sync"); err != nil {
|
|
if relErr == nil {
|
|
relErr = newReleaseFailedError(release, err)
|
|
} else {
|
|
st.logger.Warnf("warn: %v\n", err)
|
|
}
|
|
}
|
|
release.duration = time.Since(start)
|
|
|
|
if relErr == nil {
|
|
results <- syncResult{}
|
|
} else {
|
|
results <- syncResult{errors: []*ReleaseError{relErr}}
|
|
}
|
|
}
|
|
},
|
|
func() {
|
|
for i := 0; i < len(preps); {
|
|
res := <-results
|
|
if len(res.errors) > 0 {
|
|
for _, e := range res.errors {
|
|
errs = append(errs, e)
|
|
}
|
|
}
|
|
i++
|
|
}
|
|
},
|
|
)
|
|
if len(errs) > 0 {
|
|
return errs
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (st *HelmState) performSyncOrReinstallOfRelease(affectedReleases *AffectedReleases, helm helmexec.Interface, context helmexec.HelmContext, release *ReleaseSpec, chart string, m *sync.Mutex, flags ...string) *ReleaseError {
|
|
if err := st.withChartOperationLock(release, chart, func() error {
|
|
return helm.SyncRelease(context, release.Name, chart, release.Namespace, flags...)
|
|
}); err != nil {
|
|
st.logger.Debugf("update strategy - sync failed: %s", err.Error())
|
|
// Only fail if a different error than forbidden updates
|
|
if !strings.Contains(err.Error(), "Forbidden: updates") {
|
|
st.logger.Debugf("update strategy - sync failed not due to Forbidden updates")
|
|
m.Lock()
|
|
affectedReleases.Failed = append(affectedReleases.Failed, release)
|
|
m.Unlock()
|
|
return newReleaseFailedError(release, err)
|
|
}
|
|
} else {
|
|
st.logger.Debugf("update strategy - sync success")
|
|
m.Lock()
|
|
affectedReleases.Upgraded = append(affectedReleases.Upgraded, release)
|
|
m.Unlock()
|
|
installedVersion, err := st.getDeployedVersion(context, helm, release)
|
|
if err != nil { // err is not really impacting so just log it
|
|
st.logger.Debugf("update strategy - getting deployed release version failed: %v", err)
|
|
} else {
|
|
release.installedVersion = installedVersion
|
|
}
|
|
return nil
|
|
}
|
|
|
|
st.logger.Infof("Failed to sync due to forbidden updates, attempting to reinstall %q allowed by update strategy", release.Name)
|
|
installed, err := st.isReleaseInstalled(context, helm, *release)
|
|
if err != nil {
|
|
return newReleaseFailedError(release, err)
|
|
}
|
|
if installed {
|
|
var args []string
|
|
if release.Namespace != "" {
|
|
args = append(args, "--namespace", release.Namespace)
|
|
}
|
|
deleteWaitFlag := true
|
|
release.DeleteWait = &deleteWaitFlag
|
|
args = st.appendDeleteWaitFlags(args, release)
|
|
deletionFlags := st.appendConnectionFlags(args, release)
|
|
m.Lock()
|
|
if _, err := st.triggerReleaseEvent("preuninstall", nil, release, "sync"); err != nil {
|
|
affectedReleases.Failed = append(affectedReleases.Failed, release)
|
|
return newReleaseFailedError(release, err)
|
|
} else if err := helm.DeleteRelease(context, release.Name, deletionFlags...); err != nil {
|
|
affectedReleases.Failed = append(affectedReleases.Failed, release)
|
|
return newReleaseFailedError(release, err)
|
|
} else if _, err := st.triggerReleaseEvent("postuninstall", nil, release, "sync"); err != nil {
|
|
affectedReleases.Failed = append(affectedReleases.Failed, release)
|
|
return newReleaseFailedError(release, err)
|
|
}
|
|
m.Unlock()
|
|
}
|
|
if err := st.withChartOperationLock(release, chart, func() error {
|
|
return helm.SyncRelease(context, release.Name, chart, release.Namespace, flags...)
|
|
}); err != nil {
|
|
m.Lock()
|
|
affectedReleases.Failed = append(affectedReleases.Failed, release)
|
|
m.Unlock()
|
|
return newReleaseFailedError(release, err)
|
|
} else {
|
|
m.Lock()
|
|
affectedReleases.Reinstalled = append(affectedReleases.Reinstalled, release)
|
|
m.Unlock()
|
|
installedVersion, err := st.getDeployedVersion(context, helm, release)
|
|
if err != nil { // err is not really impacting so just log it
|
|
st.logger.Debugf("update strategy - getting deployed release version failed: %v", err)
|
|
} else {
|
|
release.installedVersion = installedVersion
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (st *HelmState) listReleases(context helmexec.HelmContext, helm helmexec.Interface, release *ReleaseSpec) (string, error) {
|
|
flags := st.kubeConnectionFlags(release)
|
|
if release.Namespace != "" {
|
|
flags = append(flags, "--namespace", release.Namespace)
|
|
}
|
|
flags = append(flags, "--uninstalling")
|
|
flags = append(flags, "--deployed", "--failed", "--pending")
|
|
return helm.List(context, "^"+release.Name+"$", flags...)
|
|
}
|
|
|
|
func (st *HelmState) getDeployedVersion(context helmexec.HelmContext, helm helmexec.Interface, release *ReleaseSpec) (string, error) {
|
|
//retrieve the version
|
|
if out, err := st.listReleases(context, helm, release); err == nil {
|
|
chartName := filepath.Base(release.Chart)
|
|
//the regexp without escapes : .*\s.*\s.*\s.*\schartName-(.*?)\s
|
|
pat := regexp.MustCompile(".*\\s.*\\s.*\\s.*\\s" + chartName + "-(.*?)\\s")
|
|
versions := pat.FindStringSubmatch(out)
|
|
if len(versions) > 0 {
|
|
return versions[1], nil
|
|
} else {
|
|
chartMetadata, err := helm.ShowChart(release.Chart)
|
|
if err != nil {
|
|
return "failed to get version", errors.New("Failed to get the version for: " + chartName)
|
|
}
|
|
return chartMetadata.Version, nil
|
|
}
|
|
} else {
|
|
return "failed to get version", err
|
|
}
|
|
}
|
|
|
|
func releasesNeedCharts(releases []ReleaseSpec) []ReleaseSpec {
|
|
var result []ReleaseSpec
|
|
|
|
for _, r := range releases {
|
|
if r.Desired() {
|
|
result = append(result, r)
|
|
}
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
func filterReleasesForBuild(releases []ReleaseSpec) []ReleaseSpec {
|
|
var filteredReleases []ReleaseSpec
|
|
for _, r := range releases {
|
|
if len(r.JSONPatches) == 0 && len(r.StrategicMergePatches) == 0 && len(r.Transformers) == 0 {
|
|
filteredReleases = append(filteredReleases, r)
|
|
}
|
|
}
|
|
return filteredReleases
|
|
}
|
|
|
|
type ChartPrepareOptions struct {
|
|
ForceDownload bool
|
|
SkipRepos bool
|
|
SkipDeps bool
|
|
SkipRefresh bool
|
|
SkipResolve bool
|
|
SkipCleanup bool
|
|
// SkipSchemaValidation configures chartify to pass --skip-schema-validation to helm-template run by it.
|
|
SkipSchemaValidation bool
|
|
// Validate configures chartify to pass --validate to helm-template run by it.
|
|
// It's required when one of your chart relies on Capabilities.APIVersions in a template
|
|
Validate bool
|
|
IncludeCRDs *bool
|
|
Wait bool
|
|
WaitRetries int
|
|
WaitForJobs bool
|
|
OutputDir string
|
|
OutputDirTemplate string
|
|
IncludeTransitiveNeeds bool
|
|
Concurrency int
|
|
KubeVersion string
|
|
Set []string
|
|
Values []string
|
|
// Delete wait
|
|
DeleteWait bool
|
|
DeleteTimeout int
|
|
// HelmOCIPlainHTTP uses plain HTTP for OCI registries (required for local/insecure registries in Helm 4)
|
|
HelmOCIPlainHTTP bool
|
|
}
|
|
|
|
type chartPrepareResult struct {
|
|
releaseName string
|
|
releaseNamespace string
|
|
releaseContext string
|
|
chartName string
|
|
chartPath string
|
|
err error
|
|
buildDeps bool
|
|
skipRefresh bool
|
|
chartFetchedByGoGetter bool
|
|
}
|
|
|
|
func (st *HelmState) GetRepositoryAndNameFromChartName(chartName string) (*RepositorySpec, string) {
|
|
chart := strings.Split(chartName, "/")
|
|
if len(chart) == 1 {
|
|
return nil, chartName
|
|
}
|
|
repo := chart[0]
|
|
for _, r := range st.Repositories {
|
|
if r.Name == repo {
|
|
return &r, strings.Join(chart[1:], "/")
|
|
}
|
|
}
|
|
return nil, chartName
|
|
}
|
|
|
|
// resolveOCIAdhocDepChart rewrites a release `dependencies[].chart` value that
|
|
// uses the named-repo prefix form ("repoName/chartName") into a full oci:// URL
|
|
// when the prefix matches a `repositories:` entry with `oci: true`. It returns
|
|
// (rewritten, true) on a hit and ("", false) otherwise.
|
|
//
|
|
// This avoids the chartify path that does `helm repo list` to look up the
|
|
// repository URL: that lookup never finds OCI repos because helm 3+ does not
|
|
// register OCI registries as named repos (it uses `helm registry login`
|
|
// instead). By the time chartify sees an `oci://` URL it already takes the
|
|
// correct branch, so rewriting here is enough to make the named-repo form
|
|
// behave the same as the explicit URL form.
|
|
func (st *HelmState) resolveOCIAdhocDepChart(chart string) (string, bool) {
|
|
if strings.HasPrefix(chart, "oci://") {
|
|
return "", false
|
|
}
|
|
repo, name := st.GetRepositoryAndNameFromChartName(chart)
|
|
if repo == nil || !repo.OCI {
|
|
return "", false
|
|
}
|
|
return "oci://" + strings.TrimSuffix(repo.URL, "/") + "/" + name, true
|
|
}
|
|
|
|
var rwMutexMap sync.Map
|
|
|
|
// getNamedRWMutex retrieves or creates a sync.RWMutex for a given name.
|
|
// This is used for reader-writer locking where multiple readers can access
|
|
// the resource simultaneously, but writers need exclusive access.
|
|
func (st *HelmState) getNamedRWMutex(name string) *sync.RWMutex {
|
|
mu, ok := rwMutexMap.Load(name)
|
|
if ok {
|
|
return mu.(*sync.RWMutex)
|
|
}
|
|
newMu := &sync.RWMutex{}
|
|
actualMu, _ := rwMutexMap.LoadOrStore(name, newMu)
|
|
return actualMu.(*sync.RWMutex)
|
|
}
|
|
|
|
// withChartOperationLock serializes helm operations (upgrade, diff) per
|
|
// chart+version for remote charts to prevent concurrent download races on
|
|
// helm's repository cache (issue #768). When multiple releases use the same
|
|
// remote chart, concurrent `helm upgrade`/`helm diff` calls race on helm's
|
|
// internal cache file rename, causing "Access is denied" errors on Windows.
|
|
//
|
|
// For local or pre-fetched charts (release.ChartPath is set), no lock is
|
|
// acquired because the chart is already available locally and helm won't
|
|
// download it.
|
|
//
|
|
// Trade-off: the lock wraps the entire helm operation (download + template +
|
|
// deploy), not just the download. Same-chart releases are fully serialized.
|
|
// This is unavoidable without pre-fetching because helm downloads and deploys
|
|
// atomically. Releases with different charts are unaffected and remain fully
|
|
// parallel. If kubedog tracking is enabled, note that it starts before this
|
|
// lock is acquired, so a release waiting on the lock may have its kubedog
|
|
// watcher active before helm begins.
|
|
func (st *HelmState) withChartOperationLock(release *ReleaseSpec, chart string, fn func() error) error {
|
|
if release.ChartPath != "" {
|
|
return fn()
|
|
}
|
|
mu := st.getNamedRWMutex("helm-op:" + chart + ":" + release.Version)
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
return fn()
|
|
}
|
|
|
|
type PrepareChartKey struct {
|
|
Namespace, Name, KubeContext string
|
|
}
|
|
|
|
// rewriteChartDependencies rewrites relative file:// dependencies in Chart.yaml to absolute paths
|
|
// to ensure they can be resolved from chartify's temporary directory.
|
|
// Instead of modifying the original chart in-place (which causes race conditions when multiple
|
|
// releases reference the same local chart), it creates a temporary copy only when a rewrite is
|
|
// needed and rewrites that copy. When a temp copy is created, it reflects the current chart
|
|
// contents so prepare hooks or other steps that mutate the local chart directory are honored.
|
|
// The returned cleanup function removes the temporary directory when one was created and is
|
|
// otherwise a no-op.
|
|
func (st *HelmState) rewriteChartDependencies(chartPath string) (string, func(), error) {
|
|
chartYamlPath := filepath.Join(chartPath, "Chart.yaml")
|
|
|
|
if _, err := st.fs.Stat(chartYamlPath); os.IsNotExist(err) {
|
|
return chartPath, func() {}, nil
|
|
} else if err != nil {
|
|
return chartPath, func() {}, err
|
|
}
|
|
|
|
data, err := st.fs.ReadFile(chartYamlPath)
|
|
if err != nil {
|
|
return chartPath, func() {}, err
|
|
}
|
|
|
|
type ChartDependency struct {
|
|
Name string `yaml:"name"`
|
|
Repository string `yaml:"repository"`
|
|
Data map[string]interface{} `yaml:",inline"`
|
|
}
|
|
type ChartMeta struct {
|
|
Dependencies []ChartDependency `yaml:"dependencies,omitempty"`
|
|
Data map[string]interface{} `yaml:",inline"`
|
|
}
|
|
|
|
var chartMeta ChartMeta
|
|
if err := yaml.Unmarshal(data, &chartMeta); err != nil {
|
|
return chartPath, func() {}, err
|
|
}
|
|
|
|
modified := false
|
|
for i := range chartMeta.Dependencies {
|
|
dep := &chartMeta.Dependencies[i]
|
|
if strings.HasPrefix(dep.Repository, "file://") {
|
|
relPath := strings.TrimPrefix(dep.Repository, "file://")
|
|
|
|
if !filepath.IsAbs(relPath) {
|
|
absPath := filepath.Join(chartPath, relPath)
|
|
absPath, err = filepath.Abs(absPath)
|
|
if err != nil {
|
|
return chartPath, func() {}, fmt.Errorf("failed to resolve absolute path for dependency %s: %w", dep.Name, err)
|
|
}
|
|
|
|
st.logger.Debugf("Rewriting Chart dependency %s from %s to file://%s", dep.Name, dep.Repository, absPath)
|
|
dep.Repository = "file://" + absPath
|
|
modified = true
|
|
}
|
|
}
|
|
}
|
|
|
|
if !modified {
|
|
return chartPath, func() {}, nil
|
|
}
|
|
|
|
updatedData, err := yaml.Marshal(&chartMeta)
|
|
if err != nil {
|
|
return chartPath, func() {}, fmt.Errorf("failed to marshal Chart.yaml: %w", err)
|
|
}
|
|
|
|
tempDir, err := st.fs.MkdirTemp("", "chart-deps-rewrite-*")
|
|
if err != nil {
|
|
return chartPath, func() {}, fmt.Errorf("failed to create temp directory for chart rewrite: %w", err)
|
|
}
|
|
|
|
if err := st.fs.CopyDir(chartPath, tempDir); err != nil {
|
|
if removeErr := st.fs.RemoveAll(tempDir); removeErr != nil {
|
|
st.logger.Warnf("Failed to remove temp chart directory %s: %v", tempDir, removeErr)
|
|
}
|
|
return chartPath, func() {}, fmt.Errorf("failed to copy chart to temp directory: %w", err)
|
|
}
|
|
|
|
tempChartYamlPath := filepath.Join(tempDir, "Chart.yaml")
|
|
if err := st.fs.WriteFile(tempChartYamlPath, updatedData, 0644); err != nil {
|
|
if removeErr := st.fs.RemoveAll(tempDir); removeErr != nil {
|
|
st.logger.Warnf("Failed to remove temp chart directory %s: %v", tempDir, removeErr)
|
|
}
|
|
return chartPath, func() {}, fmt.Errorf("failed to write Chart.yaml: %w", err)
|
|
}
|
|
|
|
st.logger.Debugf("Rewrote Chart.yaml with absolute dependency paths at %s", tempChartYamlPath)
|
|
|
|
// Rewriting Chart.yaml invalidates Chart.lock's digest, since helm computes the
|
|
// digest over the JSON-marshaled dependencies block. If the lock isn't refreshed,
|
|
// downstream `helm dependency build` errors with "lock file is out of sync with
|
|
// the dependencies file" and falls back to `dependency update`, which re-resolves
|
|
// version constraints (e.g. `version: "*"`) against the chart repo and silently
|
|
// pulls newer dependency versions. The version pins in the lock are still the
|
|
// intended truth — only the rewritten file:// repository URL changed. Mirror the
|
|
// rewrite into the lock and recompute the digest so `dep build` accepts it.
|
|
tempChartLockPath := filepath.Join(tempDir, "Chart.lock")
|
|
lockData, lockErr := st.fs.ReadFile(tempChartLockPath)
|
|
if lockErr != nil && !os.IsNotExist(lockErr) {
|
|
st.logger.Warnf("Failed to read Chart.lock at %s: %v", tempChartLockPath, lockErr)
|
|
}
|
|
if lockErr == nil {
|
|
var lock struct {
|
|
Dependencies []*helmchart.Dependency `yaml:"dependencies,omitempty"`
|
|
Digest string `yaml:"digest,omitempty"`
|
|
Generated string `yaml:"generated,omitempty"`
|
|
}
|
|
if err := yaml.Unmarshal(lockData, &lock); err != nil {
|
|
st.logger.Warnf("Failed to parse Chart.lock at %s: %v", tempChartLockPath, err)
|
|
} else {
|
|
// Build the request slice (rewritten Chart.yaml dependencies) using helm's
|
|
// own chart.Dependency type so the JSON used for hashing matches helm's
|
|
// exactly. All supported fields must be mapped, not just name/repository/
|
|
// version, because helm's digest algorithm hashes the full Dependency struct.
|
|
req := make([]*helmchart.Dependency, 0, len(chartMeta.Dependencies))
|
|
for _, d := range chartMeta.Dependencies {
|
|
dep := &helmchart.Dependency{
|
|
Name: d.Name,
|
|
Repository: d.Repository,
|
|
}
|
|
if v, ok := d.Data["version"].(string); ok {
|
|
dep.Version = v
|
|
}
|
|
if v, ok := d.Data["condition"].(string); ok {
|
|
dep.Condition = v
|
|
}
|
|
if v, ok := d.Data["alias"].(string); ok {
|
|
dep.Alias = v
|
|
}
|
|
if v, ok := d.Data["enabled"].(bool); ok {
|
|
dep.Enabled = v
|
|
}
|
|
if v, ok := d.Data["tags"].([]interface{}); ok {
|
|
tags := make([]string, 0, len(v))
|
|
for _, t := range v {
|
|
if s, ok := t.(string); ok {
|
|
tags = append(tags, s)
|
|
}
|
|
}
|
|
dep.Tags = tags
|
|
}
|
|
if v, ok := d.Data["import-values"].([]interface{}); ok {
|
|
normalized, err := maputil.RecursivelyStringifyMapKey(v)
|
|
if err != nil {
|
|
st.logger.Warnf("Failed to normalize import-values for dependency %s: %v", d.Name, err)
|
|
} else {
|
|
dep.ImportValues = normalized.([]interface{})
|
|
}
|
|
}
|
|
req = append(req, dep)
|
|
}
|
|
|
|
// Mirror the rewritten file:// repository URLs onto matching lock entries.
|
|
// Without this, `helm dependency build` would resolve the lock's relative
|
|
// file:// paths against the (moved) chart directory and fail with
|
|
// "directory ... not found". Versions in the lock are left untouched.
|
|
// Match on Name + Alias to handle charts with duplicate dependency names
|
|
// distinguished by alias.
|
|
for _, ld := range lock.Dependencies {
|
|
if !strings.HasPrefix(ld.Repository, "file://") {
|
|
continue
|
|
}
|
|
for _, rd := range req {
|
|
if rd.Name == ld.Name && rd.Alias == ld.Alias && strings.HasPrefix(rd.Repository, "file://") {
|
|
ld.Repository = rd.Repository
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// Normalize lock.Dependencies ImportValues to avoid json.Marshal failures
|
|
// when go-yaml v2 decodes nested maps as map[interface{}]interface{}.
|
|
for _, ld := range lock.Dependencies {
|
|
if ld.ImportValues != nil {
|
|
normalized, err := maputil.RecursivelyStringifyMapKey(ld.ImportValues)
|
|
if err != nil {
|
|
st.logger.Warnf("Failed to normalize import-values in Chart.lock for dependency %s: %v", ld.Name, err)
|
|
} else {
|
|
ld.ImportValues = normalized.([]interface{})
|
|
}
|
|
}
|
|
}
|
|
|
|
// Replicates helm's resolver.HashReq:
|
|
// json.Marshal([2][]*chart.Dependency{req, lock}) → sha256 hex.
|
|
// resolver.HashReq lives in helm.sh/helm/v3/internal/resolver, so we
|
|
// inline the (small, stable) algorithm rather than importing it.
|
|
if payload, err := json.Marshal([2][]*helmchart.Dependency{req, lock.Dependencies}); err != nil {
|
|
st.logger.Warnf("Failed to marshal deps for Chart.lock digest at %s: %v", tempChartLockPath, err)
|
|
} else {
|
|
sum := sha256.Sum256(payload)
|
|
lock.Digest = "sha256:" + hex.EncodeToString(sum[:])
|
|
if updated, err := yaml.Marshal(&lock); err != nil {
|
|
st.logger.Warnf("Failed to marshal Chart.lock at %s: %v", tempChartLockPath, err)
|
|
} else if err := st.fs.WriteFile(tempChartLockPath, updated, 0644); err != nil {
|
|
st.logger.Warnf("Failed to write Chart.lock at %s: %v", tempChartLockPath, err)
|
|
} else {
|
|
st.logger.Debugf("Refreshed Chart.lock digest at %s after Chart.yaml rewrite", tempChartLockPath)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
cleanup := func() {
|
|
if removeErr := st.fs.RemoveAll(tempDir); removeErr != nil {
|
|
st.logger.Warnf("Failed to remove temp chart directory %s: %v", tempDir, removeErr)
|
|
}
|
|
}
|
|
|
|
return tempDir, cleanup, nil
|
|
}
|
|
|
|
// Otherwise, if a chart is not a helm chart, it will call "chartify" to turn it into a chart.
|
|
//
|
|
// If exists, it will also patch resources by json patches, strategic-merge patches, and injectors.
|
|
// processChartification handles the chartification process
|
|
func (st *HelmState) processChartification(chartification *Chartify, release *ReleaseSpec, chartPath string, opts ChartPrepareOptions, skipDeps bool, helmfileCommand string) (string, bool, error) {
|
|
// Rewrite relative file:// dependencies in Chart.yaml to absolute paths before chartify processes them
|
|
// This prevents errors like "Error: directory /tmp/chartify.../argocd-application not found"
|
|
// when Chart.yaml contains dependencies like "file://../argocd-application"
|
|
if st.fs.DirectoryExistsAt(chartPath) {
|
|
rewrittenPath, cleanupTempChart, err := st.rewriteChartDependencies(chartPath)
|
|
if err != nil {
|
|
return "", false, fmt.Errorf("failed to rewrite chart dependencies: %w", err)
|
|
}
|
|
chartPath = rewrittenPath
|
|
defer cleanupTempChart()
|
|
}
|
|
|
|
c := chartify.New(
|
|
chartify.HelmBin(st.DefaultHelmBinary),
|
|
chartify.KustomizeBin(st.DefaultKustomizeBinary),
|
|
// Auto-detect Helm version (works with both Helm 3 and Helm 4)
|
|
chartify.WithLogf(st.logger.Debugf),
|
|
)
|
|
|
|
chartifyOpts := chartification.Opts
|
|
|
|
if skipDeps {
|
|
chartifyOpts.SkipDeps = true
|
|
}
|
|
|
|
// Pass --oci-plain-http flag to chartify for Helm 4 OCI support
|
|
if opts.HelmOCIPlainHTTP {
|
|
chartifyOpts.OCIPlainHTTP = true
|
|
}
|
|
|
|
includeCRDs := true
|
|
if opts.IncludeCRDs != nil {
|
|
includeCRDs = *opts.IncludeCRDs
|
|
}
|
|
chartifyOpts.IncludeCRDs = includeCRDs
|
|
|
|
chartifyOpts.Validate = opts.Validate
|
|
|
|
chartifyOpts.KubeVersion = st.getKubeVersion(release, opts.KubeVersion)
|
|
chartifyOpts.ApiVersions = st.getApiVersions(release)
|
|
|
|
if opts.Values != nil {
|
|
chartifyOpts.ValuesFiles = append(opts.Values, chartifyOpts.ValuesFiles...)
|
|
}
|
|
|
|
// https://github.com/helmfile/helmfile/pull/867
|
|
// https://github.com/helmfile/helmfile/issues/895
|
|
var flags []string
|
|
for _, s := range opts.Set {
|
|
flags = append(flags, "--set", s)
|
|
}
|
|
chartifyOpts.SetFlags = append(chartifyOpts.SetFlags, flags...)
|
|
|
|
// Enable cluster connectivity for lookup functions when using kustomize patches
|
|
// Issue #2271: helm template runs client-side by default, causing lookup() to return empty values
|
|
// Pass --dry-run=server to enable cluster access for lookup while still using client-side rendering
|
|
// Only do this for operations that already require cluster access
|
|
var requiresCluster bool
|
|
switch helmfileCommand {
|
|
case "diff", "apply", "sync", "destroy", "delete", "test", "status":
|
|
// Commands that interact with the cluster
|
|
requiresCluster = true
|
|
case "template", "lint", "build", "pull", "fetch", "write-values", "list", "show-dag", "deps", "repos", "cache", "init", "completion", "help", "version":
|
|
// Commands that work locally without cluster access
|
|
requiresCluster = false
|
|
default:
|
|
// For unknown commands, assume cluster access (safer default)
|
|
requiresCluster = true
|
|
}
|
|
|
|
// Enable --dry-run=server for cluster-requiring commands to support lookup() function
|
|
// Issue #2271: helm template runs client-side by default, causing lookup() to return empty values
|
|
// The lookup() function can be used with or without patches, so we enable cluster access
|
|
// for all cluster-requiring operations (diff, apply, sync, etc.) but not for offline
|
|
// commands (template, lint, build, etc.)
|
|
// Issue #2309: Also pass --kube-context to chartify's internal helm template call
|
|
// Issue #2355: In Helm 4, --validate and --dry-run are mutually exclusive flags.
|
|
// When --validate is set, we skip adding --dry-run=server since --validate already
|
|
// provides server-side validation.
|
|
if requiresCluster {
|
|
// Get the effective kube-context for this release
|
|
kubeContext := st.getKubeContext(release)
|
|
|
|
// Build the additional args needed for cluster-requiring commands
|
|
var additionalArgs []string
|
|
|
|
// Add --kubeconfig if configured (Issue #2444)
|
|
if st.kubeconfig != "" && !strings.Contains(chartifyOpts.TemplateArgs, "--kubeconfig") {
|
|
additionalArgs = append(additionalArgs, "--kubeconfig", st.kubeconfig)
|
|
}
|
|
|
|
// Add --kube-context if configured (Issue #2309)
|
|
// Note: kube-context is independent of the validate/dry-run mutual exclusion
|
|
if kubeContext != "" && !strings.Contains(chartifyOpts.TemplateArgs, "--kube-context") {
|
|
additionalArgs = append(additionalArgs, "--kube-context", kubeContext)
|
|
}
|
|
|
|
// Add --dry-run=server if not already present (Issue #2271)
|
|
// Skip if --validate is set to avoid Helm 4 mutual exclusion error (Issue #2355)
|
|
if !opts.Validate && !strings.Contains(chartifyOpts.TemplateArgs, "--dry-run") {
|
|
additionalArgs = append(additionalArgs, "--dry-run=server")
|
|
}
|
|
|
|
// Append the additional args to TemplateArgs
|
|
if len(additionalArgs) > 0 {
|
|
if chartifyOpts.TemplateArgs == "" {
|
|
chartifyOpts.TemplateArgs = strings.Join(additionalArgs, " ")
|
|
} else {
|
|
chartifyOpts.TemplateArgs += " " + strings.Join(additionalArgs, " ")
|
|
}
|
|
}
|
|
}
|
|
|
|
chartifyOpts.TemplateArgs = st.appendSkipSchemaValidationFlagToChartifyTemplateArgs(
|
|
chartifyOpts.TemplateArgs,
|
|
release,
|
|
opts.SkipSchemaValidation,
|
|
)
|
|
|
|
out, err := c.Chartify(release.Name, chartPath, chartify.WithChartifyOpts(chartifyOpts))
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
|
|
chartPath = out
|
|
// Track the chartify output directory for cleanup after all helm operations
|
|
// complete. The chartified chart at `out` is used by subsequent helm commands,
|
|
// so it cannot be removed yet. See issue #1799.
|
|
st.addChartifyTempDir(out)
|
|
// Skip `helm dep build` and `helm dep up` altogether when the chart is from remote or the dep is
|
|
// explicitly skipped.
|
|
buildDeps := !skipDeps
|
|
return chartPath, buildDeps, nil
|
|
}
|
|
|
|
func (st *HelmState) appendSkipSchemaValidationFlagToChartifyTemplateArgs(templateArgs string, release *ReleaseSpec, skipSchemaValidation bool) string {
|
|
if !st.shouldSkipSchemaValidation(release, skipSchemaValidation) || hasTemplateArg(templateArgs, "--skip-schema-validation") {
|
|
return templateArgs
|
|
}
|
|
|
|
return appendTemplateArg(templateArgs, "--skip-schema-validation")
|
|
}
|
|
|
|
func hasTemplateArg(templateArgs, arg string) bool {
|
|
for _, token := range strings.Fields(templateArgs) {
|
|
if token == arg || strings.HasPrefix(token, arg+"=") {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func appendTemplateArg(templateArgs, arg string) string {
|
|
if templateArgs == "" {
|
|
return arg
|
|
}
|
|
return templateArgs + " " + arg
|
|
}
|
|
|
|
// processLocalChart handles local chart processing
|
|
func (st *HelmState) processLocalChart(normalizedChart, dir string, release *ReleaseSpec, helmfileCommand string, opts ChartPrepareOptions, isLocal bool) (string, error) {
|
|
chartPath := normalizedChart
|
|
if helmfileCommand == "pull" && isLocal {
|
|
chartAbsPath := strings.TrimSuffix(filepath.Clean(normalizedChart), "/")
|
|
var err error
|
|
chartPath, err = st.generateChartPath(filepath.Base(chartAbsPath), dir, release, opts.OutputDirTemplate)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if err := st.fs.CopyDir(normalizedChart, filepath.Clean(chartPath)); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
return chartPath, nil
|
|
}
|
|
|
|
// forcedDownloadChart handles forced chart downloads.
|
|
// Locks are acquired during download and released immediately after.
|
|
// A per-chart+version mutex serializes downloads within the process so that
|
|
// concurrent releases using the same chart don't race on helm's repository
|
|
// cache (issue #768).
|
|
func (st *HelmState) forcedDownloadChart(chartName, dir string, release *ReleaseSpec, helm helmexec.Interface, opts ChartPrepareOptions) (string, error) {
|
|
cacheKey := st.getChartCacheKey(release)
|
|
|
|
// Fast path: check in-process cache without acquiring any lock.
|
|
// If found, another worker in this process already downloaded the chart.
|
|
// We don't need to acquire a lock - the tempDir won't be deleted until
|
|
// after helm operations complete (cleanup is deferred in withPreparedCharts).
|
|
if cachedPath, exists := st.checkChartCache(cacheKey); exists && st.fs.DirectoryExistsAt(cachedPath) {
|
|
st.logger.Debugf("Chart %s:%s already downloaded, using cached version at %s", chartName, release.Version, cachedPath)
|
|
return cachedPath, nil
|
|
}
|
|
|
|
// Serialize downloads per chart+version within this process.
|
|
// The chartPath generated below includes the release name (see
|
|
// DefaultFetchOutputDirTemplate), so per-chartPath file locks do NOT
|
|
// prevent concurrent `helm fetch` of the same chart by different releases.
|
|
// Without this mutex, two workers downloading the same chart concurrently
|
|
// race on helm's repository cache on Windows (issue #768).
|
|
downloadMu := st.getNamedRWMutex("chart-download:" + cacheKey.Chart + ":" + cacheKey.Version)
|
|
downloadMu.Lock()
|
|
defer downloadMu.Unlock()
|
|
|
|
// Double-check: another worker may have completed the download while we waited.
|
|
if cachedPath, exists := st.checkChartCache(cacheKey); exists && st.fs.DirectoryExistsAt(cachedPath) {
|
|
st.logger.Debugf("Chart %s:%s downloaded by another worker, using cached version at %s", chartName, release.Version, cachedPath)
|
|
return cachedPath, nil
|
|
}
|
|
|
|
chartPath, err := st.generateChartPath(chartName, dir, release, opts.OutputDirTemplate)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
// Acquire locks and determine action
|
|
// Pass a callback that checks if another worker in this process has already cached the chart.
|
|
// This prevents deleting and re-downloading a chart that was just downloaded by another worker.
|
|
lockResult, err := st.acquireChartLock(chartPath, opts, func() bool {
|
|
_, exists := st.checkChartCache(cacheKey)
|
|
return exists
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
// If cached on filesystem, release lock and return
|
|
if lockResult.action == chartActionUseCached {
|
|
st.addToChartCache(cacheKey, lockResult.cachedPath)
|
|
lockResult.Release(st.logger)
|
|
return lockResult.cachedPath, nil
|
|
}
|
|
|
|
// Download the chart
|
|
var fetchFlags []string
|
|
fetchFlags = st.appendChartVersionFlags(fetchFlags, release)
|
|
fetchFlags = append(fetchFlags, "--untar", "--untardir", chartPath)
|
|
if err := helm.Fetch(chartName, fetchFlags...); err != nil {
|
|
lockResult.Release(st.logger)
|
|
return "", err
|
|
}
|
|
|
|
// Set chartPath to be the path containing Chart.yaml, if found
|
|
fullChartPath, err := findChartDirectory(chartPath)
|
|
if err == nil {
|
|
chartPath = filepath.Dir(fullChartPath)
|
|
}
|
|
|
|
// Add to global chart cache
|
|
st.addToChartCache(cacheKey, chartPath)
|
|
|
|
// Release lock immediately after download - the tempDir cleanup is deferred
|
|
// until after helm operations complete, so the chart won't be deleted mid-use.
|
|
lockResult.Release(st.logger)
|
|
|
|
return chartPath, nil
|
|
}
|
|
|
|
// prepareChartForRelease processes a single release and prepares its chart
|
|
func (st *HelmState) prepareChartForRelease(release *ReleaseSpec, helm helmexec.Interface, dir string, helmfileCommand string, opts ChartPrepareOptions, workerIndex int) *chartPrepareResult {
|
|
if st.OverrideChart != "" {
|
|
release.Chart = st.OverrideChart
|
|
}
|
|
|
|
// Call user-defined `prepare` hooks to create/modify local charts to be used by
|
|
// the later process.
|
|
//
|
|
// If it wasn't called here, Helmfile can end up an issue like
|
|
// https://github.com/roboll/helmfile/issues/1328
|
|
if _, err := st.triggerPrepareEvent(release, helmfileCommand); err != nil {
|
|
return &chartPrepareResult{err: err}
|
|
}
|
|
|
|
chartName := release.Chart
|
|
|
|
chartPath, err := st.downloadChartWithGoGetter(release)
|
|
if err != nil {
|
|
return &chartPrepareResult{err: fmt.Errorf("release %q: %w", release.Name, err)}
|
|
}
|
|
chartFetchedByGoGetter := chartPath != chartName
|
|
|
|
if !chartFetchedByGoGetter {
|
|
ociChartPath, err := st.getOCIChart(release, dir, helm, opts)
|
|
if err != nil {
|
|
return &chartPrepareResult{err: fmt.Errorf("release %q: %w", release.Name, err)}
|
|
}
|
|
|
|
if ociChartPath != nil {
|
|
chartPath = *ociChartPath
|
|
}
|
|
}
|
|
|
|
isLocal := st.fs.DirectoryExistsAt(normalizeChart(st.basePath, chartName))
|
|
|
|
chartification, clean, err := st.PrepareChartify(helm, release, chartPath, workerIndex)
|
|
|
|
if !opts.SkipCleanup {
|
|
// nolint: staticcheck
|
|
defer clean()
|
|
}
|
|
|
|
if err != nil {
|
|
return &chartPrepareResult{err: err}
|
|
}
|
|
|
|
var buildDeps bool
|
|
|
|
skipDepsGlobal := opts.SkipDeps
|
|
skipDepsRelease := release.SkipDeps != nil && *release.SkipDeps
|
|
skipDepsDefault := release.SkipDeps == nil && st.HelmDefaults.SkipDeps
|
|
skipDeps := (!isLocal && !chartFetchedByGoGetter) || skipDepsGlobal || skipDepsRelease || skipDepsDefault
|
|
|
|
skipRefreshGlobal := opts.SkipRefresh
|
|
skipRefreshRelease := release.SkipRefresh != nil && *release.SkipRefresh
|
|
skipRefreshDefault := release.SkipRefresh == nil && st.HelmDefaults.SkipRefresh
|
|
skipRefresh := !isLocal || skipRefreshGlobal || skipRefreshRelease || skipRefreshDefault
|
|
|
|
if chartification != nil && (helmfileCommand != "pull" || chartification.NeedsChartifyForLocalDir) {
|
|
// Issue #2297: Normalize local chart paths before chartification
|
|
// When using transformers with local charts like "../chart", the chartify process
|
|
// needs an absolute path, otherwise it tries "helm pull ../chart" which fails
|
|
if isLocal {
|
|
chartPath = normalizeChart(st.basePath, chartPath)
|
|
}
|
|
chartPath, buildDeps, err = st.processChartification(chartification, release, chartPath, opts, skipDeps, helmfileCommand)
|
|
if err != nil {
|
|
return &chartPrepareResult{err: err}
|
|
}
|
|
} else if normalizedChart := normalizeChart(st.basePath, chartPath); st.fs.DirectoryExistsAt(normalizedChart) {
|
|
chartPath, err = st.processLocalChart(normalizedChart, dir, release, helmfileCommand, opts, isLocal)
|
|
if err != nil {
|
|
return &chartPrepareResult{err: err}
|
|
}
|
|
buildDeps = !skipDeps
|
|
} else if !opts.ForceDownload {
|
|
// At this point, we are sure that either:
|
|
// 1. It is a local chart and we can use it in later process (helm upgrade/template/lint/etc)
|
|
// without any modification, or
|
|
// 2. It is a remote chart which can be safely handed over to helm,
|
|
// because the version of Helm used in this transaction (helm v3 or greater) support downloading
|
|
// the chart instead, AND we don't need any modification to the chart
|
|
//
|
|
// Also see HelmState.chartVersionFlags(). For `helmfile template`, it's called before `helm template`
|
|
// only on helm v3.
|
|
// For helm 2, we `helm fetch` with the version flags and call `helm template`
|
|
// WITHOUT the version flags.
|
|
} else {
|
|
chartPath, err = st.forcedDownloadChart(chartName, dir, release, helm, opts)
|
|
if err != nil {
|
|
return &chartPrepareResult{err: err}
|
|
}
|
|
}
|
|
|
|
return &chartPrepareResult{
|
|
releaseName: release.Name,
|
|
chartName: chartName,
|
|
releaseNamespace: release.Namespace,
|
|
releaseContext: release.KubeContext,
|
|
chartPath: chartPath,
|
|
buildDeps: buildDeps,
|
|
skipRefresh: skipRefresh,
|
|
chartFetchedByGoGetter: chartFetchedByGoGetter,
|
|
}
|
|
}
|
|
|
|
// PrepareCharts downloads and prepares all charts for the selected releases.
|
|
// Returns the chart paths and any errors encountered.
|
|
//
|
|
// Note: OCI chart locks are acquired and released during chart download within this function.
|
|
// The tempDir cleanup is deferred until after helm operations complete in the caller,
|
|
// so charts remain available during helm commands even though locks are released.
|
|
func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurrency int, helmfileCommand string, opts ChartPrepareOptions) (map[PrepareChartKey]string, []error) {
|
|
if !opts.SkipResolve {
|
|
updated, err := st.ResolveDeps()
|
|
if err != nil {
|
|
return nil, []error{err}
|
|
}
|
|
*st = *updated
|
|
}
|
|
selected, err := st.GetSelectedReleases(opts.IncludeTransitiveNeeds)
|
|
if err != nil {
|
|
return nil, []error{err}
|
|
}
|
|
|
|
releases := releasesNeedCharts(selected)
|
|
|
|
// For build command, skip releases that require chartify (jsonPatches, etc.)
|
|
// as we only need to output state, not actually template the charts
|
|
if helmfileCommand == "build" {
|
|
releases = filterReleasesForBuild(releases)
|
|
}
|
|
|
|
// Initialize the chartify temp dir tracker before concurrent workers start,
|
|
// so that all workers share a single tracker instance. See issue #1799.
|
|
st.chartifyTempDirs = &chartifyTempDirTracker{}
|
|
|
|
var prepareChartInfoMutex sync.Mutex
|
|
|
|
prepareChartInfo := make(map[PrepareChartKey]string, len(releases))
|
|
|
|
errs := []error{}
|
|
|
|
jobQueue := make(chan *ReleaseSpec, len(releases))
|
|
results := make(chan *chartPrepareResult, len(releases))
|
|
|
|
var builds []*chartPrepareResult
|
|
|
|
st.scatterGather(
|
|
concurrency,
|
|
len(releases),
|
|
func() {
|
|
for i := 0; i < len(releases); i++ {
|
|
jobQueue <- &releases[i]
|
|
}
|
|
close(jobQueue)
|
|
},
|
|
func(workerIndex int) {
|
|
for release := range jobQueue {
|
|
result := st.prepareChartForRelease(release, helm, dir, helmfileCommand, opts, workerIndex)
|
|
results <- result
|
|
}
|
|
},
|
|
func() {
|
|
for i := 0; i < len(releases); i++ {
|
|
downloadRes := <-results
|
|
|
|
if downloadRes.err != nil {
|
|
errs = append(errs, downloadRes.err)
|
|
return
|
|
}
|
|
func() {
|
|
prepareChartInfoMutex.Lock()
|
|
defer prepareChartInfoMutex.Unlock()
|
|
key := PrepareChartKey{
|
|
Namespace: downloadRes.releaseNamespace,
|
|
KubeContext: downloadRes.releaseContext,
|
|
Name: downloadRes.releaseName,
|
|
}
|
|
prepareChartInfo[key] = downloadRes.chartPath
|
|
}()
|
|
|
|
if downloadRes.buildDeps {
|
|
builds = append(builds, downloadRes)
|
|
}
|
|
}
|
|
},
|
|
)
|
|
|
|
if len(errs) > 0 {
|
|
return nil, errs
|
|
}
|
|
|
|
if len(builds) > 0 {
|
|
if err := st.runHelmDepBuilds(helm, concurrency, builds, opts); err != nil {
|
|
return nil, []error{err}
|
|
}
|
|
}
|
|
|
|
return prepareChartInfo, nil
|
|
}
|
|
|
|
// nolint: unparam
|
|
func (st *HelmState) runHelmDepBuilds(helm helmexec.Interface, concurrency int, builds []*chartPrepareResult, opts ChartPrepareOptions) error {
|
|
// NOTES:
|
|
// 1. `helm dep build` fails when it was run concurrency on the same chart.
|
|
// To avoid that, we run `helm dep build` only once per each local chart.
|
|
//
|
|
// See https://github.com/roboll/helmfile/issues/1438
|
|
// 2. Even if it isn't on the same local chart, `helm dep build` intermittently fails when run concurrentl
|
|
// So we shouldn't use goroutines like we do for other helm operations here.
|
|
//
|
|
// See https://github.com/roboll/helmfile/issues/1521
|
|
|
|
// Only run a global repo update when at least one build will actually use --skip-refresh.
|
|
// If all builds have skipRefresh=false, each `helm dep build` will refresh repos itself,
|
|
// so calling `helm.UpdateRepo()` here would be redundant.
|
|
anySkipRefresh := false
|
|
for _, b := range builds {
|
|
if b.skipRefresh {
|
|
anySkipRefresh = true
|
|
break
|
|
}
|
|
}
|
|
|
|
if anySkipRefresh && !opts.SkipRefresh && !st.HelmDefaults.SkipRefresh && st.NeedsRepoUpdate() {
|
|
if err := helm.UpdateRepo(); err != nil {
|
|
return fmt.Errorf("updating repo: %w", err)
|
|
}
|
|
}
|
|
|
|
for _, r := range builds {
|
|
// skipRefresh for each release is precomputed in prepareChartForRelease to:
|
|
// - avoid redundant repo updates for non-local charts when helm repo update has run
|
|
// - preserve refresh behavior for local charts that may depend on external repos
|
|
// not listed in helmfile.yaml (fixes issue #2431).
|
|
// We intentionally do not modify r.skipRefresh here.
|
|
|
|
buildDepsFlags := getBuildDepsFlags(r)
|
|
if err := helm.BuildDeps(r.releaseName, r.chartPath, buildDepsFlags...); err != nil {
|
|
if r.chartFetchedByGoGetter {
|
|
diagnostic := fmt.Sprintf(
|
|
"WARN: `helm dep build` failed. While processing release %q, Helmfile observed that remote chart %q fetched by go-getter is seemingly broken. "+
|
|
"One of well-known causes of this is that the chart has outdated Chart.lock, which needs the chart maintainer needs to run `helm dep up`. "+
|
|
"Helmfile is tolerating the error to avoid blocking you until the remote chart gets fixed. "+
|
|
"But this may result in any failure later if the chart is broken badly. FYI, the tolerated error was: %v",
|
|
r.releaseName,
|
|
r.chartName,
|
|
err.Error(),
|
|
)
|
|
|
|
st.logger.Warn(diagnostic)
|
|
|
|
continue
|
|
}
|
|
|
|
return fmt.Errorf("building dependencies of local chart: %w", err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type TemplateOpts struct {
|
|
Set []string
|
|
SkipCleanup bool
|
|
OutputDirTemplate string
|
|
IncludeCRDs bool
|
|
NoHooks bool
|
|
SkipTests bool
|
|
PostRenderer string
|
|
PostRendererArgs []string
|
|
KubeVersion string
|
|
ShowOnly []string
|
|
// Propagate '--skip-schema-validation' to helmv3 template and helm install
|
|
SkipSchemaValidation bool
|
|
}
|
|
|
|
type TemplateOpt interface{ Apply(*TemplateOpts) }
|
|
|
|
func (o *TemplateOpts) Apply(opts *TemplateOpts) {
|
|
*opts = *o
|
|
}
|
|
|
|
// TemplateReleases wrapper for executing helm template on the releases
|
|
func (st *HelmState) TemplateReleases(helm helmexec.Interface, outputDir string, additionalValues []string, args []string, workerLimit int,
|
|
validate bool, opt ...TemplateOpt) []error {
|
|
opts := &TemplateOpts{}
|
|
for _, o := range opt {
|
|
o.Apply(opts)
|
|
}
|
|
|
|
errs := []error{}
|
|
|
|
for i := range st.Releases {
|
|
release := &st.Releases[i]
|
|
|
|
if !release.Desired() {
|
|
continue
|
|
}
|
|
|
|
st.ApplyOverrides(release)
|
|
|
|
flags, files, err := st.flagsForTemplate(helm, release, 0, opts)
|
|
|
|
if !opts.SkipCleanup {
|
|
defer st.removeFiles(files)
|
|
}
|
|
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
|
|
for _, value := range additionalValues {
|
|
valfile, err := filepath.Abs(value)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
|
|
if _, err := os.Stat(valfile); os.IsNotExist(err) {
|
|
errs = append(errs, err)
|
|
}
|
|
flags = append(flags, "--values", valfile)
|
|
}
|
|
|
|
if opts.Set != nil {
|
|
for _, s := range opts.Set {
|
|
flags = append(flags, "--set", s)
|
|
}
|
|
}
|
|
|
|
if len(outputDir) > 0 || len(opts.OutputDirTemplate) > 0 {
|
|
releaseOutputDir, err := st.GenerateOutputDir(outputDir, release, opts.OutputDirTemplate)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
|
|
flags = append(flags, "--output-dir", releaseOutputDir)
|
|
st.logger.Debugf("Generating templates to : %s\n", releaseOutputDir)
|
|
err = os.MkdirAll(releaseOutputDir, 0755)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
|
|
if validate {
|
|
flags = append(flags, "--validate")
|
|
}
|
|
|
|
if opts.IncludeCRDs {
|
|
flags = append(flags, "--include-crds")
|
|
}
|
|
|
|
if opts.NoHooks {
|
|
flags = append(flags, "--no-hooks")
|
|
}
|
|
|
|
if opts.SkipTests {
|
|
flags = append(flags, "--skip-tests")
|
|
}
|
|
|
|
if len(errs) == 0 {
|
|
if err := helm.TemplateRelease(release.Name, release.ChartPathOrName(), flags...); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
|
|
if _, err := st.TriggerCleanupEvent(release, "template"); err != nil {
|
|
st.logger.Warnf("warn: %v\n", err)
|
|
}
|
|
}
|
|
|
|
if len(errs) != 0 {
|
|
return errs
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type WriteValuesOpts struct {
|
|
Set []string
|
|
OutputFileTemplate string
|
|
SkipCleanup bool
|
|
}
|
|
|
|
type WriteValuesOpt interface{ Apply(*WriteValuesOpts) }
|
|
|
|
func (o *WriteValuesOpts) Apply(opts *WriteValuesOpts) {
|
|
*opts = *o
|
|
}
|
|
|
|
// WriteReleasesValues writes values files for releases
|
|
func (st *HelmState) WriteReleasesValues(helm helmexec.Interface, additionalValues []string, opt ...WriteValuesOpt) []error {
|
|
opts := &WriteValuesOpts{}
|
|
for _, o := range opt {
|
|
o.Apply(opts)
|
|
}
|
|
|
|
for i := range st.Releases {
|
|
release := &st.Releases[i]
|
|
|
|
if !release.Desired() {
|
|
continue
|
|
}
|
|
|
|
st.ApplyOverrides(release)
|
|
|
|
generatedFiles, err := st.generateValuesFiles(helm, release, i)
|
|
if err != nil {
|
|
return []error{err}
|
|
}
|
|
|
|
if !opts.SkipCleanup {
|
|
defer st.removeFiles(generatedFiles)
|
|
}
|
|
|
|
for _, value := range additionalValues {
|
|
valfile, err := filepath.Abs(value)
|
|
if err != nil {
|
|
return []error{err}
|
|
}
|
|
|
|
if _, err := os.Stat(valfile); os.IsNotExist(err) {
|
|
return []error{err}
|
|
}
|
|
}
|
|
|
|
outputValuesFile, err := st.GenerateOutputFilePath(release, opts.OutputFileTemplate)
|
|
if err != nil {
|
|
return []error{err}
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Dir(outputValuesFile), 0755); err != nil {
|
|
return []error{err}
|
|
}
|
|
|
|
st.logger.Infof("Writing values file %s", outputValuesFile)
|
|
|
|
merged := map[string]any{}
|
|
|
|
for _, f := range append(generatedFiles, additionalValues...) {
|
|
src := map[string]any{}
|
|
|
|
srcBytes, err := st.fs.ReadFile(f)
|
|
if err != nil {
|
|
return []error{fmt.Errorf("reading %s: %w", f, err)}
|
|
}
|
|
|
|
if err := yaml.Unmarshal(srcBytes, &src); err != nil {
|
|
return []error{fmt.Errorf("unmarshalling yaml %s: %w", f, err)}
|
|
}
|
|
|
|
if err := mergo.Merge(&merged, &src, mergo.WithOverride); err != nil {
|
|
return []error{fmt.Errorf("merging %s: %w", f, err)}
|
|
}
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
|
|
encoder := yaml.NewEncoder(&buf)
|
|
if err := encoder.Encode(merged); err != nil {
|
|
return []error{err}
|
|
}
|
|
|
|
if err := os.WriteFile(outputValuesFile, buf.Bytes(), 0644); err != nil {
|
|
return []error{fmt.Errorf("writing values file %s: %w", outputValuesFile, err)}
|
|
}
|
|
|
|
if _, err := st.TriggerCleanupEvent(release, "write-values"); err != nil {
|
|
st.logger.Warnf("warn: %v\n", err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type LintOpts struct {
|
|
Set []string
|
|
SkipCleanup bool
|
|
}
|
|
|
|
type LintOpt interface{ Apply(*LintOpts) }
|
|
|
|
func (o *LintOpts) Apply(opts *LintOpts) {
|
|
*opts = *o
|
|
}
|
|
|
|
// LintReleases wrapper for executing helm lint on the releases
|
|
func (st *HelmState) LintReleases(helm helmexec.Interface, additionalValues []string, args []string, workerLimit int, opt ...LintOpt) []error {
|
|
opts := &LintOpts{}
|
|
for _, o := range opt {
|
|
o.Apply(opts)
|
|
}
|
|
|
|
// Reset the extra args if already set, not to break `helm fetch` by adding the args intended for `lint`
|
|
helm.SetExtraArgs()
|
|
|
|
errs := []error{}
|
|
|
|
if len(args) > 0 {
|
|
helm.SetExtraArgs(args...)
|
|
}
|
|
|
|
for i := range st.Releases {
|
|
release := st.Releases[i]
|
|
|
|
if !release.Desired() {
|
|
continue
|
|
}
|
|
|
|
flags, files, err := st.flagsForLint(helm, &release, 0)
|
|
|
|
if !opts.SkipCleanup {
|
|
defer st.removeFiles(files)
|
|
}
|
|
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
for _, value := range additionalValues {
|
|
valfile, err := filepath.Abs(value)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
|
|
if _, err := os.Stat(valfile); os.IsNotExist(err) {
|
|
errs = append(errs, err)
|
|
}
|
|
flags = append(flags, "--values", valfile)
|
|
}
|
|
|
|
if opts.Set != nil {
|
|
for _, s := range opts.Set {
|
|
flags = append(flags, "--set", s)
|
|
}
|
|
}
|
|
|
|
if len(errs) == 0 {
|
|
if err := helm.Lint(release.Name, release.ChartPathOrName(), flags...); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
|
|
if _, err := st.TriggerCleanupEvent(&release, "lint"); err != nil {
|
|
st.logger.Warnf("warn: %v\n", err)
|
|
}
|
|
}
|
|
|
|
if len(errs) != 0 {
|
|
return errs
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// UnittestOpts is the options for the unittest command
|
|
type UnittestOpts struct {
|
|
Set []string
|
|
SkipCleanup bool
|
|
FailFast bool
|
|
Color bool
|
|
DebugPlugin bool
|
|
}
|
|
|
|
// UnittestOpt is a functional option for UnittestOpts
|
|
type UnittestOpt interface {
|
|
Apply(*UnittestOpts)
|
|
}
|
|
|
|
// Apply implements UnittestOpt
|
|
func (o *UnittestOpts) Apply(opts *UnittestOpts) {
|
|
*opts = *o
|
|
}
|
|
|
|
// UnittestReleases runs helm unittest on each release that has unitTests defined.
|
|
// The workerLimit parameter is currently unused but kept for API consistency with
|
|
// similar methods (e.g., LintReleases).
|
|
func (st *HelmState) UnittestReleases(helm helmexec.Interface, additionalValues []string, args []string, _ int, opt ...UnittestOpt) []error {
|
|
opts := &UnittestOpts{}
|
|
for _, o := range opt {
|
|
o.Apply(opts)
|
|
}
|
|
|
|
helm.SetExtraArgs()
|
|
|
|
errs := []error{}
|
|
|
|
if len(args) > 0 {
|
|
helm.SetExtraArgs(args...)
|
|
}
|
|
|
|
for i := range st.Releases {
|
|
release := st.Releases[i]
|
|
|
|
if !release.Desired() {
|
|
continue
|
|
}
|
|
|
|
if len(release.UnitTests) == 0 {
|
|
continue
|
|
}
|
|
|
|
flags, files, err := st.flagsForLint(helm, &release, 0)
|
|
|
|
if !opts.SkipCleanup {
|
|
defer st.removeFiles(files)
|
|
}
|
|
|
|
releaseErr := false
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
releaseErr = true
|
|
}
|
|
for _, value := range additionalValues {
|
|
valfile, err := filepath.Abs(value)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
releaseErr = true
|
|
break
|
|
}
|
|
|
|
// Check for any stat error (not just IsNotExist) to also catch
|
|
// permission denied, I/O errors, etc. before passing to helm.
|
|
// This intentionally differs from LintReleases which only checks IsNotExist.
|
|
if _, err := os.Stat(valfile); err != nil {
|
|
errs = append(errs, err)
|
|
releaseErr = true
|
|
break
|
|
}
|
|
flags = append(flags, "--values", valfile)
|
|
}
|
|
|
|
if releaseErr {
|
|
continue
|
|
}
|
|
|
|
if opts.Set != nil {
|
|
for _, s := range opts.Set {
|
|
flags = append(flags, "--set", s)
|
|
}
|
|
}
|
|
|
|
if opts.FailFast {
|
|
flags = append(flags, "--failfast")
|
|
}
|
|
|
|
if opts.Color {
|
|
// In Helm 4, --color is parsed by Helm itself before reaching the plugin.
|
|
// See https://github.com/helmfile/helmfile/issues/2280 for details.
|
|
// Skip the flag with a warning since helm-unittest does not currently
|
|
// support an env var alternative for colored output.
|
|
if helm.IsHelm4() {
|
|
st.logger.Warnf("warn: --color flag is not supported with Helm 4 due to flag parsing issues, ignoring\n")
|
|
} else {
|
|
flags = append(flags, "--color")
|
|
}
|
|
}
|
|
|
|
if opts.DebugPlugin {
|
|
flags = append(flags, "--debugPlugin")
|
|
}
|
|
|
|
// Add unit test file/directory paths as glob patterns for --file flag.
|
|
// Paths are relative to the chart directory (matching helm-unittest conventions).
|
|
// If the path has no glob characters and does not look like a YAML file,
|
|
// treat it as a directory and append a glob suffix.
|
|
// Validate and add unit test file/directory paths.
|
|
// Reject absolute paths and paths that escape the chart directory via "..".
|
|
for _, testPath := range release.UnitTests {
|
|
cleanPath := filepath.Clean(testPath)
|
|
if filepath.IsAbs(cleanPath) || cleanPath == ".." || strings.HasPrefix(cleanPath, ".."+string(filepath.Separator)) {
|
|
errs = append(errs, fmt.Errorf("release %q: unitTests path %q must be a relative path within the chart directory", release.Name, testPath))
|
|
releaseErr = true
|
|
break
|
|
}
|
|
if !strings.ContainsAny(testPath, "*?[") {
|
|
lowerPath := strings.ToLower(testPath)
|
|
if !strings.HasSuffix(lowerPath, ".yaml") && !strings.HasSuffix(lowerPath, ".yml") {
|
|
testPath = strings.TrimRight(testPath, "/") + "/*_test.yaml"
|
|
}
|
|
}
|
|
flags = append(flags, "--file", testPath)
|
|
}
|
|
|
|
if len(errs) == 0 || !opts.FailFast {
|
|
if err := helm.Unittest(release.Name, release.ChartPathOrName(), flags...); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
|
|
if _, err := st.TriggerCleanupEvent(&release, "unittest"); err != nil {
|
|
st.logger.Warnf("warn: %v\n", err)
|
|
}
|
|
|
|
if opts.FailFast && len(errs) > 0 {
|
|
break
|
|
}
|
|
}
|
|
|
|
if len(errs) != 0 {
|
|
return errs
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type diffResult struct {
|
|
release *ReleaseSpec
|
|
err *ReleaseError
|
|
buf *bytes.Buffer
|
|
}
|
|
|
|
type diffPrepareResult struct {
|
|
release *ReleaseSpec
|
|
flags []string
|
|
errors []*ReleaseError
|
|
files []string
|
|
upgradeDueToSkippedDiff bool
|
|
suppressDiff bool
|
|
}
|
|
|
|
// commonDiffFlags returns common flags for helm diff, not in release-specific context
|
|
func (st *HelmState) commonDiffFlags(detailedExitCode bool, stripTrailingCR bool, includeTests bool, suppress []string, suppressSecrets bool, showSecrets bool, noHooks bool, opt *DiffOpts) []string {
|
|
var flags = []string{}
|
|
|
|
if detailedExitCode {
|
|
flags = append(flags, "--detailed-exitcode")
|
|
}
|
|
|
|
if stripTrailingCR {
|
|
flags = append(flags, "--strip-trailing-cr")
|
|
}
|
|
|
|
if includeTests {
|
|
flags = append(flags, "--include-tests")
|
|
}
|
|
|
|
for _, s := range suppress {
|
|
flags = append(flags, "--suppress", s)
|
|
}
|
|
|
|
if suppressSecrets {
|
|
flags = append(flags, "--suppress-secrets")
|
|
}
|
|
|
|
if showSecrets {
|
|
flags = append(flags, "--show-secrets")
|
|
}
|
|
|
|
if noHooks {
|
|
flags = append(flags, "--no-hooks")
|
|
}
|
|
|
|
if opt.NoColor {
|
|
flags = append(flags, "--no-color")
|
|
} else if opt.Color {
|
|
flags = append(flags, "--color")
|
|
}
|
|
|
|
if opt.Context > 0 {
|
|
flags = append(flags, "--context", fmt.Sprintf("%d", opt.Context))
|
|
}
|
|
|
|
if opt.Output != "" {
|
|
flags = append(flags, "--output", opt.Output)
|
|
}
|
|
|
|
if opt.Set != nil {
|
|
for _, s := range opt.Set {
|
|
flags = append(flags, "--set", s)
|
|
}
|
|
}
|
|
flags = st.appendExtraDiffFlags(flags, opt)
|
|
|
|
return flags
|
|
}
|
|
|
|
func (st *HelmState) prepareDiffReleases(helm helmexec.Interface, additionalValues []string, concurrency int, detailedExitCode bool, stripTrailingCR bool, includeTests bool, suppress []string, suppressSecrets bool, showSecrets bool, noHooks bool, opts ...DiffOpt) ([]diffPrepareResult, []error) {
|
|
opt := &DiffOpts{}
|
|
for _, o := range opts {
|
|
o.Apply(opt)
|
|
}
|
|
|
|
mu := &sync.RWMutex{}
|
|
installedReleases := map[string]bool{}
|
|
|
|
isInstalled := func(r *ReleaseSpec) (bool, error) {
|
|
id := ReleaseToID(r)
|
|
|
|
mu.RLock()
|
|
v, ok := installedReleases[id]
|
|
mu.RUnlock()
|
|
|
|
if ok {
|
|
return v, nil
|
|
}
|
|
|
|
v, err := st.isReleaseInstalled(st.createHelmContext(r, 0), helm, *r)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
mu.Lock()
|
|
installedReleases[id] = v
|
|
mu.Unlock()
|
|
|
|
return v, nil
|
|
}
|
|
|
|
releases := []*ReleaseSpec{}
|
|
for i := range st.Releases {
|
|
if !st.Releases[i].Desired() {
|
|
continue
|
|
}
|
|
if st.Releases[i].Installed != nil && !*(st.Releases[i].Installed) {
|
|
continue
|
|
}
|
|
releases = append(releases, &st.Releases[i])
|
|
}
|
|
|
|
numReleases := len(releases)
|
|
jobs := make(chan *ReleaseSpec, numReleases)
|
|
results := make(chan diffPrepareResult, numReleases)
|
|
resultsMap := map[string]diffPrepareResult{}
|
|
commonDiffFlags := st.commonDiffFlags(detailedExitCode, stripTrailingCR, includeTests, suppress, suppressSecrets, showSecrets, noHooks, opt)
|
|
|
|
rs := []diffPrepareResult{}
|
|
errs := []error{}
|
|
|
|
mut := sync.Mutex{}
|
|
|
|
st.scatterGather(
|
|
concurrency,
|
|
numReleases,
|
|
func() {
|
|
for i := 0; i < numReleases; i++ {
|
|
jobs <- releases[i]
|
|
}
|
|
close(jobs)
|
|
},
|
|
func(workerIndex int) {
|
|
for release := range jobs {
|
|
errs := []error{}
|
|
|
|
st.ApplyOverrides(release)
|
|
|
|
suppressDiff := false
|
|
if release.SuppressDiff != nil && *release.SuppressDiff {
|
|
suppressDiff = true
|
|
}
|
|
|
|
if opt.SkipDiffOnInstall {
|
|
installed, err := isInstalled(release)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
} else if !installed {
|
|
results <- diffPrepareResult{release: release, upgradeDueToSkippedDiff: true, suppressDiff: suppressDiff}
|
|
continue
|
|
}
|
|
}
|
|
|
|
var disableValidation bool
|
|
if release.DisableValidationOnInstall != nil && *release.DisableValidationOnInstall {
|
|
installed, err := isInstalled(release)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
} else {
|
|
disableValidation = !installed
|
|
}
|
|
}
|
|
|
|
// TODO We need a long-term fix for this :)
|
|
// See https://github.com/roboll/helmfile/issues/737
|
|
mut.Lock()
|
|
// release level diff flags in here
|
|
flags, files, err := st.flagsForDiff(helm, release, disableValidation, workerIndex, opt)
|
|
mut.Unlock()
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
|
|
for _, value := range additionalValues {
|
|
valfile, err := filepath.Abs(value)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
|
|
if _, err := os.Stat(valfile); os.IsNotExist(err) {
|
|
errs = append(errs, err)
|
|
}
|
|
flags = append(flags, "--values", valfile)
|
|
}
|
|
|
|
flags = st.appendValuesControlModeFlag(flags, opt.ReuseValues, opt.ResetValues, release)
|
|
|
|
flags = append(flags, commonDiffFlags...)
|
|
|
|
if len(errs) > 0 {
|
|
rsErrs := make([]*ReleaseError, len(errs))
|
|
for i, e := range errs {
|
|
rsErrs[i] = newReleaseFailedError(release, e)
|
|
}
|
|
results <- diffPrepareResult{errors: rsErrs, files: files, suppressDiff: suppressDiff}
|
|
} else {
|
|
results <- diffPrepareResult{release: release, flags: flags, errors: []*ReleaseError{}, files: files, suppressDiff: suppressDiff}
|
|
}
|
|
}
|
|
},
|
|
func() {
|
|
for i := 0; i < numReleases; i++ {
|
|
res := <-results
|
|
if len(res.errors) > 0 {
|
|
for _, e := range res.errors {
|
|
errs = append(errs, e)
|
|
}
|
|
} else if res.release != nil {
|
|
resultsMap[ReleaseToID(res.release)] = res
|
|
}
|
|
}
|
|
},
|
|
)
|
|
|
|
for _, r := range releases {
|
|
if p, ok := resultsMap[ReleaseToID(r)]; ok {
|
|
rs = append(rs, p)
|
|
}
|
|
}
|
|
|
|
return rs, errs
|
|
}
|
|
|
|
func (st *HelmState) createHelmContext(spec *ReleaseSpec, workerIndex int) helmexec.HelmContext {
|
|
historyMax := 10
|
|
if st.HelmDefaults.HistoryMax != nil {
|
|
historyMax = *st.HelmDefaults.HistoryMax
|
|
}
|
|
if spec.HistoryMax != nil {
|
|
historyMax = *spec.HistoryMax
|
|
}
|
|
|
|
return helmexec.HelmContext{
|
|
WorkerIndex: workerIndex,
|
|
HistoryMax: historyMax,
|
|
}
|
|
}
|
|
|
|
func (st *HelmState) createHelmContextWithWriter(spec *ReleaseSpec, w io.Writer) helmexec.HelmContext {
|
|
ctx := st.createHelmContext(spec, 0)
|
|
|
|
ctx.Writer = w
|
|
|
|
return ctx
|
|
}
|
|
|
|
type DiffOpts struct {
|
|
Context int
|
|
Output string
|
|
// Color forces the color output on helm-diff.
|
|
// This takes effect only when NoColor is false.
|
|
Color bool
|
|
// NoColor forces disabling the color output on helm-diff.
|
|
// If this is true, Color has no effect.
|
|
NoColor bool
|
|
Set []string
|
|
SkipCleanup bool
|
|
SkipDiffOnInstall bool
|
|
DiffArgs string
|
|
ReuseValues bool
|
|
ResetValues bool
|
|
PostRenderer string
|
|
PostRendererArgs []string
|
|
SuppressOutputLineRegex []string
|
|
SkipSchemaValidation bool
|
|
TakeOwnership bool
|
|
ServerSide string
|
|
// DetectedKubeVersion is the Kubernetes version detected from the cluster.
|
|
// This is used when kubeVersion is not specified in helmfile.yaml
|
|
DetectedKubeVersion string
|
|
}
|
|
|
|
func (o *DiffOpts) Apply(opts *DiffOpts) {
|
|
*opts = *o
|
|
}
|
|
|
|
type DiffOpt interface{ Apply(*DiffOpts) }
|
|
|
|
// DiffReleases wrapper for executing helm diff on the releases
|
|
// It returns releases that had any changes, and errors if any.
|
|
//
|
|
// This function has responsibility to stabilize the order of writes to stdout from multiple concurrent helm-diff runs.
|
|
// It's required to use the stdout from helmfile-diff to detect if there was another change(s) between 2 points in time.
|
|
// For example, terraform-provider-helmfile runs a helmfile-diff on `terraform plan` and another on `terraform apply`.
|
|
// `terraform`, by design, fails when helmfile-diff outputs were not equivalent.
|
|
// Stabilized helmfile-diff output rescues that.
|
|
func (st *HelmState) DiffReleases(helm helmexec.Interface, additionalValues []string, workerLimit int, detailedExitCode bool, stripTrailingCR bool, includeTests bool, suppress []string, suppressSecrets, showSecrets, noHooks bool, suppressDiff, triggerCleanupEvents bool, opt ...DiffOpt) ([]ReleaseSpec, []error) {
|
|
opts := &DiffOpts{}
|
|
for _, o := range opt {
|
|
o.Apply(opts)
|
|
}
|
|
|
|
preps, prepErrs := st.prepareDiffReleases(helm, additionalValues, workerLimit, detailedExitCode, stripTrailingCR, includeTests, suppress, suppressSecrets, showSecrets, noHooks, opts)
|
|
|
|
if !opts.SkipCleanup {
|
|
defer func() {
|
|
for _, p := range preps {
|
|
st.removeFiles(p.files)
|
|
}
|
|
}()
|
|
}
|
|
|
|
if len(prepErrs) > 0 {
|
|
return []ReleaseSpec{}, prepErrs
|
|
}
|
|
|
|
jobQueue := make(chan *diffPrepareResult, len(preps))
|
|
results := make(chan diffResult, len(preps))
|
|
|
|
rs := []ReleaseSpec{}
|
|
outputs := map[string]*bytes.Buffer{}
|
|
errs := []error{}
|
|
|
|
// The exit code returned by helm-diff when it detected any changes
|
|
HelmDiffExitCodeChanged := 2
|
|
|
|
st.scatterGather(
|
|
workerLimit,
|
|
len(preps),
|
|
func() {
|
|
for i := 0; i < len(preps); i++ {
|
|
jobQueue <- &preps[i]
|
|
}
|
|
close(jobQueue)
|
|
},
|
|
func(workerIndex int) {
|
|
for prep := range jobQueue {
|
|
flags := prep.flags
|
|
release := prep.release
|
|
buf := &bytes.Buffer{}
|
|
|
|
releaseSuppressDiff := suppressDiff
|
|
if prep.suppressDiff {
|
|
releaseSuppressDiff = true
|
|
}
|
|
|
|
// Use ChartPath directly if already set (normalized during chart preparation),
|
|
// otherwise normalize the original chart name.
|
|
chartPath := release.ChartPathOrName()
|
|
if release.ChartPath == "" {
|
|
chartPath = normalizeChart(st.basePath, chartPath)
|
|
}
|
|
|
|
if prep.upgradeDueToSkippedDiff {
|
|
results <- diffResult{release, &ReleaseError{ReleaseSpec: release, err: nil, Code: HelmDiffExitCodeChanged}, buf}
|
|
} else if err := st.withChartOperationLock(release, chartPath, func() error {
|
|
return helm.DiffRelease(st.createHelmContextWithWriter(release, buf), release.Name, chartPath, release.Namespace, releaseSuppressDiff, flags...)
|
|
}); err != nil {
|
|
switch e := err.(type) {
|
|
case helmexec.ExitError:
|
|
// Propagate any non-zero exit status from the external command like `helm` that is failed under the hood
|
|
results <- diffResult{release, &ReleaseError{release, err, e.ExitStatus()}, buf}
|
|
default:
|
|
results <- diffResult{release, &ReleaseError{release, err, 0}, buf}
|
|
}
|
|
} else {
|
|
// diff succeeded, found no changes
|
|
results <- diffResult{release, nil, buf}
|
|
}
|
|
|
|
if triggerCleanupEvents {
|
|
if _, err := st.TriggerCleanupEvent(prep.release, "diff"); err != nil {
|
|
st.logger.Warnf("warn: %v\n", err)
|
|
}
|
|
}
|
|
}
|
|
},
|
|
func() {
|
|
for i := 0; i < len(preps); i++ {
|
|
res := <-results
|
|
if res.err != nil {
|
|
errs = append(errs, res.err)
|
|
if res.err.Code == HelmDiffExitCodeChanged {
|
|
rs = append(rs, *res.err.ReleaseSpec)
|
|
}
|
|
}
|
|
|
|
outputs[ReleaseToID(res.release)] = res.buf
|
|
}
|
|
},
|
|
)
|
|
|
|
for _, p := range preps {
|
|
id := ReleaseToID(p.release)
|
|
if stdout, ok := outputs[id]; ok {
|
|
fmt.Print(stdout.String())
|
|
} else {
|
|
panic(fmt.Sprintf("missing output for release %s", id))
|
|
}
|
|
}
|
|
|
|
return rs, errs
|
|
}
|
|
|
|
func (st *HelmState) ReleaseStatuses(helm helmexec.Interface, workerLimit int) []error {
|
|
return st.scatterGatherReleases(helm, workerLimit, func(release ReleaseSpec, workerIndex int) error {
|
|
if !release.Desired() {
|
|
return nil
|
|
}
|
|
|
|
st.ApplyOverrides(&release)
|
|
|
|
flags := []string{}
|
|
if release.Namespace != "" {
|
|
flags = append(flags, "--namespace", release.Namespace)
|
|
}
|
|
flags = st.appendConnectionFlags(flags, &release)
|
|
|
|
return helm.ReleaseStatus(st.createHelmContext(&release, workerIndex), release.Name, flags...)
|
|
})
|
|
}
|
|
|
|
// DeleteReleases wrapper for executing helm delete on the releases
|
|
func (st *HelmState) DeleteReleases(affectedReleases *AffectedReleases, helm helmexec.Interface, concurrency int, purge bool, cascade string) []error {
|
|
return st.scatterGatherReleases(helm, concurrency, func(release ReleaseSpec, workerIndex int) error {
|
|
st.ApplyOverrides(&release)
|
|
|
|
flags := make([]string, 0)
|
|
flags = st.appendConnectionFlags(flags, &release)
|
|
flags = st.appendCascadeFlags(flags, helm, &release, cascade)
|
|
flags = st.appendDeleteWaitFlags(flags, &release)
|
|
if release.Namespace != "" {
|
|
flags = append(flags, "--namespace", release.Namespace)
|
|
}
|
|
context := st.createHelmContext(&release, workerIndex)
|
|
|
|
start := time.Now()
|
|
if _, err := st.triggerReleaseEvent("preuninstall", nil, &release, "delete"); err != nil {
|
|
release.duration = time.Since(start)
|
|
|
|
affectedReleases.DeleteFailed = append(affectedReleases.Failed, &release)
|
|
|
|
return err
|
|
}
|
|
|
|
if err := helm.DeleteRelease(context, release.Name, flags...); err != nil {
|
|
release.duration = time.Since(start)
|
|
|
|
affectedReleases.DeleteFailed = append(affectedReleases.Failed, &release)
|
|
return err
|
|
}
|
|
|
|
if _, err := st.triggerReleaseEvent("postuninstall", nil, &release, "delete"); err != nil {
|
|
release.duration = time.Since(start)
|
|
|
|
affectedReleases.DeleteFailed = append(affectedReleases.Failed, &release)
|
|
return err
|
|
}
|
|
release.duration = time.Since(start)
|
|
|
|
affectedReleases.Deleted = append(affectedReleases.Deleted, &release)
|
|
return nil
|
|
})
|
|
}
|
|
|
|
type TestOpts struct {
|
|
Logs bool
|
|
}
|
|
|
|
type TestOption func(*TestOpts)
|
|
|
|
func Logs(v bool) func(*TestOpts) {
|
|
return func(o *TestOpts) {
|
|
o.Logs = v
|
|
}
|
|
}
|
|
|
|
// TestReleases wrapper for executing helm test on the releases
|
|
func (st *HelmState) TestReleases(helm helmexec.Interface, cleanup bool, timeout int, concurrency int, options ...TestOption) []error {
|
|
var opts TestOpts
|
|
|
|
for _, o := range options {
|
|
o(&opts)
|
|
}
|
|
|
|
return st.scatterGatherReleases(helm, concurrency, func(release ReleaseSpec, workerIndex int) error {
|
|
if !release.Desired() {
|
|
return nil
|
|
}
|
|
|
|
flags := []string{}
|
|
if release.Namespace != "" {
|
|
flags = append(flags, "--namespace", release.Namespace)
|
|
}
|
|
if opts.Logs {
|
|
flags = append(flags, "--logs")
|
|
}
|
|
|
|
if timeout == EmptyTimeout {
|
|
flags = append(flags, st.timeoutFlags(&release, nil)...)
|
|
} else {
|
|
duration := strconv.Itoa(timeout)
|
|
duration += "s"
|
|
flags = append(flags, "--timeout", duration)
|
|
}
|
|
|
|
flags = st.appendConnectionFlags(flags, &release)
|
|
flags = st.appendChartDownloadFlags(flags, &release)
|
|
|
|
return helm.TestRelease(st.createHelmContext(&release, workerIndex), release.Name, flags...)
|
|
})
|
|
}
|
|
|
|
// Clean will remove any generated secrets
|
|
func (st *HelmState) Clean() []error {
|
|
return nil
|
|
}
|
|
|
|
func (st *HelmState) GetReleasesWithOverrides() ([]ReleaseSpec, error) {
|
|
var rs []ReleaseSpec
|
|
for _, r := range st.Releases {
|
|
spec := r
|
|
st.ApplyOverrides(&spec)
|
|
rs = append(rs, spec)
|
|
}
|
|
return rs, nil
|
|
}
|
|
|
|
func (st *HelmState) GetReleasesWithLabels() []ReleaseSpec {
|
|
var rs []ReleaseSpec
|
|
for _, r := range st.Releases {
|
|
spec := r
|
|
labels := map[string]string{}
|
|
// apply common labels
|
|
for k, v := range st.CommonLabels {
|
|
labels[k] = v
|
|
}
|
|
for k, v := range spec.Labels {
|
|
labels[k] = v
|
|
}
|
|
// Let the release name, namespace, and chart be used as a tag
|
|
labels["name"] = r.Name
|
|
labels["namespace"] = r.Namespace
|
|
// Strip off just the last portion for the name stable/newrelic would give newrelic
|
|
chartSplit := strings.Split(r.Chart, "/")
|
|
labels["chart"] = chartSplit[len(chartSplit)-1]
|
|
spec.Labels = labels
|
|
rs = append(rs, spec)
|
|
}
|
|
return rs
|
|
}
|
|
|
|
func (st *HelmState) SelectReleases(includeTransitiveNeeds bool) ([]Release, error) {
|
|
values := st.Values()
|
|
rs, err := markExcludedReleases(st.Releases, st.Selectors, values, includeTransitiveNeeds)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return rs, nil
|
|
}
|
|
|
|
func markExcludedReleases(releases []ReleaseSpec, selectors []string, values map[string]any, includeTransitiveNeeds bool) ([]Release, error) {
|
|
var filteredReleases []Release
|
|
filters := []ReleaseFilter{}
|
|
for _, label := range selectors {
|
|
f, err := ParseLabels(label)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
filters = append(filters, f)
|
|
}
|
|
for _, r := range releases {
|
|
var filterMatch bool
|
|
for _, f := range filters {
|
|
if f.Match(r) {
|
|
filterMatch = true
|
|
break
|
|
}
|
|
}
|
|
var conditionMatch bool
|
|
conditionMatch, err := ConditionEnabled(r, values)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse condition in release %s: %w", r.Name, err)
|
|
}
|
|
// reset the labels to the original
|
|
res := Release{
|
|
ReleaseSpec: r,
|
|
Filtered: (len(filters) > 0 && !filterMatch) || (!conditionMatch),
|
|
}
|
|
filteredReleases = append(filteredReleases, res)
|
|
}
|
|
if includeTransitiveNeeds {
|
|
unmarkNeedsAndTransitives(filteredReleases, releases)
|
|
}
|
|
return filteredReleases, nil
|
|
}
|
|
|
|
// ConditionEnabled checks if a release condition is enabled based on the provided values.
|
|
// It takes a ReleaseSpec and a map of values as input.
|
|
// If the condition is not specified, it returns true.
|
|
// If the condition is specified but not in the form 'foo.enabled', it returns an error.
|
|
// If the condition is specified and the corresponding value is found in the values map,
|
|
// it checks if the 'enabled' field is set to true. If so, it returns true.
|
|
// Otherwise, it returns false.
|
|
// If the condition is specified but the corresponding value is not found in the values map,
|
|
// it returns an error.
|
|
func ConditionEnabled(r ReleaseSpec, values map[string]any) (bool, error) {
|
|
if len(r.Condition) == 0 {
|
|
return true, nil
|
|
}
|
|
iValues := values
|
|
keys := strings.Split(r.Condition, ".")
|
|
if keys[len(keys)-1] != "enabled" {
|
|
return false, fmt.Errorf("Condition value must be in the form 'foo.enabled' where 'foo' can be modified as necessary")
|
|
}
|
|
|
|
currentKey := ""
|
|
for _, key := range keys[:len(keys)-1] {
|
|
currentKey = fmt.Sprintf("%s.%s", currentKey, key)
|
|
value, ok := iValues[key]
|
|
if !ok {
|
|
return false, fmt.Errorf("environment values field '%s' not found", currentKey)
|
|
}
|
|
|
|
iValues, ok = value.(map[string]interface{})
|
|
if !ok {
|
|
return false, fmt.Errorf("environment values field '%s' is not a map", currentKey)
|
|
}
|
|
}
|
|
|
|
enabled, ok := iValues["enabled"]
|
|
|
|
if !ok {
|
|
return false, nil
|
|
}
|
|
|
|
e, ok := enabled.(bool)
|
|
|
|
if !ok {
|
|
return false, nil
|
|
}
|
|
|
|
return e, nil
|
|
}
|
|
|
|
func unmarkNeedsAndTransitives(filteredReleases []Release, allReleases []ReleaseSpec) {
|
|
needsWithTranstives := collectAllNeedsWithTransitives(filteredReleases, allReleases)
|
|
unmarkReleases(needsWithTranstives, filteredReleases)
|
|
}
|
|
|
|
func collectAllNeedsWithTransitives(filteredReleases []Release, allReleases []ReleaseSpec) map[string]struct{} {
|
|
needsWithTranstives := map[string]struct{}{}
|
|
for _, r := range filteredReleases {
|
|
if !r.Filtered {
|
|
collectNeedsWithTransitives(r.ReleaseSpec, allReleases, needsWithTranstives)
|
|
}
|
|
}
|
|
return needsWithTranstives
|
|
}
|
|
|
|
func unmarkReleases(toUnmark map[string]struct{}, releases []Release) {
|
|
for i, r := range releases {
|
|
if _, ok := toUnmark[ReleaseToID(&r.ReleaseSpec)]; ok {
|
|
releases[i].Filtered = false
|
|
}
|
|
}
|
|
}
|
|
|
|
func collectNeedsWithTransitives(release ReleaseSpec, allReleases []ReleaseSpec, needsWithTranstives map[string]struct{}) {
|
|
for _, id := range release.Needs {
|
|
if _, exists := needsWithTranstives[id]; !exists {
|
|
needsWithTranstives[id] = struct{}{}
|
|
releaseParts := strings.Split(id, "/")
|
|
releaseName := releaseParts[len(releaseParts)-1]
|
|
for _, r := range allReleases {
|
|
if r.Name == releaseName {
|
|
collectNeedsWithTransitives(r, allReleases, needsWithTranstives)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (st *HelmState) GetSelectedReleases(includeTransitiveNeeds bool) ([]ReleaseSpec, error) {
|
|
filteredReleases, err := st.SelectReleases(includeTransitiveNeeds)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var releases []ReleaseSpec
|
|
for _, r := range filteredReleases {
|
|
if !r.Filtered {
|
|
releases = append(releases, r.ReleaseSpec)
|
|
}
|
|
}
|
|
|
|
return releases, nil
|
|
}
|
|
|
|
// FilterReleases allows for the execution of helm commands against a subset of the releases in the helmfile.
|
|
func (st *HelmState) FilterReleases(includeTransitiveNeeds bool) error {
|
|
releases, err := st.GetSelectedReleases(includeTransitiveNeeds)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
st.Releases = releases
|
|
return nil
|
|
}
|
|
|
|
func (st *HelmState) TriggerGlobalPrepareEvent(helmfileCommand string) (bool, error) {
|
|
return st.triggerGlobalReleaseEvent("prepare", nil, helmfileCommand)
|
|
}
|
|
|
|
func (st *HelmState) TriggerGlobalCleanupEvent(helmfileCommand string, evtErr error) (bool, error) {
|
|
return st.triggerGlobalReleaseEvent("cleanup", evtErr, helmfileCommand)
|
|
}
|
|
|
|
func (st *HelmState) triggerGlobalReleaseEvent(evt string, evtErr error, helmfileCmd string) (bool, error) {
|
|
bus := &event.Bus{
|
|
Hooks: st.Hooks,
|
|
StateFilePath: st.FilePath,
|
|
BasePath: st.basePath,
|
|
Namespace: st.OverrideNamespace,
|
|
Chart: st.OverrideChart,
|
|
Env: st.Env,
|
|
Logger: st.logger,
|
|
Fs: st.fs,
|
|
}
|
|
data := map[string]any{
|
|
"HelmfileCommand": helmfileCmd,
|
|
}
|
|
return bus.Trigger(evt, evtErr, data)
|
|
}
|
|
|
|
func (st *HelmState) triggerPrepareEvent(r *ReleaseSpec, helmfileCommand string) (bool, error) {
|
|
return st.triggerReleaseEvent("prepare", nil, r, helmfileCommand)
|
|
}
|
|
|
|
func (st *HelmState) TriggerCleanupEvent(r *ReleaseSpec, helmfileCommand string) (bool, error) {
|
|
return st.triggerReleaseEvent("cleanup", nil, r, helmfileCommand)
|
|
}
|
|
|
|
func (st *HelmState) triggerPresyncEvent(r *ReleaseSpec, helmfileCommand string) (bool, error) {
|
|
return st.triggerReleaseEvent("presync", nil, r, helmfileCommand)
|
|
}
|
|
|
|
func (st *HelmState) triggerPostsyncEvent(r *ReleaseSpec, evtErr error, helmfileCommand string) (bool, error) {
|
|
return st.triggerReleaseEvent("postsync", evtErr, r, helmfileCommand)
|
|
}
|
|
|
|
func (st *HelmState) TriggerPreapplyEvent(r *ReleaseSpec, helmfileCommand string) (bool, error) {
|
|
return st.triggerReleaseEvent("preapply", nil, r, helmfileCommand)
|
|
}
|
|
|
|
func (st *HelmState) triggerReleaseEvent(evt string, evtErr error, r *ReleaseSpec, helmfileCmd string) (bool, error) {
|
|
bus := &event.Bus{
|
|
Hooks: r.Hooks,
|
|
StateFilePath: st.FilePath,
|
|
BasePath: st.basePath,
|
|
Namespace: st.OverrideNamespace,
|
|
Chart: st.OverrideChart,
|
|
Env: st.Env,
|
|
Logger: st.logger,
|
|
Fs: st.fs,
|
|
}
|
|
vals := st.Values()
|
|
data := map[string]any{
|
|
"Values": vals,
|
|
"Release": r,
|
|
"HelmfileCommand": helmfileCmd,
|
|
}
|
|
|
|
return bus.Trigger(evt, evtErr, data)
|
|
}
|
|
|
|
// ResolveDeps returns a copy of this helmfile state with the concrete chart version numbers filled in for remote chart dependencies
|
|
func (st *HelmState) ResolveDeps() (*HelmState, error) {
|
|
return st.mergeLockedDependencies()
|
|
}
|
|
|
|
// UpdateDeps wrapper for updating dependencies on the releases
|
|
func (st *HelmState) UpdateDeps(helm helmexec.Interface, includeTransitiveNeeds bool) []error {
|
|
var selected []ReleaseSpec
|
|
|
|
if len(st.Selectors) > 0 {
|
|
var err error
|
|
|
|
// This and releasesNeedCharts ensures that we run operations like helm-dep-build and prepare-hook calls only on
|
|
// releases that are (1) selected by the selectors and (2) to be installed.
|
|
selected, err = st.GetSelectedReleases(includeTransitiveNeeds)
|
|
if err != nil {
|
|
return []error{err}
|
|
}
|
|
} else {
|
|
selected = st.Releases
|
|
}
|
|
|
|
releases := releasesNeedCharts(selected)
|
|
|
|
var errs []error
|
|
|
|
for _, release := range releases {
|
|
if !st.fs.DirectoryExistsAt(release.ChartPathOrName()) {
|
|
st.logger.Debugf("skipped updating dependencies for remote chart %s", release.Chart)
|
|
} else if !st.fs.FileExistsAt(filepath.Join(release.ChartPathOrName(), "Chart.yaml")) {
|
|
st.logger.Debugf("skipped updating dependencies for %s as it does not have a Chart.yaml", release.Chart)
|
|
} else {
|
|
if err := helm.UpdateDeps(release.ChartPathOrName()); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(errs) == 0 {
|
|
tempDir := st.tempDir
|
|
if tempDir == nil {
|
|
tempDir = os.MkdirTemp
|
|
}
|
|
_, err := st.updateDependenciesInTempDir(helm, tempDir)
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("unable to update deps: %v", err))
|
|
}
|
|
}
|
|
|
|
if len(errs) != 0 {
|
|
return errs
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// DependencyError holds information about a release and its dependency that is not installed.
|
|
type DependencyError struct {
|
|
Release ReleaseSpec
|
|
Dependency ReleaseSpec
|
|
}
|
|
|
|
// hasTransitiveDependencyWithInstallFalse checks if a release has a transitive dependency with Install set to false.
|
|
func (st *HelmState) HasTransitiveDependencyWithInstalledFalse(release ReleaseSpec, visited map[string]bool) *DependencyError {
|
|
if visited[release.Name] {
|
|
return nil
|
|
}
|
|
visited[release.Name] = true
|
|
|
|
for _, dep := range release.Needs {
|
|
for _, r := range st.Releases {
|
|
if r.Name == dep {
|
|
if r.Installed != nil && !*r.Installed {
|
|
return &DependencyError{Release: release, Dependency: r}
|
|
}
|
|
if depErr := st.HasTransitiveDependencyWithInstalledFalse(r, visited); depErr != nil {
|
|
return depErr
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// find "Chart.yaml"
|
|
func findChartDirectory(topLevelDir string) (string, error) {
|
|
var files []string
|
|
err := filepath.Walk(topLevelDir, func(path string, f os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return fmt.Errorf("error walking through %s: %v", path, err)
|
|
}
|
|
if !f.IsDir() {
|
|
r, err := regexp.MatchString("Chart.yaml", f.Name())
|
|
if err == nil && r {
|
|
files = append(files, path)
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return topLevelDir, err
|
|
}
|
|
// Sort to get the shortest path
|
|
sort.Strings(files)
|
|
if len(files) > 0 {
|
|
first := files[0]
|
|
return first, nil
|
|
}
|
|
|
|
return topLevelDir, errors.New("no Chart.yaml found")
|
|
}
|
|
|
|
// appendConnectionFlags append all the helm command-line flags related to K8s API including the kubecontext
|
|
func (st *HelmState) appendConnectionFlags(flags []string, release *ReleaseSpec) []string {
|
|
kubeFlagAdds := st.kubeConnectionFlags(release)
|
|
flags = append(flags, kubeFlagAdds...)
|
|
return flags
|
|
}
|
|
|
|
// appendExtraDiffFlags appends extra diff flags to the given flags slice based on the provided options.
|
|
// If opt is not nil and opt.DiffArgs is not empty, it collects the arguments from opt.DiffArgs and appends them to flags.
|
|
// If st.HelmDefaults.DiffArgs is not nil, it joins the arguments with a space and appends them to flags.
|
|
// The updated flags slice is returned.
|
|
func (st *HelmState) appendExtraDiffFlags(flags []string, opt *DiffOpts) []string {
|
|
switch {
|
|
case opt != nil && opt.DiffArgs != "":
|
|
flags = append(flags, argparser.CollectArgs(opt.DiffArgs)...)
|
|
case st.HelmDefaults.DiffArgs != nil:
|
|
flags = append(flags, argparser.CollectArgs(strings.Join(st.HelmDefaults.DiffArgs, " "))...)
|
|
}
|
|
return flags
|
|
}
|
|
|
|
// appendExtraSyncFlags appends extra diff flags to the given flags slice based on the provided options.
|
|
// If opt is not nil and opt.SyncArgs is not empty, it collects the arguments from opt.SyncArgs and appends them to flags.
|
|
// If st.HelmDefaults.SyncArgs is not nil, it joins the arguments with a space and appends them to flags.
|
|
// The updated flags slice is returned.
|
|
func (st *HelmState) appendExtraSyncFlags(flags []string, opt *SyncOpts) []string {
|
|
switch {
|
|
case opt != nil && opt.SyncArgs != "":
|
|
flags = append(flags, argparser.CollectArgs(opt.SyncArgs)...)
|
|
case st.HelmDefaults.SyncArgs != nil:
|
|
flags = append(flags, argparser.CollectArgs(strings.Join(st.HelmDefaults.SyncArgs, " "))...)
|
|
}
|
|
return flags
|
|
}
|
|
|
|
// appendVerifyFlags append the --verify flags related to verify
|
|
func (st *HelmState) appendVerifyFlags(flags []string, release *ReleaseSpec) []string {
|
|
repo, _ := st.GetRepositoryAndNameFromChartName(release.Chart)
|
|
switch {
|
|
case release.Verify != nil:
|
|
// If the release has a verify flag, use it
|
|
v := *release.Verify
|
|
if v {
|
|
flags = append(flags, "--verify")
|
|
}
|
|
return flags
|
|
case repo != nil && repo.Verify:
|
|
flags = append(flags, "--verify")
|
|
case st.HelmDefaults.Verify:
|
|
flags = append(flags, "--verify")
|
|
}
|
|
return flags
|
|
}
|
|
|
|
// appendKeyringFlags append all the helm command-line flags related to keyring
|
|
func (st *HelmState) appendKeyringFlags(flags []string, release *ReleaseSpec) []string {
|
|
repo, _ := st.GetRepositoryAndNameFromChartName(release.Chart)
|
|
switch {
|
|
case release.Keyring != "":
|
|
flags = append(flags, "--keyring", release.Keyring)
|
|
case repo != nil && repo.Keyring != "":
|
|
flags = append(flags, "--keyring", repo.Keyring)
|
|
case st.HelmDefaults.Keyring != "":
|
|
flags = append(flags, "--keyring", st.HelmDefaults.Keyring)
|
|
}
|
|
|
|
return flags
|
|
}
|
|
|
|
// appendEnableDNSFlags append the helm command-line flag for DNS resolution
|
|
func (st *HelmState) appendEnableDNSFlags(flags []string, release *ReleaseSpec) []string {
|
|
if release.EnableDNS != nil && *release.EnableDNS || release.EnableDNS == nil && st.HelmDefaults.EnableDNS {
|
|
flags = append(flags, "--enable-dns")
|
|
}
|
|
return flags
|
|
}
|
|
|
|
// getKubeContext returns the effective kube-context for a release.
|
|
// It follows the priority: release.KubeContext > environment.KubeContext > helmDefaults.KubeContext
|
|
// Issue #2309: This is the single source of truth for kube-context resolution
|
|
func (st *HelmState) getKubeContext(release *ReleaseSpec) string {
|
|
if release.KubeContext != "" {
|
|
return release.KubeContext
|
|
}
|
|
if st.Environments[st.Env.Name].KubeContext != "" {
|
|
return st.Environments[st.Env.Name].KubeContext
|
|
}
|
|
if st.HelmDefaults.KubeContext != "" {
|
|
return st.HelmDefaults.KubeContext
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (st *HelmState) kubeConnectionFlags(release *ReleaseSpec) []string {
|
|
flags := []string{}
|
|
if kubeContext := st.getKubeContext(release); kubeContext != "" {
|
|
flags = append(flags, "--kube-context", kubeContext)
|
|
}
|
|
return flags
|
|
}
|
|
|
|
func (st *HelmState) appendChartDownloadFlags(flags []string, release *ReleaseSpec) []string {
|
|
repo, _ := st.GetRepositoryAndNameFromChartName(release.Chart)
|
|
if st.needsPlainHttp(release, repo) {
|
|
flags = append(flags, "--plain-http")
|
|
// --insecure-skip-tls-verify nullifies --plain-http in helm, omit it if PlainHttp is specified
|
|
return flags
|
|
}
|
|
|
|
if st.needsInsecureSkipTLSVerify(release, repo) {
|
|
flags = append(flags, "--insecure-skip-tls-verify")
|
|
}
|
|
|
|
return flags
|
|
}
|
|
|
|
// appendDescriptionFlags appends the helm command-line flag for release description
|
|
// Command line takes precedence over config file
|
|
func (st *HelmState) appendDescriptionFlags(flags []string, release *ReleaseSpec, opt *SyncOpts, helm helmexec.Interface) ([]string, error) {
|
|
description := release.Description
|
|
if opt != nil && opt.Description != "" {
|
|
description = opt.Description
|
|
}
|
|
|
|
if description != "" {
|
|
if !helm.IsVersionAtLeast("3.3.0") {
|
|
// Determine error message based on source
|
|
if opt != nil && opt.Description != "" {
|
|
return nil, fmt.Errorf("--description flag requires Helm 3.3.0 or greater")
|
|
}
|
|
return nil, fmt.Errorf("releases[].description requires Helm 3.3.0 or greater")
|
|
}
|
|
flags = append(flags, "--description", description)
|
|
}
|
|
|
|
return flags, nil
|
|
}
|
|
|
|
func (st *HelmState) needsPlainHttp(release *ReleaseSpec, repo *RepositorySpec) bool {
|
|
var repoPlainHttp, relPlainHttp bool
|
|
if repo != nil {
|
|
repoPlainHttp = repo.PlainHttp
|
|
}
|
|
|
|
if release != nil {
|
|
relPlainHttp = release.PlainHttp
|
|
}
|
|
|
|
return relPlainHttp || st.HelmDefaults.PlainHttp || repoPlainHttp
|
|
}
|
|
|
|
func (st *HelmState) needsInsecureSkipTLSVerify(release *ReleaseSpec, repo *RepositorySpec) bool {
|
|
var repoSkipTLSVerify, relSkipTLSVerify bool
|
|
if repo != nil {
|
|
repoSkipTLSVerify = repo.SkipTLSVerify
|
|
}
|
|
|
|
if release != nil {
|
|
relSkipTLSVerify = release.InsecureSkipTLSVerify
|
|
}
|
|
|
|
return relSkipTLSVerify || st.HelmDefaults.InsecureSkipTLSVerify || repoSkipTLSVerify
|
|
}
|
|
|
|
func (st *HelmState) timeoutFlags(release *ReleaseSpec, ops *SyncOpts) []string {
|
|
var flags []string
|
|
|
|
timeout := st.HelmDefaults.Timeout
|
|
if release.Timeout != nil {
|
|
timeout = *release.Timeout
|
|
}
|
|
if ops != nil && ops.Timeout > 0 {
|
|
timeout = ops.Timeout
|
|
}
|
|
if timeout != 0 {
|
|
duration := strconv.Itoa(timeout)
|
|
duration += "s"
|
|
flags = append(flags, "--timeout", duration)
|
|
}
|
|
|
|
return flags
|
|
}
|
|
|
|
func (st *HelmState) flagsForUpgrade(helm helmexec.Interface, release *ReleaseSpec, workerIndex int, opt *SyncOpts) ([]string, []string, error) {
|
|
var flags []string
|
|
flags = st.appendChartVersionFlags(flags, release)
|
|
flags = st.appendEnableDNSFlags(flags, release)
|
|
|
|
flags = st.appendWaitFlags(flags, helm, release, opt)
|
|
flags = st.appendWaitForJobsFlags(flags, release, opt)
|
|
|
|
// non-OCI chart should be verified here
|
|
if !st.IsOCIChart(release.Chart) {
|
|
flags = st.appendVerifyFlags(flags, release)
|
|
flags = st.appendKeyringFlags(flags, release)
|
|
}
|
|
|
|
flags = append(flags, st.timeoutFlags(release, opt)...)
|
|
|
|
forceEnabled := (release.Force != nil && *release.Force) || (release.Force == nil && st.HelmDefaults.Force)
|
|
forceConflictsEnabled := (release.ForceConflicts != nil && *release.ForceConflicts) || (release.ForceConflicts == nil && st.HelmDefaults.ForceConflicts)
|
|
|
|
if forceConflictsEnabled && !helm.IsHelm4() {
|
|
return nil, nil, fmt.Errorf("forceConflicts requires Helm 4 or greater (set via releases[].forceConflicts or helmDefaults.forceConflicts)")
|
|
}
|
|
|
|
if forceEnabled && forceConflictsEnabled {
|
|
return nil, nil, fmt.Errorf("force and forceConflicts are mutually exclusive (check both releases[].force/forceConflicts and helmDefaults.force/forceConflicts)")
|
|
}
|
|
|
|
if forceEnabled {
|
|
if helm.IsHelm4() {
|
|
flags = append(flags, "--force-replace")
|
|
} else {
|
|
flags = append(flags, "--force")
|
|
}
|
|
}
|
|
|
|
if forceConflictsEnabled {
|
|
flags = append(flags, "--force-conflicts")
|
|
}
|
|
|
|
if release.RecreatePods != nil && *release.RecreatePods || release.RecreatePods == nil && st.HelmDefaults.RecreatePods {
|
|
flags = append(flags, "--recreate-pods")
|
|
}
|
|
|
|
if release.Atomic != nil && *release.Atomic || release.Atomic == nil && st.HelmDefaults.Atomic {
|
|
flags = append(flags, "--atomic")
|
|
}
|
|
|
|
if release.CleanupOnFail != nil && *release.CleanupOnFail || release.CleanupOnFail == nil && st.HelmDefaults.CleanupOnFail {
|
|
flags = append(flags, "--cleanup-on-fail")
|
|
}
|
|
|
|
if release.CreateNamespace != nil && *release.CreateNamespace ||
|
|
release.CreateNamespace == nil && (st.HelmDefaults.CreateNamespace == nil || *st.HelmDefaults.CreateNamespace) {
|
|
if helm.IsVersionAtLeast("3.2.0") {
|
|
flags = append(flags, "--create-namespace")
|
|
} else if release.CreateNamespace != nil || st.HelmDefaults.CreateNamespace != nil {
|
|
// createNamespace was set explicitly, but not running supported version of helm - error
|
|
return nil, nil, fmt.Errorf("releases[].createNamespace requires Helm 3.2.0 or greater")
|
|
}
|
|
}
|
|
|
|
if release.DisableOpenAPIValidation != nil && *release.DisableOpenAPIValidation ||
|
|
release.DisableOpenAPIValidation == nil && st.HelmDefaults.DisableOpenAPIValidation != nil && *st.HelmDefaults.DisableOpenAPIValidation {
|
|
flags = append(flags, "--disable-openapi-validation")
|
|
}
|
|
|
|
postRenderer := ""
|
|
syncReleaseLabels := false
|
|
takeOwnership := false
|
|
serverSide := ""
|
|
if opt != nil {
|
|
postRenderer = opt.PostRenderer
|
|
syncReleaseLabels = opt.SyncReleaseLabels
|
|
takeOwnership = opt.TakeOwnership
|
|
serverSide = opt.ServerSide
|
|
}
|
|
|
|
flags = st.appendConnectionFlags(flags, release)
|
|
flags = st.appendChartDownloadFlags(flags, release)
|
|
|
|
flags = st.appendHelmXFlags(flags, release)
|
|
|
|
flags = st.appendLabelsFlags(flags, helm, release, syncReleaseLabels)
|
|
|
|
flags = st.appendPostRenderFlags(flags, release, postRenderer, helm)
|
|
|
|
flags, err := st.appendDescriptionFlags(flags, release, opt, helm)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
var postRendererArgs []string
|
|
if opt != nil {
|
|
postRendererArgs = opt.PostRendererArgs
|
|
}
|
|
flags, err = st.appendPostRenderArgsFlags(flags, release, postRendererArgs)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
skipSchemaValidation := false
|
|
if opt != nil {
|
|
skipSchemaValidation = opt.SkipSchemaValidation
|
|
}
|
|
|
|
flags = st.appendSkipSchemaValidationFlags(flags, release, skipSchemaValidation)
|
|
|
|
// append hide-notes flag
|
|
flags = st.appendHideNotesFlags(flags, helm, opt)
|
|
|
|
// append take-ownership flag
|
|
flags = st.appendTakeOwnershipFlagsForUpgrade(flags, helm, release, takeOwnership)
|
|
|
|
// append server-side flag
|
|
var ssErr error
|
|
flags, ssErr = st.appendServerSideFlagsForUpgrade(flags, helm, release, serverSide)
|
|
if ssErr != nil {
|
|
return nil, nil, ssErr
|
|
}
|
|
|
|
flags = st.appendExtraSyncFlags(flags, opt)
|
|
|
|
common, clean, err := st.namespaceAndValuesFlags(helm, release, workerIndex)
|
|
if err != nil {
|
|
return nil, clean, err
|
|
}
|
|
return append(flags, common...), clean, nil
|
|
}
|
|
|
|
func (st *HelmState) flagsForTemplate(helm helmexec.Interface, release *ReleaseSpec, workerIndex int, opt *TemplateOpts) ([]string, []string, error) {
|
|
var flags []string
|
|
flags = st.appendChartVersionFlags(flags, release)
|
|
flags = st.appendHelmXFlags(flags, release)
|
|
flags = st.appendEnableDNSFlags(flags, release)
|
|
|
|
var postRendererArgs []string
|
|
var showOnly []string
|
|
postRenderer := ""
|
|
kubeVersion := ""
|
|
skipSchemaValidation := false
|
|
if opt != nil {
|
|
postRenderer = opt.PostRenderer
|
|
postRendererArgs = opt.PostRendererArgs
|
|
kubeVersion = opt.KubeVersion
|
|
showOnly = opt.ShowOnly
|
|
skipSchemaValidation = opt.SkipSchemaValidation
|
|
}
|
|
flags = st.appendPostRenderFlags(flags, release, postRenderer, helm)
|
|
flags, err := st.appendPostRenderArgsFlags(flags, release, postRendererArgs)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
flags = st.appendApiVersionsFlags(flags, release, kubeVersion)
|
|
flags = st.appendChartDownloadFlags(flags, release)
|
|
flags = st.appendShowOnlyFlags(flags, showOnly)
|
|
flags = st.appendSkipSchemaValidationFlags(flags, release, skipSchemaValidation)
|
|
// Issue #2309: Add kube-context flags for helm template when using jsonPatches with --dry-run=server
|
|
flags = st.appendConnectionFlags(flags, release)
|
|
|
|
common, files, err := st.namespaceAndValuesFlags(helm, release, workerIndex)
|
|
if err != nil {
|
|
return nil, files, err
|
|
}
|
|
return append(flags, common...), files, nil
|
|
}
|
|
|
|
func (st *HelmState) flagsForDiff(helm helmexec.Interface, release *ReleaseSpec, disableValidation bool, workerIndex int, opt *DiffOpts) ([]string, []string, error) {
|
|
// Use version-specific cli based on detected Helm version
|
|
var pluginsDir string
|
|
if helm.IsHelm3() {
|
|
pluginsDir = cliv3.New().PluginsDirectory
|
|
} else {
|
|
pluginsDir = cliv4.New().PluginsDirectory
|
|
}
|
|
var flags []string
|
|
flags = st.appendChartVersionFlags(flags, release)
|
|
flags = st.appendEnableDNSFlags(flags, release)
|
|
|
|
disableOpenAPIValidation := false
|
|
if release.DisableOpenAPIValidation != nil {
|
|
disableOpenAPIValidation = *release.DisableOpenAPIValidation
|
|
} else if st.HelmDefaults.DisableOpenAPIValidation != nil {
|
|
disableOpenAPIValidation = *st.HelmDefaults.DisableOpenAPIValidation
|
|
}
|
|
|
|
if disableOpenAPIValidation {
|
|
flags = append(flags, "--disable-openapi-validation")
|
|
}
|
|
|
|
if release.DisableValidation != nil {
|
|
disableValidation = *release.DisableValidation
|
|
} else if st.HelmDefaults.DisableValidation != nil {
|
|
disableValidation = *st.HelmDefaults.DisableValidation
|
|
}
|
|
|
|
if disableValidation {
|
|
flags = append(flags, "--disable-validation")
|
|
}
|
|
|
|
// Determine which kubeVersion to pass to helm diff based on configuration.
|
|
// By default (disableAutoDetectedKubeVersionForDiff=false), pass auto-detected kubeVersion
|
|
// to helm-diff. This fixes issue #2275 where charts requiring newer Kubernetes versions
|
|
// would fail because helm-diff defaults to v1.20.0.
|
|
//
|
|
// If disableAutoDetectedKubeVersionForDiff=true, only pass explicit kubeVersion from
|
|
// helmfile.yaml. This prevents helm-diff from normalizing server-side defaults which
|
|
// could hide real changes (e.g., ipFamilyPolicy, ipFamilies). Use this when server-side
|
|
// normalization causes issues with diff output.
|
|
disableAutoDetected := false
|
|
if release.DisableAutoDetectedKubeVersionForDiff != nil {
|
|
disableAutoDetected = *release.DisableAutoDetectedKubeVersionForDiff
|
|
} else if st.HelmDefaults.DisableAutoDetectedKubeVersionForDiff != nil {
|
|
disableAutoDetected = *st.HelmDefaults.DisableAutoDetectedKubeVersionForDiff
|
|
}
|
|
|
|
kubeVersionForDiff := ""
|
|
if disableAutoDetected {
|
|
// Only pass explicit kubeVersion from helmfile.yaml
|
|
if release.KubeVersion != "" {
|
|
kubeVersionForDiff = release.KubeVersion
|
|
} else if st.KubeVersion != "" {
|
|
kubeVersionForDiff = st.KubeVersion
|
|
}
|
|
} else {
|
|
// Pass auto-detected version (default behavior)
|
|
kubeVersionForDiff = ""
|
|
if opt != nil && opt.DetectedKubeVersion != "" {
|
|
kubeVersionForDiff = opt.DetectedKubeVersion
|
|
}
|
|
}
|
|
flags = st.appendApiVersionsFlags(flags, release, kubeVersionForDiff)
|
|
flags = st.appendConnectionFlags(flags, release)
|
|
flags = st.appendChartDownloadFlags(flags, release)
|
|
|
|
// `helm diff` does not support the `--plain-http` flag, this needs to be removed
|
|
repo, _ := st.GetRepositoryAndNameFromChartName(release.Chart)
|
|
if st.needsPlainHttp(release, repo) {
|
|
var cleanFlags []string
|
|
for _, flag := range flags {
|
|
if flag != "--plain-http" {
|
|
cleanFlags = append(cleanFlags, flag)
|
|
}
|
|
}
|
|
flags = cleanFlags
|
|
}
|
|
|
|
for _, flag := range flags {
|
|
if flag == "--insecure-skip-tls-verify" {
|
|
diffVersion, err := helmexec.GetPluginVersion("diff", pluginsDir)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
dv, _ := semver.NewVersion("v3.8.1")
|
|
|
|
if diffVersion.LessThan(dv) {
|
|
return nil, nil, fmt.Errorf("insecureSkipTLSVerify is not supported by helm-diff plugin version %s, please use at least v3.8.1", diffVersion)
|
|
}
|
|
|
|
break
|
|
}
|
|
}
|
|
|
|
flags = st.appendHelmXFlags(flags, release)
|
|
|
|
postRenderer := ""
|
|
if opt != nil {
|
|
postRenderer = opt.PostRenderer
|
|
}
|
|
flags = st.appendPostRenderFlags(flags, release, postRenderer, helm)
|
|
|
|
postRendererArgs := []string{}
|
|
if opt != nil {
|
|
postRendererArgs = opt.PostRendererArgs
|
|
}
|
|
var err error
|
|
flags, err = st.appendPostRenderArgsFlags(flags, release, postRendererArgs)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
skipSchemaValidation := false
|
|
if opt != nil {
|
|
skipSchemaValidation = opt.SkipSchemaValidation
|
|
}
|
|
flags = st.appendSkipSchemaValidationFlags(flags, release, skipSchemaValidation)
|
|
|
|
suppressOutputLineRegex := []string{}
|
|
if opt != nil {
|
|
suppressOutputLineRegex = opt.SuppressOutputLineRegex
|
|
}
|
|
if len(suppressOutputLineRegex) > 0 || len(st.HelmDefaults.SuppressOutputLineRegex) > 0 || len(release.SuppressOutputLineRegex) > 0 {
|
|
diffVersion, err := helmexec.GetPluginVersion("diff", pluginsDir)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
dv, _ := semver.NewVersion("v3.9.0")
|
|
|
|
if diffVersion.LessThan(dv) {
|
|
return nil, nil, fmt.Errorf("suppressOutputLineRegex is not supported by helm-diff plugin version %s, please use at least v3.9.0", diffVersion)
|
|
}
|
|
flags = st.appendSuppressOutputLineRegexFlags(flags, release, suppressOutputLineRegex)
|
|
}
|
|
|
|
takeOwnership := false
|
|
serverSide := ""
|
|
if opt != nil {
|
|
takeOwnership = opt.TakeOwnership
|
|
serverSide = opt.ServerSide
|
|
}
|
|
|
|
flags, err = st.appendTakeOwnershipFlagsForDiff(flags, release, takeOwnership, pluginsDir)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
// append server-side flag
|
|
flags, err = st.appendServerSideFlagsForDiff(flags, helm, release, serverSide, pluginsDir)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
common, files, err := st.namespaceAndValuesFlags(helm, release, workerIndex)
|
|
if err != nil {
|
|
return nil, files, err
|
|
}
|
|
|
|
return append(flags, common...), files, nil
|
|
}
|
|
|
|
func (st *HelmState) appendTakeOwnershipFlagsForDiff(flags []string, release *ReleaseSpec, takeOwnership bool, pluginsDir string) ([]string, error) {
|
|
isAppendTakeOwnership := false
|
|
|
|
switch {
|
|
case release.TakeOwnership != nil && *release.TakeOwnership:
|
|
isAppendTakeOwnership = true
|
|
case takeOwnership:
|
|
isAppendTakeOwnership = true
|
|
case st.HelmDefaults.TakeOwnership != nil && *st.HelmDefaults.TakeOwnership:
|
|
isAppendTakeOwnership = true
|
|
}
|
|
if isAppendTakeOwnership {
|
|
diffVersion, err := helmexec.GetPluginVersion("diff", pluginsDir)
|
|
if err != nil {
|
|
return flags, err
|
|
}
|
|
dv, _ := semver.NewVersion("v3.11.0")
|
|
|
|
if diffVersion.LessThan(dv) {
|
|
return flags, fmt.Errorf("take-ownership is not supported by helm-diff plugin version %s, please use at least v3.11.0", diffVersion)
|
|
}
|
|
flags = append(flags, "--take-ownership")
|
|
}
|
|
return flags, nil
|
|
}
|
|
|
|
// appendServerSideFlagsForDiff appends the helm 4 --server-side flag for helm-diff.
|
|
// It requires helm-diff plugin version v3.15.10 or later.
|
|
func (st *HelmState) appendServerSideFlagsForDiff(flags []string, helm helmexec.Interface, release *ReleaseSpec, serverSide string, pluginsDir string) ([]string, error) {
|
|
if helm.IsHelm4() {
|
|
value := st.resolveServerSideValue(release, serverSide)
|
|
if value != "" {
|
|
diffVersion, err := helmexec.GetPluginVersion("diff", pluginsDir)
|
|
if err != nil {
|
|
return flags, err
|
|
}
|
|
minVersion, _ := semver.NewVersion("v3.15.10")
|
|
|
|
if diffVersion.LessThan(minVersion) {
|
|
return flags, fmt.Errorf("server-side is not supported by helm-diff plugin version %s, please use at least v3.15.10", diffVersion)
|
|
}
|
|
}
|
|
}
|
|
|
|
return st.appendServerSideFlagsForUpgrade(flags, helm, release, serverSide)
|
|
}
|
|
|
|
func (st *HelmState) appendChartVersionFlags(flags []string, release *ReleaseSpec) []string {
|
|
version := release.Version
|
|
// Strip OCI digest from version (digest is handled in chart URL, not --version flag)
|
|
if idx := strings.Index(version, "@"); idx >= 0 {
|
|
version = version[:idx]
|
|
}
|
|
if version != "" {
|
|
flags = append(flags, "--version", version)
|
|
}
|
|
|
|
if st.isDevelopment(release) {
|
|
flags = append(flags, "--devel")
|
|
}
|
|
|
|
return flags
|
|
}
|
|
|
|
func (st *HelmState) chartOCIFlags(r *ReleaseSpec) []string {
|
|
flags := []string{}
|
|
repo, _ := st.GetRepositoryAndNameFromChartName(r.Chart)
|
|
if repo != nil {
|
|
if repo.PlainHttp {
|
|
flags = append(flags, "--plain-http")
|
|
} else {
|
|
// TLS options will nullify --plain-http in helm if passed, omit it them PlainHttp is specified
|
|
if repo.SkipTLSVerify {
|
|
flags = append(flags, "--insecure-skip-tls-verify")
|
|
}
|
|
if repo.CaFile != "" {
|
|
flags = append(flags, "--ca-file", repo.CaFile)
|
|
}
|
|
if repo.CertFile != "" && repo.KeyFile != "" {
|
|
flags = append(flags, "--cert-file", repo.CertFile, "--key-file", repo.KeyFile)
|
|
}
|
|
}
|
|
if repo.Verify {
|
|
flags = append(flags, "--verify")
|
|
}
|
|
if repo.Keyring != "" {
|
|
flags = append(flags, "--keyring", repo.Keyring)
|
|
}
|
|
if repo.RegistryConfig != "" {
|
|
flags = append(flags, "--registry-config", repo.RegistryConfig)
|
|
}
|
|
}
|
|
|
|
return flags
|
|
}
|
|
|
|
func (st *HelmState) appendValuesControlModeFlag(flags []string, reuseValues bool, resetValues bool, release *ReleaseSpec) []string {
|
|
if !resetValues && ((release.ReuseValues != nil && *release.ReuseValues || release.ReuseValues == nil && st.HelmDefaults.ReuseValues) || reuseValues) {
|
|
flags = append(flags, "--reuse-values")
|
|
} else {
|
|
flags = append(flags, "--reset-values")
|
|
}
|
|
|
|
return flags
|
|
}
|
|
|
|
func (st *HelmState) getApiVersions(r *ReleaseSpec) []string {
|
|
if len(r.ApiVersions) != 0 {
|
|
return r.ApiVersions
|
|
}
|
|
return st.ApiVersions
|
|
}
|
|
|
|
func (st *HelmState) getKubeVersion(r *ReleaseSpec, kubeVersion string) string {
|
|
switch {
|
|
case kubeVersion != "":
|
|
return kubeVersion
|
|
case r.KubeVersion != "":
|
|
return r.KubeVersion
|
|
case st.KubeVersion != "":
|
|
return st.KubeVersion
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (st *HelmState) appendApiVersionsFlags(flags []string, r *ReleaseSpec, kubeVersion string) []string {
|
|
for _, a := range st.getApiVersions(r) {
|
|
flags = append(flags, "--api-versions", a)
|
|
}
|
|
|
|
if version := st.getKubeVersion(r, kubeVersion); version != "" {
|
|
flags = append(flags, "--kube-version", version)
|
|
}
|
|
|
|
return flags
|
|
}
|
|
|
|
func (st *HelmState) isDevelopment(release *ReleaseSpec) bool {
|
|
result := st.HelmDefaults.Devel
|
|
if release.Devel != nil {
|
|
result = *release.Devel
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
func (st *HelmState) flagsForLint(helm helmexec.Interface, release *ReleaseSpec, workerIndex int) ([]string, []string, error) {
|
|
flags, files, err := st.namespaceAndValuesFlags(helm, release, workerIndex)
|
|
if err != nil {
|
|
return nil, files, err
|
|
}
|
|
|
|
return st.appendHelmXFlags(flags, release), files, nil
|
|
}
|
|
|
|
func (st *HelmState) newReleaseTemplateData(release *ReleaseSpec) releaseTemplateData {
|
|
vals := st.Values()
|
|
templateData := st.createReleaseTemplateData(release, vals)
|
|
|
|
return templateData
|
|
}
|
|
|
|
func (st *HelmState) mergedReleaseTemplateData(release *ReleaseSpec) (releaseTemplateData, error) {
|
|
releaseValues, err := st.resolveReleaseValues(release)
|
|
if err != nil {
|
|
return releaseTemplateData{}, err
|
|
}
|
|
mergedVals := maputil.MergeMaps(st.Values(), releaseValues)
|
|
return st.createReleaseTemplateData(release, mergedVals), nil
|
|
}
|
|
|
|
// prepareReleaseValuesEntries normalizes release.Values path entries (applying ValuesPathPrefix)
|
|
// and evaluates any vals ref+ secrets, returning the fully-rendered values slice ready for processing.
|
|
func (st *HelmState) prepareReleaseValuesEntries(release *ReleaseSpec) ([]any, error) {
|
|
values := []any{}
|
|
for _, v := range release.Values {
|
|
switch typedValue := v.(type) {
|
|
case string:
|
|
path := st.storage().normalizePath(release.ValuesPathPrefix + typedValue)
|
|
values = append(values, path)
|
|
default:
|
|
values = append(values, typedValue)
|
|
}
|
|
}
|
|
|
|
valuesMapSecretsRendered, err := st.valsRuntime.Eval(map[string]any{"values": values})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
valuesSecretsRendered, ok := valuesMapSecretsRendered["values"].([]any)
|
|
if !ok {
|
|
return nil, fmt.Errorf("failed to render values in %s for release %s: type %T isn't supported", st.FilePath, release.Name, valuesMapSecretsRendered["values"])
|
|
}
|
|
|
|
return valuesSecretsRendered, nil
|
|
}
|
|
|
|
func (st *HelmState) resolveReleaseValues(release *ReleaseSpec) (map[string]any, error) {
|
|
merged := map[string]any{}
|
|
|
|
valuesSecretsRendered, err := st.prepareReleaseValuesEntries(release)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
for _, v := range valuesSecretsRendered {
|
|
switch typedValue := v.(type) {
|
|
case string:
|
|
paths, skip, err := st.storage().resolveFile(st.getReleaseMissingFileHandler(release), "values", typedValue, st.getReleaseMissingFileHandlerConfig(release).resolveFileOptions()...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if skip {
|
|
continue
|
|
}
|
|
if len(paths) > 1 {
|
|
return nil, fmt.Errorf("glob patterns in release values are not supported for template data resolution: value=%q, resolvedPaths=%v", typedValue, paths)
|
|
}
|
|
path := paths[0]
|
|
|
|
yamlBytes, err := st.RenderReleaseValuesFileToBytes(release, path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to render values file \"%s\": %w", typedValue, err)
|
|
}
|
|
|
|
var rawVals map[string]any
|
|
if err := yaml.Unmarshal(yamlBytes, &rawVals); err != nil {
|
|
return nil, fmt.Errorf("failed to parse values file \"%s\": %w", typedValue, err)
|
|
}
|
|
|
|
// Normalize nested keys: yaml v2 may produce map[any]any for nested maps.
|
|
// CastKeysToStrings recurses through both map[any]any and map[string]any so it is
|
|
// safe to call even when yaml v3 is in use and keys are already strings.
|
|
normalizedVals, err := maputil.CastKeysToStrings(rawVals)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to normalize keys in values file \"%s\": %w", typedValue, err)
|
|
}
|
|
|
|
merged = maputil.MergeMaps(merged, normalizedVals)
|
|
case map[any]any:
|
|
strMap, err := maputil.CastKeysToStrings(typedValue)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
merged = maputil.MergeMaps(merged, strMap)
|
|
case map[string]any:
|
|
merged = maputil.MergeMaps(merged, typedValue)
|
|
default:
|
|
return nil, fmt.Errorf("unexpected type of value in release values: value=%v, type=%T", typedValue, typedValue)
|
|
}
|
|
}
|
|
|
|
return merged, nil
|
|
}
|
|
|
|
func (st *HelmState) renderValuesFileToBytesWithData(path string, templateData releaseTemplateData) ([]byte, error) {
|
|
r := tmpl.NewFileRenderer(st.fs, filepath.Dir(path), templateData)
|
|
rawBytes, err := r.RenderToBytes(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// If 'ref+.*' exists in file, run vals against the file
|
|
match, err := regexp.Match("ref\\+.*", rawBytes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if match {
|
|
var rawYaml map[string]any
|
|
|
|
if err := yaml.Unmarshal(rawBytes, &rawYaml); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
parsedYaml, err := st.valsRuntime.Eval(rawYaml)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return yaml.Marshal(parsedYaml)
|
|
}
|
|
|
|
return rawBytes, nil
|
|
}
|
|
|
|
func (st *HelmState) generateTemporaryReleaseValuesFilesWithData(release *ReleaseSpec, values []any, getTemplateData func() (releaseTemplateData, error)) ([]string, error) {
|
|
return st.generateTemporaryReleaseValuesFilesCore(release, values, func(path string) ([]byte, error) {
|
|
templateData, err := getTemplateData()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return st.renderValuesFileToBytesWithData(path, templateData)
|
|
})
|
|
}
|
|
|
|
func (st *HelmState) newReleaseTemplateFuncMap(dir string) template.FuncMap {
|
|
r := tmpl.NewFileRenderer(st.fs, dir, nil)
|
|
|
|
return r.Context.CreateFuncMap()
|
|
}
|
|
|
|
func (st *HelmState) RenderReleaseValuesFileToBytes(release *ReleaseSpec, path string) ([]byte, error) {
|
|
templateData := st.newReleaseTemplateData(release)
|
|
return st.renderValuesFileToBytesWithData(path, templateData)
|
|
}
|
|
|
|
func (st *HelmState) storage() *Storage {
|
|
return &Storage{
|
|
FilePath: st.FilePath,
|
|
basePath: st.basePath,
|
|
logger: st.logger,
|
|
fs: st.fs,
|
|
}
|
|
}
|
|
|
|
func (st *HelmState) ExpandedHelmfiles() ([]SubHelmfileSpec, error) {
|
|
helmfiles := []SubHelmfileSpec{}
|
|
for _, hf := range st.Helmfiles {
|
|
if remote.IsRemote(hf.Path) {
|
|
helmfiles = append(helmfiles, hf)
|
|
continue
|
|
}
|
|
|
|
matches, err := st.storage().ExpandPaths(hf.Path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(matches) == 0 {
|
|
err := fmt.Errorf("no matches for path: %s", hf.Path)
|
|
if *st.getMissingFileHandler() == "Error" {
|
|
return nil, err
|
|
}
|
|
st.logger.Warnf("no matches for path: %s", hf.Path)
|
|
continue
|
|
}
|
|
for _, match := range matches {
|
|
newHelmfile := hf
|
|
newHelmfile.Path = match
|
|
helmfiles = append(helmfiles, newHelmfile)
|
|
}
|
|
}
|
|
|
|
return helmfiles, nil
|
|
}
|
|
|
|
func (st *HelmState) removeFiles(files []string) {
|
|
dirsToClean := map[string]int{}
|
|
for _, f := range files {
|
|
dirsToClean[filepath.Dir(f)] = 1
|
|
if err := st.fs.DeleteFile(f); err != nil {
|
|
st.logger.Warnf("Removing %s: %v", f, err)
|
|
} else {
|
|
st.logger.Debugf("Removed %s", f)
|
|
}
|
|
}
|
|
for d := range dirsToClean {
|
|
// check if the directory is empty
|
|
des, err := st.fs.ReadDir(d)
|
|
if err != nil {
|
|
st.logger.Warnf("Reading dir %s: %v", d, err)
|
|
continue
|
|
}
|
|
|
|
if len(des) > 0 {
|
|
st.logger.Debugf("Not removing %s because it's not empty", d)
|
|
continue
|
|
}
|
|
|
|
if err := st.fs.DeleteFile(d); err != nil {
|
|
st.logger.Warnf("Removing %s: %v", d, err)
|
|
} else {
|
|
st.logger.Debugf("Removed %s", d)
|
|
}
|
|
}
|
|
}
|
|
|
|
// addChartifyTempDir records a chartify output directory for deferred cleanup.
|
|
// The directory is removed by CleanupChartifyTempDirs after all helm operations
|
|
// have completed, since the chartified chart may still be in use. See issue #1799.
|
|
func (st *HelmState) addChartifyTempDir(dir string) {
|
|
if st.chartifyTempDirs == nil {
|
|
st.chartifyTempDirs = &chartifyTempDirTracker{}
|
|
}
|
|
st.chartifyTempDirs.mu.Lock()
|
|
defer st.chartifyTempDirs.mu.Unlock()
|
|
st.chartifyTempDirs.dirs = append(st.chartifyTempDirs.dirs, dir)
|
|
}
|
|
|
|
// CleanupChartifyTempDirs removes all chartify output directories tracked via
|
|
// addChartifyTempDir. It should be called after all helm operations complete.
|
|
// See issue #1799.
|
|
func (st *HelmState) CleanupChartifyTempDirs() {
|
|
if st.chartifyTempDirs == nil {
|
|
return
|
|
}
|
|
|
|
st.chartifyTempDirs.mu.Lock()
|
|
dirs := st.chartifyTempDirs.dirs
|
|
st.chartifyTempDirs.dirs = nil
|
|
st.chartifyTempDirs.mu.Unlock()
|
|
|
|
seen := make(map[string]bool, len(dirs))
|
|
parents := make([]string, 0, len(dirs))
|
|
|
|
for _, dir := range dirs {
|
|
if dir == "" || seen[dir] {
|
|
continue
|
|
}
|
|
seen[dir] = true
|
|
|
|
if err := st.fs.RemoveAll(dir); err != nil {
|
|
st.logger.Warnf("Removing chartify temp dir %s: %v", dir, err)
|
|
} else {
|
|
st.logger.Debugf("Removed chartify temp dir %s", dir)
|
|
}
|
|
|
|
// Collect the parent temp directory (e.g. /tmp/chartify<random>) for
|
|
// later removal. os.Remove only succeeds on empty directories, so we
|
|
// attempt it after all child directories have been removed.
|
|
parent := filepath.Dir(dir)
|
|
if parent != "" && parent != "/" && !seen[parent] {
|
|
seen[parent] = true
|
|
parents = append(parents, parent)
|
|
}
|
|
}
|
|
|
|
for _, parent := range parents {
|
|
if err := os.Remove(parent); err != nil {
|
|
st.logger.Debugf("Not removing parent temp dir %s: %v", parent, err)
|
|
} else {
|
|
st.logger.Debugf("Removed parent temp dir %s", parent)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *MissingFileHandlerConfig) resolveFileOptions() []resolveFileOption {
|
|
if c == nil {
|
|
return []resolveFileOption{
|
|
ignoreMissingGitBranch(false),
|
|
}
|
|
}
|
|
return []resolveFileOption{
|
|
ignoreMissingGitBranch(c.IgnoreMissingGitBranch),
|
|
}
|
|
}
|
|
|
|
// getReleaseMissingFileHandlerConfig returns the first non-nil MissingFileHandlerConfig in the following order:
|
|
// - release.MissingFileHandlerConfig
|
|
// - st.MissingFileHandlerConfig
|
|
func (st *HelmState) getReleaseMissingFileHandlerConfig(release *ReleaseSpec) *MissingFileHandlerConfig {
|
|
switch {
|
|
case release.MissingFileHandlerConfig != nil:
|
|
return release.MissingFileHandlerConfig
|
|
case st.MissingFileHandlerConfig != nil:
|
|
return st.MissingFileHandlerConfig
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// getReleaseMissingFileHandler returns the first non-nil MissingFileHandler in the following order:
|
|
// - release.MissingFileHandler
|
|
// - st.MissingFileHandler
|
|
// - "Error"
|
|
func (st *HelmState) getReleaseMissingFileHandler(release *ReleaseSpec) *string {
|
|
defaultMissingFileHandler := "Error"
|
|
switch {
|
|
case release.MissingFileHandler != nil:
|
|
return release.MissingFileHandler
|
|
case st.MissingFileHandler != nil:
|
|
return st.MissingFileHandler
|
|
default:
|
|
return &defaultMissingFileHandler
|
|
}
|
|
}
|
|
|
|
// getMissingFileHandler returns the first non-nil MissingFileHandler in the following order:
|
|
// - st.MissingFileHandler
|
|
// - "Error"
|
|
func (st *HelmState) getMissingFileHandler() *string {
|
|
defaultMissingFileHandler := "Error"
|
|
switch {
|
|
case st.MissingFileHandler != nil:
|
|
return st.MissingFileHandler
|
|
default:
|
|
return &defaultMissingFileHandler
|
|
}
|
|
}
|
|
|
|
func (st *HelmState) generateTemporaryReleaseValuesFiles(release *ReleaseSpec, values []any) ([]string, error) {
|
|
return st.generateTemporaryReleaseValuesFilesCore(release, values, func(path string) ([]byte, error) {
|
|
return st.RenderReleaseValuesFileToBytes(release, path)
|
|
})
|
|
}
|
|
|
|
// generateTemporaryReleaseValuesFilesCore is the shared implementation for generating temporary values files.
|
|
// renderStringValue is called for each string value entry after the file path has been resolved.
|
|
func (st *HelmState) generateTemporaryReleaseValuesFilesCore(release *ReleaseSpec, values []any, renderStringValue func(path string) ([]byte, error)) ([]string, error) {
|
|
generatedFiles := []string{}
|
|
|
|
for _, value := range values {
|
|
switch typedValue := value.(type) {
|
|
case string:
|
|
paths, skip, err := st.storage().resolveFile(st.getReleaseMissingFileHandler(release), "values", typedValue, st.getReleaseMissingFileHandlerConfig(release).resolveFileOptions()...)
|
|
if err != nil {
|
|
return generatedFiles, err
|
|
}
|
|
if skip {
|
|
continue
|
|
}
|
|
|
|
if len(paths) > 1 {
|
|
return generatedFiles, fmt.Errorf("glob patterns in release values and secrets is not supported yet. please submit a feature request if necessary")
|
|
}
|
|
path := paths[0]
|
|
|
|
yamlBytes, err := renderStringValue(path)
|
|
if err != nil {
|
|
return generatedFiles, fmt.Errorf("failed to render values files \"%s\": %v", typedValue, err)
|
|
}
|
|
|
|
if err := func() error {
|
|
valfile, err := createTempValuesFile(release, yamlBytes)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() {
|
|
_ = valfile.Close()
|
|
}()
|
|
|
|
if _, err := valfile.Write(yamlBytes); err != nil {
|
|
return fmt.Errorf("failed to write %s: %v", valfile.Name(), err)
|
|
}
|
|
|
|
st.logger.Debugf("Successfully generated the value file from %s to %s", path, valfile.Name())
|
|
|
|
generatedFiles = append(generatedFiles, valfile.Name())
|
|
|
|
return nil
|
|
}(); err != nil {
|
|
return generatedFiles, err
|
|
}
|
|
case map[any]any:
|
|
strMap, err := maputil.CastKeysToStrings(typedValue)
|
|
if err != nil {
|
|
return generatedFiles, err
|
|
}
|
|
if err := func() error {
|
|
valfile, err := createTempValuesFile(release, strMap)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() {
|
|
_ = valfile.Close()
|
|
}()
|
|
|
|
encoder := yaml.NewEncoder(valfile)
|
|
defer func() {
|
|
_ = encoder.Close()
|
|
}()
|
|
|
|
if err := encoder.Encode(strMap); err != nil {
|
|
return err
|
|
}
|
|
|
|
generatedFiles = append(generatedFiles, valfile.Name())
|
|
|
|
return nil
|
|
}(); err != nil {
|
|
return generatedFiles, err
|
|
}
|
|
case map[string]any:
|
|
if err := func() error {
|
|
valfile, err := createTempValuesFile(release, typedValue)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() {
|
|
_ = valfile.Close()
|
|
}()
|
|
|
|
encoder := yaml.NewEncoder(valfile)
|
|
defer func() {
|
|
_ = encoder.Close()
|
|
}()
|
|
|
|
if err := encoder.Encode(typedValue); err != nil {
|
|
return err
|
|
}
|
|
|
|
generatedFiles = append(generatedFiles, valfile.Name())
|
|
|
|
return nil
|
|
}(); err != nil {
|
|
return generatedFiles, err
|
|
}
|
|
default:
|
|
return generatedFiles, fmt.Errorf("unexpected type of value: value=%v, type=%T", typedValue, typedValue)
|
|
}
|
|
}
|
|
return generatedFiles, nil
|
|
}
|
|
|
|
func (st *HelmState) generateVanillaValuesFiles(release *ReleaseSpec) ([]string, error) {
|
|
valuesSecretsRendered, err := st.prepareReleaseValuesEntries(release)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
generatedFiles, err := st.generateTemporaryReleaseValuesFiles(release, valuesSecretsRendered)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return generatedFiles, nil
|
|
}
|
|
|
|
func (st *HelmState) generateSecretValuesFiles(helm helmexec.Interface, release *ReleaseSpec, workerIndex int) ([]string, error) {
|
|
var generatedDecryptedFiles []any
|
|
|
|
for _, v := range release.Secrets {
|
|
var (
|
|
paths []string
|
|
skip bool
|
|
err error
|
|
)
|
|
|
|
switch value := v.(type) {
|
|
case string:
|
|
paths, skip, err = st.storage().resolveFile(release.MissingFileHandler, "secrets", release.ValuesPathPrefix+value, st.MissingFileHandlerConfig.resolveFileOptions()...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
default:
|
|
bs, err := yaml.Marshal(value)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
path, err := os.CreateTemp(os.TempDir(), "helmfile-embdedded-secrets-*.yaml.enc")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
_ = path.Close()
|
|
defer func() {
|
|
_ = os.Remove(path.Name())
|
|
}()
|
|
|
|
if err := os.WriteFile(path.Name(), bs, 0644); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
paths = []string{path.Name()}
|
|
}
|
|
|
|
if skip {
|
|
continue
|
|
}
|
|
|
|
if len(paths) > 1 {
|
|
return nil, fmt.Errorf("glob patterns in release secret file is not supported yet. please submit a feature request if necessary")
|
|
}
|
|
path := paths[0]
|
|
|
|
valfile, err := helm.DecryptSecret(st.createHelmContext(release, workerIndex), path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() {
|
|
_ = os.Remove(valfile)
|
|
}()
|
|
|
|
generatedDecryptedFiles = append(generatedDecryptedFiles, valfile)
|
|
}
|
|
|
|
generatedFiles, err := st.generateTemporaryReleaseValuesFiles(release, generatedDecryptedFiles)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return generatedFiles, nil
|
|
}
|
|
|
|
func (st *HelmState) generateValuesFiles(helm helmexec.Interface, release *ReleaseSpec, workerIndex int) ([]string, error) {
|
|
valuesFiles, err := st.generateVanillaValuesFiles(release)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
secretValuesFiles, err := st.generateSecretValuesFiles(helm, release, workerIndex)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
files := append(valuesFiles, secretValuesFiles...)
|
|
|
|
return files, nil
|
|
}
|
|
|
|
func (st *HelmState) namespaceAndValuesFlags(helm helmexec.Interface, release *ReleaseSpec, workerIndex int) ([]string, []string, error) {
|
|
flags := []string{}
|
|
if release.Namespace != "" {
|
|
flags = append(flags, "--namespace", release.Namespace)
|
|
}
|
|
|
|
var files []string
|
|
|
|
generatedFiles, err := st.generateValuesFiles(helm, release, workerIndex)
|
|
if err != nil {
|
|
return nil, files, err
|
|
}
|
|
|
|
files = generatedFiles
|
|
|
|
for _, f := range generatedFiles {
|
|
flags = append(flags, "--values", f)
|
|
}
|
|
|
|
if len(release.SetValues) > 0 {
|
|
setFlags, err := st.setFlags(release.SetValues)
|
|
if err != nil {
|
|
return nil, files, fmt.Errorf("Failed to render set value entry in %s for release %s: %v", st.FilePath, release.Name, err)
|
|
}
|
|
|
|
flags = append(flags, setFlags...)
|
|
}
|
|
|
|
if len(release.SetStringValues) > 0 {
|
|
setStringFlags, err := st.setStringFlags(release.SetStringValues)
|
|
if err != nil {
|
|
return nil, files, fmt.Errorf("Failed to render set string value entry in %s for release %s: %v", st.FilePath, release.Name, err)
|
|
}
|
|
|
|
flags = append(flags, setStringFlags...)
|
|
}
|
|
|
|
/***********
|
|
* START 'env' section for backwards compatibility
|
|
***********/
|
|
// The 'env' section is not really necessary any longer, as 'set' would now provide the same functionality
|
|
if len(release.EnvValues) > 0 {
|
|
val := []string{}
|
|
envValErrs := []string{}
|
|
for _, set := range release.EnvValues {
|
|
value, isSet := os.LookupEnv(set.Value)
|
|
if isSet {
|
|
val = append(val, fmt.Sprintf("%s=%s", escape(set.Name), escape(value)))
|
|
} else {
|
|
errMsg := fmt.Sprintf("\t%s", set.Value)
|
|
envValErrs = append(envValErrs, errMsg)
|
|
}
|
|
}
|
|
if len(envValErrs) != 0 {
|
|
joinedEnvVals := strings.Join(envValErrs, "\n")
|
|
errMsg := fmt.Sprintf("Environment Variables not found. Please make sure they are set and try again:\n%s", joinedEnvVals)
|
|
return nil, files, errors.New(errMsg)
|
|
}
|
|
flags = append(flags, "--set", strings.Join(val, ","))
|
|
}
|
|
/**************
|
|
* END 'env' section for backwards compatibility
|
|
**************/
|
|
|
|
return flags, files, nil
|
|
}
|
|
|
|
func (st *HelmState) setFlags(setValues []SetValue) ([]string, error) {
|
|
var flags []string
|
|
|
|
for _, set := range setValues {
|
|
if set.Value != "" {
|
|
renderedValue, err := renderValsSecrets(st.valsRuntime, set.Value)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
flags = append(flags, "--set", fmt.Sprintf("%s=%s", escape(set.Name), escape(renderedValue[0])))
|
|
} else if set.File != "" {
|
|
flags = append(flags, "--set-file", fmt.Sprintf("%s=%s", escape(set.Name), st.storage().normalizeSetFilePath(set.File, runtime.GOOS)))
|
|
} else if len(set.Values) > 0 {
|
|
renderedValues, err := renderValsSecretsAny(st.valsRuntime, set.Values)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items := make([]string, len(renderedValues))
|
|
for i, raw := range renderedValues {
|
|
items[i] = escape(raw)
|
|
}
|
|
v := strings.Join(items, ",")
|
|
flags = append(flags, "--set", fmt.Sprintf("%s={%s}", escape(set.Name), v))
|
|
}
|
|
}
|
|
|
|
return flags, nil
|
|
}
|
|
|
|
// setStringFlags is to generate the set-string flags for helm
|
|
func (st *HelmState) setStringFlags(setValues []SetValue) ([]string, error) {
|
|
var flags []string
|
|
|
|
for _, set := range setValues {
|
|
if set.Value != "" {
|
|
renderedValue, err := renderValsSecrets(st.valsRuntime, set.Value)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
flags = append(flags, "--set-string", fmt.Sprintf("%s=%s", escape(set.Name), escape(renderedValue[0])))
|
|
} else if len(set.Values) > 0 {
|
|
renderedValues, err := renderValsSecretsAny(st.valsRuntime, set.Values)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items := make([]string, len(renderedValues))
|
|
for i, raw := range renderedValues {
|
|
items[i] = escape(raw)
|
|
}
|
|
v := strings.Join(items, ",")
|
|
flags = append(flags, "--set-string", fmt.Sprintf("%s={%s}", escape(set.Name), v))
|
|
}
|
|
}
|
|
|
|
return flags, nil
|
|
}
|
|
|
|
// renderValsSecrets helper function which renders 'ref+.*' secrets
|
|
func renderValsSecrets(e vals.Evaluator, input ...string) ([]string, error) {
|
|
output := make([]string, len(input))
|
|
if len(input) > 0 {
|
|
mapRendered, err := e.Eval(map[string]any{"values": input})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rendered, ok := mapRendered["values"].([]any)
|
|
if !ok {
|
|
return nil, fmt.Errorf("type %T isn't supported", mapRendered["values"])
|
|
}
|
|
|
|
for i := 0; i < len(rendered); i++ {
|
|
output[i] = fmt.Sprintf("%v", rendered[i])
|
|
}
|
|
}
|
|
return output, nil
|
|
}
|
|
|
|
// renderValsSecretsAny renders 'ref+.*' secrets in a slice of any-typed values.
|
|
// Map values are serialized to JSON; string values are rendered via vals.
|
|
func renderValsSecretsAny(e vals.Evaluator, input []any) ([]string, error) {
|
|
output := make([]string, len(input))
|
|
if len(input) == 0 {
|
|
return output, nil
|
|
}
|
|
|
|
strInputs := make([]string, 0, len(input))
|
|
strIndexMap := make([]int, 0, len(input))
|
|
for i, v := range input {
|
|
switch tv := v.(type) {
|
|
case string:
|
|
strInputs = append(strInputs, tv)
|
|
strIndexMap = append(strIndexMap, i)
|
|
default:
|
|
jsonBytes, err := json.Marshal(tv)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to marshal set value at index %d: %w", i, err)
|
|
}
|
|
output[i] = string(jsonBytes)
|
|
}
|
|
}
|
|
|
|
if len(strInputs) > 0 {
|
|
rendered, err := renderValsSecrets(e, strInputs...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for idx, renderedIdx := range strIndexMap {
|
|
output[renderedIdx] = rendered[idx]
|
|
}
|
|
}
|
|
|
|
return output, nil
|
|
}
|
|
|
|
func hideChartCredentials(chartCredentials string) (string, error) {
|
|
u, err := url.Parse(chartCredentials)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if u.User != nil {
|
|
u.User = url.UserPassword("---", "---")
|
|
}
|
|
modifiedURL := u.String()
|
|
return modifiedURL, nil
|
|
}
|
|
|
|
// DisplayAffectedReleases logs the upgraded, deleted and in error releases.
|
|
// useColor controls whether the section headers carry the bold+blue style we
|
|
// use elsewhere for visual section breaks; pass false (e.g. when --no-color is
|
|
// set) for clean output in non-TTY consumers.
|
|
func (ar *AffectedReleases) DisplayAffectedReleases(logger *zap.SugaredLogger, useColor bool) {
|
|
if len(ar.Upgraded) > 0 {
|
|
logger.Infof("\n%s", kubedog.HeaderDividerStyled("Updated Releases", useColor))
|
|
tbl, _ := prettytable.NewTable(prettytable.Column{Header: "NAME"},
|
|
prettytable.Column{Header: "NAMESPACE", MinWidth: 6},
|
|
prettytable.Column{Header: "CHART", MinWidth: 6},
|
|
prettytable.Column{Header: "VERSION", MinWidth: 6},
|
|
prettytable.Column{Header: "DURATION", AlignRight: true},
|
|
)
|
|
tbl.Separator = " "
|
|
for _, release := range ar.Upgraded {
|
|
modifiedChart, modErr := hideChartCredentials(release.Chart)
|
|
if modErr != nil {
|
|
logger.Warn("Could not modify chart credentials, %v", modErr)
|
|
continue
|
|
}
|
|
err := tbl.AddRow(release.Name, release.Namespace, modifiedChart, release.installedVersion, release.duration.Round(time.Second))
|
|
if err != nil {
|
|
logger.Warn("Could not add row, %v", err)
|
|
}
|
|
}
|
|
logger.Info(tbl.String())
|
|
}
|
|
if len(ar.Reinstalled) > 0 {
|
|
logger.Infof("\n%s", kubedog.HeaderDividerStyled("Reinstalled Releases", useColor))
|
|
tbl, _ := prettytable.NewTable(prettytable.Column{Header: "NAME"},
|
|
prettytable.Column{Header: "NAMESPACE", MinWidth: 6},
|
|
prettytable.Column{Header: "CHART", MinWidth: 6},
|
|
prettytable.Column{Header: "VERSION", MinWidth: 6},
|
|
prettytable.Column{Header: "DURATION", AlignRight: true},
|
|
)
|
|
tbl.Separator = " "
|
|
for _, release := range ar.Reinstalled {
|
|
modifiedChart, modErr := hideChartCredentials(release.Chart)
|
|
if modErr != nil {
|
|
logger.Warn("Could not modify chart credentials, %v", modErr)
|
|
continue
|
|
}
|
|
err := tbl.AddRow(release.Name, release.Namespace, modifiedChart, release.installedVersion, release.duration.Round(time.Second))
|
|
if err != nil {
|
|
logger.Warn("Could not add row, %v", err)
|
|
}
|
|
}
|
|
logger.Info(tbl.String())
|
|
}
|
|
if len(ar.Deleted) > 0 {
|
|
logger.Infof("\n%s", kubedog.HeaderDividerStyled("Deleted Releases", useColor))
|
|
tbl, _ := prettytable.NewTable(prettytable.Column{Header: "NAME"},
|
|
prettytable.Column{Header: "NAMESPACE", MinWidth: 6},
|
|
prettytable.Column{Header: "DURATION", AlignRight: true},
|
|
)
|
|
tbl.Separator = " "
|
|
for _, release := range ar.Deleted {
|
|
err := tbl.AddRow(release.Name, release.Namespace, release.duration.Round(time.Second))
|
|
if err != nil {
|
|
logger.Warn("Could not add row, %v", err)
|
|
}
|
|
}
|
|
logger.Info(tbl.String())
|
|
}
|
|
if len(ar.Failed) > 0 {
|
|
logger.Infof("\n%s", kubedog.HeaderDividerStyled("Failed Releases", useColor))
|
|
tbl, _ := prettytable.NewTable(prettytable.Column{Header: "NAME"},
|
|
prettytable.Column{Header: "NAMESPACE", MinWidth: 6},
|
|
prettytable.Column{Header: "CHART", MinWidth: 6},
|
|
prettytable.Column{Header: "VERSION", MinWidth: 6},
|
|
prettytable.Column{Header: "DURATION", AlignRight: true},
|
|
)
|
|
tbl.Separator = " "
|
|
for _, release := range ar.Failed {
|
|
err := tbl.AddRow(release.Name, release.Namespace, release.Chart, release.installedVersion, release.duration.Round(time.Second))
|
|
if err != nil {
|
|
logger.Warn("Could not add row, %v", err)
|
|
}
|
|
}
|
|
logger.Info(tbl.String())
|
|
}
|
|
if len(ar.DeleteFailed) > 0 {
|
|
logger.Infof("\n%s", kubedog.HeaderDividerStyled("Failed to Delete Releases", useColor))
|
|
tbl, _ := prettytable.NewTable(prettytable.Column{Header: "NAME"},
|
|
prettytable.Column{Header: "NAMESPACE", MinWidth: 6},
|
|
prettytable.Column{Header: "DURATION", AlignRight: true},
|
|
)
|
|
tbl.Separator = " "
|
|
for _, release := range ar.DeleteFailed {
|
|
err := tbl.AddRow(release.Name, release.Namespace, release.duration.Round(time.Second))
|
|
if err != nil {
|
|
logger.Warn("Could not add row, %v", err)
|
|
}
|
|
}
|
|
logger.Info(tbl.String())
|
|
}
|
|
}
|
|
|
|
func escape(value string) string {
|
|
intermediate := strings.ReplaceAll(value, "{", "\\{")
|
|
intermediate = strings.ReplaceAll(intermediate, "}", "\\}")
|
|
return strings.ReplaceAll(intermediate, ",", "\\,")
|
|
}
|
|
|
|
// MarshalYAML will ensure we correctly marshal SubHelmfileSpec structure correctly so it can be unmarshalled at some
|
|
// future time
|
|
func (p SubHelmfileSpec) MarshalYAML() (any, error) {
|
|
type SubHelmfileSpecTmp struct {
|
|
Path string `yaml:"path,omitempty"`
|
|
Selectors []string `yaml:"selectors,omitempty"`
|
|
SelectorsInherited bool `yaml:"selectorsInherited,omitempty"`
|
|
OverrideValues []any `yaml:"values,omitempty"`
|
|
}
|
|
return &SubHelmfileSpecTmp{
|
|
Path: p.Path,
|
|
Selectors: p.Selectors,
|
|
SelectorsInherited: p.SelectorsInherited,
|
|
OverrideValues: p.Environment.OverrideValues,
|
|
}, nil
|
|
}
|
|
|
|
// UnmarshalYAML will unmarshal the helmfile yaml section and fill the SubHelmfileSpec structure
|
|
// this is required go-yto keep allowing string scalar for defining helmfile
|
|
func (hf *SubHelmfileSpec) UnmarshalYAML(unmarshal func(any) error) error {
|
|
var tmp any
|
|
if err := unmarshal(&tmp); err != nil {
|
|
return err
|
|
}
|
|
|
|
switch i := tmp.(type) {
|
|
case string: // single path definition without sub items, legacy sub helmfile definition
|
|
hf.Path = i
|
|
case map[any]any, map[string]any: // helmfile path with sub section
|
|
var subHelmfileSpecTmp struct {
|
|
Path string `yaml:"path"`
|
|
Selectors []string `yaml:"selectors"`
|
|
SelectorsInherited bool `yaml:"selectorsInherited"`
|
|
|
|
Environment SubhelmfileEnvironmentSpec `yaml:",inline"`
|
|
}
|
|
if err := unmarshal(&subHelmfileSpecTmp); err != nil {
|
|
return err
|
|
}
|
|
hf.Path = subHelmfileSpecTmp.Path
|
|
hf.Selectors = subHelmfileSpecTmp.Selectors
|
|
hf.SelectorsInherited = subHelmfileSpecTmp.SelectorsInherited
|
|
hf.Environment = subHelmfileSpecTmp.Environment
|
|
}
|
|
// since we cannot make sur the "console" string can be red after the "path" we must check we don't have
|
|
// a SubHelmfileSpec with only selector and no path
|
|
if hf.Selectors != nil && hf.Path == "" {
|
|
return fmt.Errorf("found 'selectors' definition without path: %v", hf.Selectors)
|
|
}
|
|
// also exclude SelectorsInherited to true and explicit selectors
|
|
if hf.SelectorsInherited && len(hf.Selectors) > 0 {
|
|
return fmt.Errorf("you cannot use 'SelectorsInherited: true' along with and explicit selector for path: %v", hf.Path)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (st *HelmState) GenerateOutputDir(outputDir string, release *ReleaseSpec, outputDirTemplate string) (string, error) {
|
|
// get absolute path of state file to generate a hash
|
|
// use this hash to write helm output in a specific directory by state file and release name
|
|
// ie. in a directory named stateFileName-stateFileHash-releaseName
|
|
stateAbsPath, err := filepath.Abs(st.FilePath)
|
|
if err != nil {
|
|
return stateAbsPath, err
|
|
}
|
|
|
|
hasher := sha1.New()
|
|
_, err = io.WriteString(hasher, stateAbsPath)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
var stateFileExtension = filepath.Ext(st.FilePath)
|
|
var stateFileName = st.FilePath[0 : len(st.FilePath)-len(stateFileExtension)]
|
|
|
|
sha1sum := hex.EncodeToString(hasher.Sum(nil))[:8]
|
|
|
|
var sb strings.Builder
|
|
sb.WriteString(stateFileName)
|
|
sb.WriteString("-")
|
|
sb.WriteString(sha1sum)
|
|
sb.WriteString("-")
|
|
sb.WriteString(release.Name)
|
|
|
|
if outputDirTemplate == "" {
|
|
outputDirTemplate = filepath.Join("{{ .OutputDir }}", "{{ .State.BaseName }}-{{ .State.AbsPathSHA1 }}-{{ .Release.Name}}")
|
|
}
|
|
|
|
t, err := template.New("output-dir").Parse(outputDirTemplate)
|
|
if err != nil {
|
|
return "", fmt.Errorf("parsing output-dir template")
|
|
}
|
|
|
|
buf := &bytes.Buffer{}
|
|
|
|
type state struct {
|
|
BaseName string
|
|
Path string
|
|
AbsPath string
|
|
AbsPathSHA1 string
|
|
}
|
|
|
|
// Template data for output-dir-template. Environment provides .Name, .KubeContext, and .Values fields.
|
|
data := struct {
|
|
OutputDir string
|
|
State state
|
|
Release *ReleaseSpec
|
|
Environment environment.Environment
|
|
}{
|
|
OutputDir: outputDir,
|
|
State: state{
|
|
BaseName: stateFileName,
|
|
Path: st.FilePath,
|
|
AbsPath: stateAbsPath,
|
|
AbsPathSHA1: sha1sum,
|
|
},
|
|
Release: release,
|
|
Environment: st.Env,
|
|
}
|
|
|
|
if err := t.Execute(buf, data); err != nil {
|
|
return "", fmt.Errorf("executing output-dir template: %w", err)
|
|
}
|
|
|
|
return buf.String(), nil
|
|
}
|
|
|
|
// generateChartPath generates the path of the output directory for chart downloads (e.g., fetch, pull, OCI chart downloads).
|
|
// It uses a go template with data from the chart name, output directory, release spec, and environment.
|
|
// If no template was provided (via the `--output-dir-template` flag) it uses the DefaultFetchOutputDirTemplate.
|
|
func (st *HelmState) generateChartPath(chartName string, outputDir string, release *ReleaseSpec, outputDirTemplate string) (string, error) {
|
|
if outputDirTemplate == "" {
|
|
outputDirTemplate = DefaultFetchOutputDirTemplate
|
|
}
|
|
|
|
t, err := template.New("output-dir-template").Parse(outputDirTemplate)
|
|
if err != nil {
|
|
return "", fmt.Errorf("parsing output-dir-template template %q: %w", outputDirTemplate, err)
|
|
}
|
|
|
|
buf := &bytes.Buffer{}
|
|
// Template data for output-dir-template. Environment provides .Name, .KubeContext, and .Values fields.
|
|
data := struct {
|
|
ChartName string
|
|
OutputDir string
|
|
Release ReleaseSpec
|
|
Environment environment.Environment
|
|
}{
|
|
ChartName: chartName,
|
|
OutputDir: outputDir,
|
|
Release: *release,
|
|
Environment: st.Env,
|
|
}
|
|
if err := t.Execute(buf, data); err != nil {
|
|
return "", fmt.Errorf("executing output-dir-template template: %w", err)
|
|
}
|
|
|
|
return buf.String(), nil
|
|
}
|
|
|
|
func (st *HelmState) GenerateOutputFilePath(release *ReleaseSpec, outputFileTemplate string) (string, error) {
|
|
// get absolute path of state file to generate a hash
|
|
// use this hash to write helm output in a specific directory by state file and release name
|
|
// ie. in a directory named stateFileName-stateFileHash-releaseName
|
|
stateAbsPath, err := filepath.Abs(st.FilePath)
|
|
if err != nil {
|
|
return stateAbsPath, err
|
|
}
|
|
|
|
hasher := sha1.New()
|
|
_, err = io.WriteString(hasher, stateAbsPath)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
var stateFileExtension = filepath.Ext(st.FilePath)
|
|
var stateFileName = st.FilePath[0 : len(st.FilePath)-len(stateFileExtension)]
|
|
|
|
sha1sum := hex.EncodeToString(hasher.Sum(nil))[:8]
|
|
|
|
var sb strings.Builder
|
|
sb.WriteString(stateFileName)
|
|
sb.WriteString("-")
|
|
sb.WriteString(sha1sum)
|
|
sb.WriteString("-")
|
|
sb.WriteString(release.Name)
|
|
|
|
if outputFileTemplate == "" {
|
|
outputFileTemplate = filepath.Join("{{ .State.BaseName }}-{{ .State.AbsPathSHA1 }}", "{{ .Release.Name }}.yaml")
|
|
}
|
|
|
|
t, err := template.New("output-file").Parse(outputFileTemplate)
|
|
if err != nil {
|
|
return "", fmt.Errorf("parsing output-file template %q: %w", outputFileTemplate, err)
|
|
}
|
|
|
|
buf := &bytes.Buffer{}
|
|
|
|
type state struct {
|
|
BaseName string
|
|
Path string
|
|
AbsPath string
|
|
AbsPathSHA1 string
|
|
}
|
|
|
|
data := struct {
|
|
State state
|
|
Release *ReleaseSpec
|
|
Environment *environment.Environment
|
|
}{
|
|
State: state{
|
|
BaseName: stateFileName,
|
|
Path: st.FilePath,
|
|
AbsPath: stateAbsPath,
|
|
AbsPathSHA1: sha1sum,
|
|
},
|
|
Release: release,
|
|
Environment: &st.Env,
|
|
}
|
|
|
|
if err := t.Execute(buf, data); err != nil {
|
|
return "", fmt.Errorf("executing output-file template: %w", err)
|
|
}
|
|
|
|
return buf.String(), nil
|
|
}
|
|
|
|
func (st *HelmState) ToYaml() (string, error) {
|
|
if result, err := yaml.Marshal(st); err != nil {
|
|
return "", err
|
|
} else {
|
|
return string(result), nil
|
|
}
|
|
}
|
|
|
|
func (st *HelmState) LoadYAMLForEmbedding(release *ReleaseSpec, entries []any, missingFileHandler *string, pathPrefix string) ([]any, error) {
|
|
var result []any
|
|
|
|
for _, v := range entries {
|
|
switch t := v.(type) {
|
|
case string:
|
|
var values map[string]any
|
|
|
|
paths, skip, err := st.storage().resolveFile(missingFileHandler, "values", pathPrefix+t, st.MissingFileHandlerConfig.resolveFileOptions()...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if skip {
|
|
continue
|
|
}
|
|
|
|
if len(paths) > 1 {
|
|
return nil, fmt.Errorf("glob patterns in release values and secrets is not supported yet. please submit a feature request if necessary")
|
|
}
|
|
yamlOrTemplatePath := paths[0]
|
|
|
|
yamlBytes, err := st.RenderReleaseValuesFileToBytes(release, yamlOrTemplatePath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to render values files \"%s\": %v", t, err)
|
|
}
|
|
|
|
if err := yaml.Unmarshal(yamlBytes, &values); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
result = append(result, values)
|
|
default:
|
|
result = append(result, v)
|
|
}
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
func (st *HelmState) Reverse() {
|
|
for i, j := 0, len(st.Releases)-1; i < j; i, j = i+1, j-1 {
|
|
st.Releases[i], st.Releases[j] = st.Releases[j], st.Releases[i]
|
|
}
|
|
|
|
for i, j := 0, len(st.Helmfiles)-1; i < j; i, j = i+1, j-1 {
|
|
st.Helmfiles[i], st.Helmfiles[j] = st.Helmfiles[j], st.Helmfiles[i]
|
|
}
|
|
}
|
|
|
|
// Chart cache for both OCI and non-OCI charts to avoid duplicate downloads
|
|
type ChartCacheKey struct {
|
|
Chart string
|
|
Version string
|
|
}
|
|
|
|
var downloadedCharts = make(map[ChartCacheKey]string) // key -> chart path
|
|
var downloadedChartsMutex sync.RWMutex
|
|
|
|
// getChartCacheKey creates a cache key for a chart and version
|
|
func (st *HelmState) getChartCacheKey(release *ReleaseSpec) ChartCacheKey {
|
|
version := release.Version
|
|
if version == "" {
|
|
// Use empty string for latest version
|
|
version = ""
|
|
}
|
|
return ChartCacheKey{
|
|
Chart: release.Chart,
|
|
Version: version,
|
|
}
|
|
}
|
|
|
|
// checkChartCache checks if a chart is already downloaded and returns its path
|
|
func (st *HelmState) checkChartCache(key ChartCacheKey) (string, bool) {
|
|
downloadedChartsMutex.RLock()
|
|
defer downloadedChartsMutex.RUnlock()
|
|
path, exists := downloadedCharts[key]
|
|
return path, exists
|
|
}
|
|
|
|
// addToChartCache adds a chart to the cache
|
|
func (st *HelmState) addToChartCache(key ChartCacheKey, path string) {
|
|
downloadedChartsMutex.Lock()
|
|
defer downloadedChartsMutex.Unlock()
|
|
downloadedCharts[key] = path
|
|
}
|
|
|
|
// resetChartCacheForTest clears the global chart cache. For testing only.
|
|
func resetChartCacheForTest() {
|
|
downloadedChartsMutex.Lock()
|
|
defer downloadedChartsMutex.Unlock()
|
|
downloadedCharts = make(map[ChartCacheKey]string)
|
|
}
|
|
|
|
// isSharedCachePath returns true if the chartPath is within the shared cache directory.
|
|
// Charts in the shared cache should not be deleted during refresh to prevent race conditions
|
|
// when multiple processes are using the same cached chart.
|
|
func (st *HelmState) isSharedCachePath(chartPath string) bool {
|
|
sharedCacheDir := remote.CacheDir()
|
|
absChartPath, err := filepath.Abs(chartPath)
|
|
if err != nil {
|
|
st.logger.Debugf("failed to get absolute path for chartPath %q: %v", chartPath, err)
|
|
return false
|
|
}
|
|
absSharedCache, err := filepath.Abs(sharedCacheDir)
|
|
if err != nil {
|
|
st.logger.Debugf("failed to get absolute path for shared cache dir %q: %v", sharedCacheDir, err)
|
|
return false
|
|
}
|
|
resolvedChartPath, err := filepath.EvalSymlinks(absChartPath)
|
|
if err != nil {
|
|
resolvedChartPath = absChartPath
|
|
}
|
|
resolvedSharedCache, err := filepath.EvalSymlinks(absSharedCache)
|
|
if err != nil {
|
|
resolvedSharedCache = absSharedCache
|
|
}
|
|
return resolvedChartPath == resolvedSharedCache ||
|
|
strings.HasPrefix(resolvedChartPath, resolvedSharedCache+string(filepath.Separator))
|
|
}
|
|
|
|
// chartDownloadAction represents what action should be taken after acquiring locks
|
|
type chartDownloadAction int
|
|
|
|
const (
|
|
chartActionUseCached chartDownloadAction = iota // Use the cached chart
|
|
chartActionDownload // Download the chart
|
|
chartActionRefresh // Delete and re-download (refresh)
|
|
)
|
|
|
|
// chartLockResult contains the result of acquiring locks and checking cache.
|
|
// It supports both shared (read) and exclusive (write) locks using reader-writer semantics.
|
|
// Multiple processes can hold shared locks simultaneously, but exclusive locks block all others.
|
|
type chartLockResult struct {
|
|
action chartDownloadAction
|
|
cachedPath string // Path to use if action is chartActionUseCached
|
|
fileLock *flock.Flock // File lock for cross-process synchronization
|
|
inProcessMutex *sync.RWMutex // In-process RWMutex for goroutine coordination
|
|
isExclusive bool // True if holding exclusive lock, false if shared
|
|
chartPath string // The chart path this lock is for (used for logging)
|
|
}
|
|
|
|
// Release releases all acquired locks (both file and in-process).
|
|
// This method is safe to call even if locks were not acquired.
|
|
func (r *chartLockResult) Release(logger *zap.SugaredLogger) {
|
|
if r == nil {
|
|
return
|
|
}
|
|
if r.inProcessMutex != nil {
|
|
if r.isExclusive {
|
|
r.inProcessMutex.Unlock()
|
|
} else {
|
|
r.inProcessMutex.RUnlock()
|
|
}
|
|
}
|
|
if r.fileLock != nil {
|
|
if err := r.fileLock.Unlock(); err != nil && logger != nil {
|
|
logger.Warnf("Failed to release file lock for %s: %v", r.chartPath, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// acquireChartLock acquires file and in-process locks for chart download coordination.
|
|
// - Shared (read) lock: Used when chart already exists in cache
|
|
// - Exclusive (write) lock: Used when downloading or refreshing a chart
|
|
//
|
|
// The optional skipRefreshCheck callback, if provided, is called before deleting an existing
|
|
// chart for refresh. If it returns true, the existing chart is used instead of being deleted.
|
|
// This allows callers to check in-memory caches to prevent redundant re-downloads within a process.
|
|
// NOTE: The callback is executed while holding an exclusive lock, so it should be fast and non-blocking.
|
|
//
|
|
// IMPORTANT: Locks are released immediately after download completes.
|
|
// For process-specific temp directories, cleanup is deferred until after helm operations.
|
|
// For the shared cache directory, refresh is skipped entirely to prevent race conditions
|
|
// between processes (use `helmfile cache cleanup` to force refresh).
|
|
func (st *HelmState) acquireChartLock(chartPath string, opts ChartPrepareOptions, skipRefreshCheck func() bool) (*chartLockResult, error) {
|
|
result := &chartLockResult{
|
|
chartPath: chartPath,
|
|
}
|
|
|
|
// Ensure lock directory exists
|
|
lockFilePath := chartPath + ".lock"
|
|
lockFileDir := filepath.Dir(lockFilePath)
|
|
if err := os.MkdirAll(lockFileDir, 0755); err != nil {
|
|
return nil, fmt.Errorf("failed to create lock directory %s: %w", lockFileDir, err)
|
|
}
|
|
result.fileLock = flock.New(lockFilePath)
|
|
|
|
needsRefresh := !opts.SkipDeps && !opts.SkipRefresh
|
|
|
|
// Optimistic path: Try shared lock first if cache likely exists and no refresh needed
|
|
if st.fs.DirectoryExistsAt(chartPath) && !needsRefresh {
|
|
if err := st.acquireSharedLock(result, chartPath); err != nil {
|
|
// Failed to acquire shared lock, will try exclusive below
|
|
st.logger.Debugf("Failed to acquire shared lock for %s: %v, trying exclusive", chartPath, err)
|
|
} else {
|
|
// Got shared lock, validate cache
|
|
if fullChartPath, err := findChartDirectory(chartPath); err == nil {
|
|
result.action = chartActionUseCached
|
|
result.cachedPath = filepath.Dir(fullChartPath)
|
|
st.logger.Debugf("Using cached chart at %s (shared lock)", result.cachedPath)
|
|
return result, nil
|
|
}
|
|
// Invalid cache - release shared lock and acquire exclusive
|
|
st.logger.Debugf("Chart cache at %s is invalid, upgrading to exclusive lock", chartPath)
|
|
result.Release(st.logger)
|
|
// Reinitialize fileLock after releasing shared lock for exclusive lock acquisition
|
|
result.fileLock = flock.New(lockFilePath)
|
|
}
|
|
}
|
|
|
|
// Need exclusive lock for: download, refresh, or cache validation failure
|
|
if err := st.acquireExclusiveLock(result, chartPath); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Double-check after acquiring exclusive lock (another process may have populated cache)
|
|
if st.fs.DirectoryExistsAt(chartPath) {
|
|
fullChartPath, err := findChartDirectory(chartPath)
|
|
if err == nil {
|
|
// Valid cache exists now
|
|
if !needsRefresh {
|
|
// Another process populated the cache while we waited for lock
|
|
result.action = chartActionUseCached
|
|
result.cachedPath = filepath.Dir(fullChartPath)
|
|
st.logger.Debugf("Using cached chart at %s (populated by another process)", result.cachedPath)
|
|
return result, nil
|
|
}
|
|
// Refresh mode requested - check if we should skip (e.g., in-memory cache populated by another worker)
|
|
if skipRefreshCheck != nil && skipRefreshCheck() {
|
|
result.action = chartActionUseCached
|
|
result.cachedPath = filepath.Dir(fullChartPath)
|
|
st.logger.Debugf("Skipping refresh for chart at %s (another worker already cached)", result.cachedPath)
|
|
return result, nil
|
|
}
|
|
// Skip refresh for shared cache paths to prevent race conditions when multiple
|
|
// processes are using the same cached chart. Use `helmfile cache cleanup` to force refresh.
|
|
if st.isSharedCachePath(chartPath) {
|
|
st.logger.Infof("Skipping refresh for chart at %s (shared cache, another process may be using it; run 'helmfile cache cleanup' to force refresh)", chartPath)
|
|
result.action = chartActionUseCached
|
|
result.cachedPath = filepath.Dir(fullChartPath)
|
|
return result, nil
|
|
}
|
|
// Proceed with refresh: delete and re-download
|
|
st.logger.Debugf("Refreshing chart at %s (exclusive lock)", chartPath)
|
|
if err := os.RemoveAll(chartPath); err != nil {
|
|
result.Release(st.logger)
|
|
return nil, fmt.Errorf("failed to remove chart directory for refresh: %w", err)
|
|
}
|
|
result.action = chartActionRefresh
|
|
} else {
|
|
// Directory exists but invalid (no Chart.yaml) - corrupted cache
|
|
// Skip deletion for shared cache paths to prevent race conditions
|
|
if st.isSharedCachePath(chartPath) {
|
|
result.Release(st.logger)
|
|
return nil, fmt.Errorf("chart directory at %s is corrupted (%w); cannot safely remove from shared cache while other processes may be running. Please run 'helmfile cache cleanup' to clear the cache", chartPath, err)
|
|
} else {
|
|
st.logger.Debugf("Chart directory at %s is corrupted: %v, will re-download", chartPath, err)
|
|
if err := os.RemoveAll(chartPath); err != nil {
|
|
result.Release(st.logger)
|
|
return nil, fmt.Errorf("failed to remove corrupted chart directory: %w", err)
|
|
}
|
|
result.action = chartActionDownload
|
|
}
|
|
}
|
|
} else {
|
|
result.action = chartActionDownload
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
// acquireSharedLock acquires a shared (read) lock for concurrent read access.
|
|
// Multiple processes can hold shared locks simultaneously, but shared locks
|
|
// block exclusive lock acquisition.
|
|
// Lock order: in-process mutex first, then file lock (to prevent deadlock).
|
|
func (st *HelmState) acquireSharedLock(result *chartLockResult, chartPath string) error {
|
|
const lockTimeout = 30 * time.Second
|
|
|
|
// Acquire in-process RLock FIRST (consistent ordering prevents deadlock)
|
|
result.inProcessMutex = st.getNamedRWMutex(chartPath)
|
|
result.inProcessMutex.RLock()
|
|
|
|
ctx, cancel := gocontext.WithTimeout(gocontext.Background(), lockTimeout)
|
|
defer cancel()
|
|
|
|
locked, err := result.fileLock.TryRLockContext(ctx, 500*time.Millisecond)
|
|
if err != nil {
|
|
result.inProcessMutex.RUnlock()
|
|
return fmt.Errorf("failed to acquire shared file lock for chart %s: %w", chartPath, err)
|
|
}
|
|
if !locked {
|
|
result.inProcessMutex.RUnlock()
|
|
return fmt.Errorf("timeout waiting for shared file lock on chart %s", chartPath)
|
|
}
|
|
|
|
result.isExclusive = false
|
|
return nil
|
|
}
|
|
|
|
// acquireExclusiveLock acquires an exclusive (write) lock with retry logic.
|
|
// Exclusive locks block all other locks (both shared and exclusive).
|
|
// Lock order: in-process mutex first, then file lock (to prevent deadlock).
|
|
func (st *HelmState) acquireExclusiveLock(result *chartLockResult, chartPath string) error {
|
|
const (
|
|
lockTimeout = 30 * time.Second
|
|
maxRetries = 3
|
|
retryBackoff = 1 * time.Second
|
|
)
|
|
|
|
// Acquire in-process Lock FIRST (consistent ordering prevents deadlock)
|
|
result.inProcessMutex = st.getNamedRWMutex(chartPath)
|
|
result.inProcessMutex.Lock()
|
|
|
|
var locked bool
|
|
var lockErr error
|
|
|
|
for attempt := 1; attempt <= maxRetries; attempt++ {
|
|
ctx, cancel := gocontext.WithTimeout(gocontext.Background(), lockTimeout)
|
|
locked, lockErr = result.fileLock.TryLockContext(ctx, 500*time.Millisecond)
|
|
cancel()
|
|
|
|
if locked {
|
|
break
|
|
}
|
|
if lockErr != nil {
|
|
// Actual file system error (not timeout) - worth retrying
|
|
st.logger.Warnf("Failed to acquire exclusive file lock (attempt %d/%d): %v", attempt, maxRetries, lockErr)
|
|
if attempt < maxRetries {
|
|
// Release in-process mutex during backoff to avoid blocking other goroutines
|
|
result.inProcessMutex.Unlock()
|
|
time.Sleep(retryBackoff)
|
|
result.inProcessMutex.Lock()
|
|
}
|
|
} else {
|
|
// Timeout (flock returns false, nil when context deadline exceeded)
|
|
// Don't retry on timeout - 30s is already a long wait, another process likely holds the lock
|
|
result.inProcessMutex.Unlock()
|
|
return fmt.Errorf("timeout waiting for exclusive file lock on chart %s after %v", chartPath, lockTimeout)
|
|
}
|
|
}
|
|
|
|
if !locked {
|
|
result.inProcessMutex.Unlock()
|
|
return fmt.Errorf("failed to acquire exclusive file lock for chart %s after %d attempts: %w", chartPath, maxRetries, lockErr)
|
|
}
|
|
|
|
result.isExclusive = true
|
|
return nil
|
|
}
|
|
|
|
// getOCIChart downloads or retrieves an OCI chart from cache.
|
|
// Locks are acquired during download and released immediately after.
|
|
// A per-chart+version mutex serializes downloads within the process so that
|
|
// concurrent releases using the same OCI chart don't race (issue #768).
|
|
func (st *HelmState) getOCIChart(release *ReleaseSpec, tempDir string, helm helmexec.Interface, opts ChartPrepareOptions) (*string, error) {
|
|
qualifiedChartName, chartName, chartVersion, err := st.getOCIQualifiedChartName(release)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if qualifiedChartName == "" {
|
|
return nil, nil
|
|
}
|
|
|
|
cacheKey := st.getChartCacheKey(release)
|
|
|
|
// Fast path: check in-process cache without acquiring any lock.
|
|
if cachedPath, exists := st.checkChartCache(cacheKey); exists && st.fs.DirectoryExistsAt(cachedPath) {
|
|
st.logger.Debugf("OCI chart %s:%s already downloaded, using cached version at %s", chartName, chartVersion, cachedPath)
|
|
return &cachedPath, nil
|
|
}
|
|
|
|
// Serialize downloads per chart+version within this process (issue #768).
|
|
downloadMu := st.getNamedRWMutex("chart-download:" + cacheKey.Chart + ":" + cacheKey.Version)
|
|
downloadMu.Lock()
|
|
defer downloadMu.Unlock()
|
|
|
|
// Double-check: another worker may have completed the download while we waited.
|
|
if cachedPath, exists := st.checkChartCache(cacheKey); exists && st.fs.DirectoryExistsAt(cachedPath) {
|
|
st.logger.Debugf("OCI chart %s:%s downloaded by another worker, using cached version at %s", chartName, chartVersion, cachedPath)
|
|
return &cachedPath, nil
|
|
}
|
|
|
|
if opts.OutputDirTemplate == "" {
|
|
tempDir = remote.CacheDir()
|
|
}
|
|
|
|
chartPath, _ := st.getOCIChartPath(tempDir, release, chartName, chartVersion, opts.OutputDirTemplate)
|
|
|
|
// Acquire locks and determine action
|
|
// Pass a callback that checks if another worker in this process has already cached the chart.
|
|
// This prevents deleting and re-downloading a chart that was just downloaded by another worker.
|
|
lockResult, err := st.acquireChartLock(chartPath, opts, func() bool {
|
|
_, exists := st.checkChartCache(cacheKey)
|
|
return exists
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// If cached on filesystem, release lock and return
|
|
if lockResult.action == chartActionUseCached {
|
|
st.addToChartCache(cacheKey, lockResult.cachedPath)
|
|
lockResult.Release(st.logger)
|
|
return &lockResult.cachedPath, nil
|
|
}
|
|
|
|
// Ensure temp directory exists
|
|
if _, err := os.Stat(tempDir); err != nil {
|
|
if err := os.MkdirAll(tempDir, 0755); err != nil {
|
|
lockResult.Release(st.logger)
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// Download the chart
|
|
flags := st.chartOCIFlags(release)
|
|
flags = st.appendVerifyFlags(flags, release)
|
|
flags = st.appendKeyringFlags(flags, release)
|
|
flags = st.appendChartDownloadFlags(flags, release)
|
|
// Use the clean chartVersion (without digest) from getOCIQualifiedChartName
|
|
// rather than appendChartVersionFlags which uses release.Version verbatim.
|
|
// The digest is already embedded in qualifiedChartName.
|
|
// When a digest is present, omit --version: the digest is the authoritative
|
|
// content identifier, and passing both causes errors in some Helm versions.
|
|
if chartVersion != "" && !strings.Contains(qualifiedChartName, "@") {
|
|
flags = append(flags, "--version", chartVersion)
|
|
}
|
|
if st.isDevelopment(release) {
|
|
flags = append(flags, "--devel")
|
|
}
|
|
|
|
if err := helm.ChartPull(qualifiedChartName, chartPath, flags...); err != nil {
|
|
lockResult.Release(st.logger)
|
|
return nil, err
|
|
}
|
|
|
|
if err := helm.ChartExport(qualifiedChartName, chartPath); err != nil {
|
|
lockResult.Release(st.logger)
|
|
return nil, err
|
|
}
|
|
|
|
fullChartPath, err := findChartDirectory(chartPath)
|
|
if err != nil {
|
|
lockResult.Release(st.logger)
|
|
return nil, err
|
|
}
|
|
|
|
chartPath = filepath.Dir(fullChartPath)
|
|
|
|
// Add to global chart cache
|
|
st.addToChartCache(cacheKey, chartPath)
|
|
|
|
// Release lock immediately after download - the tempDir cleanup is deferred
|
|
// until after helm operations complete, so the chart won't be deleted mid-use.
|
|
lockResult.Release(st.logger)
|
|
|
|
return &chartPath, nil
|
|
}
|
|
|
|
// IsOCIChart returns true if the chart is an OCI chart
|
|
func (st *HelmState) IsOCIChart(chart string) bool {
|
|
if strings.HasPrefix(chart, "oci://") {
|
|
return true
|
|
}
|
|
|
|
repo, _ := st.GetRepositoryAndNameFromChartName(chart)
|
|
if repo == nil {
|
|
return false
|
|
}
|
|
return repo.OCI
|
|
}
|
|
|
|
// NeedsRepoUpdate returns true if there are any repositories that require `helm repo update`.
|
|
// OCI repositories don't need `helm repo update` as they use `helm registry login` instead.
|
|
func (st *HelmState) NeedsRepoUpdate() bool {
|
|
for _, repo := range st.Repositories {
|
|
if !repo.OCI {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// parseOCIChartRef parses an OCI chart URL into base URL, version tag, and digest.
|
|
// Examples:
|
|
//
|
|
// oci://registry/chart → (oci://registry/chart, "", "")
|
|
// oci://registry/chart:2.0.0 → (oci://registry/chart, "2.0.0", "")
|
|
// oci://registry/chart@sha256:a → (oci://registry/chart, "", "sha256:a")
|
|
// oci://registry/chart:2.0@sha256:a → (oci://registry/chart, "2.0", "sha256:a")
|
|
// oci://reg:5000/chart:1.0@sha256:a → (oci://reg:5000/chart, "1.0", "sha256:a")
|
|
func parseOCIChartRef(chartURL string) (baseURL, version, digest string) {
|
|
// Split off digest first (everything after @)
|
|
if atIdx := strings.Index(chartURL, "@"); atIdx >= 0 {
|
|
digest = chartURL[atIdx+1:]
|
|
chartURL = chartURL[:atIdx]
|
|
}
|
|
|
|
// Find version tag: last ":" that comes after the last "/"
|
|
lastSlash := strings.LastIndex(chartURL, "/")
|
|
lastColon := strings.LastIndex(chartURL, ":")
|
|
if lastColon > lastSlash {
|
|
version = chartURL[lastColon+1:]
|
|
baseURL = chartURL[:lastColon]
|
|
} else {
|
|
baseURL = chartURL
|
|
}
|
|
|
|
return baseURL, version, digest
|
|
}
|
|
|
|
// parseVersionDigest splits a version string that may contain an OCI digest.
|
|
// Examples:
|
|
//
|
|
// "2.0.0" → ("2.0.0", "")
|
|
// "2.0.0@sha256:abc" → ("2.0.0", "sha256:abc")
|
|
// "@sha256:abc" → ("", "sha256:abc")
|
|
// "" → ("", "")
|
|
func parseVersionDigest(version string) (ver, digest string) {
|
|
if atIdx := strings.Index(version, "@"); atIdx >= 0 {
|
|
return version[:atIdx], version[atIdx+1:]
|
|
}
|
|
return version, ""
|
|
}
|
|
|
|
func (st *HelmState) getOCIQualifiedChartName(release *ReleaseSpec) (string, string, string, error) {
|
|
// For issue #2247: Don't default to "latest" - use empty string to let Helm pull the latest version
|
|
// Only use the version explicitly provided by the user
|
|
chartVersion := release.Version
|
|
|
|
// In development mode with no version, omit version flag so --devel works correctly
|
|
if st.isDevelopment(release) && release.Version == "" {
|
|
chartVersion = ""
|
|
}
|
|
|
|
if !st.IsOCIChart(release.Chart) {
|
|
return "", "", chartVersion, nil
|
|
}
|
|
|
|
// Reject explicit "latest" for OCI charts (issue #1047, #2247)
|
|
// This only applies if user explicitly specified "latest", not when version is omitted
|
|
// We reject for all Helm versions to ensure consistent behavior
|
|
// Strip any digest suffix before checking (e.g. "latest@sha256:..." is still invalid)
|
|
versionForCheck, _ := parseVersionDigest(release.Version)
|
|
if versionForCheck == "latest" {
|
|
return "", "", "", fmt.Errorf("the version for OCI charts should be semver compliant, the latest tag is not supported")
|
|
}
|
|
|
|
var qualifiedChartName, chartName string
|
|
if strings.HasPrefix(release.Chart, "oci://") {
|
|
// Parse version and digest from the chart URL
|
|
baseURL, versionInURL, digestInURL := parseOCIChartRef(release.Chart)
|
|
|
|
// Parse version and digest from the version field
|
|
versionInField, digestInField := parseVersionDigest(chartVersion)
|
|
|
|
// Merge: version field takes precedence; fall back to URL-embedded version
|
|
finalVersion := versionInField
|
|
if finalVersion == "" && versionInURL != "" {
|
|
finalVersion = versionInURL
|
|
}
|
|
|
|
// Merge: URL-embedded digest takes precedence; fall back to version field digest
|
|
finalDigest := digestInURL
|
|
if finalDigest == "" {
|
|
finalDigest = digestInField
|
|
}
|
|
|
|
// Extract chart name from base URL (last path segment)
|
|
parts := strings.Split(baseURL, "/")
|
|
chartName = parts[len(parts)-1]
|
|
|
|
// Build qualifiedChartName: base (without oci:// prefix) + digest or version
|
|
base := strings.TrimPrefix(baseURL, "oci://")
|
|
if finalDigest != "" {
|
|
// Digest present — put it in the URL; version goes through --version flag only
|
|
qualifiedChartName = fmt.Sprintf("%s@%s", base, finalDigest)
|
|
} else if versionInURL == "" && finalVersion != "" {
|
|
// Version from field only (backward compatible format)
|
|
qualifiedChartName = fmt.Sprintf("%s:%s", base, finalVersion)
|
|
} else {
|
|
// Version came from URL (handled via --version flag) or no version at all
|
|
qualifiedChartName = base
|
|
}
|
|
|
|
chartVersion = finalVersion
|
|
} else {
|
|
var repo *RepositorySpec
|
|
repo, chartName = st.GetRepositoryAndNameFromChartName(release.Chart)
|
|
|
|
// Handle digest in version field for repo-aliased OCI charts too
|
|
base := fmt.Sprintf("%s/%s", repo.URL, chartName)
|
|
finalVersion, digest := parseVersionDigest(chartVersion)
|
|
|
|
switch {
|
|
case digest != "":
|
|
qualifiedChartName = fmt.Sprintf("%s@%s", base, digest)
|
|
case finalVersion != "":
|
|
qualifiedChartName = fmt.Sprintf("%s:%s", base, finalVersion)
|
|
default:
|
|
qualifiedChartName = base
|
|
}
|
|
|
|
chartVersion = finalVersion
|
|
}
|
|
|
|
return qualifiedChartName, chartName, chartVersion, nil
|
|
}
|
|
|
|
func (st *HelmState) FullFilePath() (string, error) {
|
|
var wd string
|
|
var err error
|
|
if st.fs != nil {
|
|
wd, err = st.fs.Getwd()
|
|
}
|
|
// When FilePath already contains basePath as a prefix (e.g. when loaded
|
|
// with a baseDir), avoid double-counting basePath in the result.
|
|
if st.basePath != "" && strings.HasPrefix(st.FilePath, st.basePath+string(filepath.Separator)) {
|
|
return filepath.Join(wd, st.FilePath), err
|
|
}
|
|
return filepath.Join(wd, st.basePath, st.FilePath), err
|
|
}
|
|
|
|
func (st *HelmState) getOCIChartPath(tempDir string, release *ReleaseSpec, chartName, chartVersion, outputDirTemplate string) (string, error) {
|
|
if outputDirTemplate != "" {
|
|
return st.generateChartPath(chartName, tempDir, release, outputDirTemplate)
|
|
}
|
|
|
|
pathElems := []string{tempDir}
|
|
|
|
replacer := strings.NewReplacer(
|
|
":", "_",
|
|
"//", "_",
|
|
".", "_",
|
|
"&", "_",
|
|
)
|
|
qName := strings.Split(replacer.Replace(release.Chart), "/")
|
|
|
|
pathElems = append(pathElems, qName...)
|
|
pathElems = append(pathElems, safeVersionPath(chartVersion))
|
|
return filepath.Join(pathElems...), nil
|
|
}
|