helmfile/pkg/state/chart_dependency.go

package state

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/Masterminds/semver/v3"
	"go.uber.org/zap"

	"github.com/helmfile/helmfile/pkg/app/version"
	"github.com/helmfile/helmfile/pkg/helmexec"
	"github.com/helmfile/helmfile/pkg/yaml"
)

type ChartMeta struct {
	Name string `yaml:"name"`
}

// unresolvedChartDependency represents a dependency that is to be resolved.
//
// Helmfile generates Helm Chart.yaml containing unresolved dependencies, and runs `helm dependency update` to produce Helm Chart.lock
// which becomes helmfile.lock, which is then used to resolve the dependencies.
type unresolvedChartDependency struct {
	// ChartName identifies the dependant chart. In Helmfile, ChartName for `chart: stable/envoy` would be just `envoy`.
	// It can't be collided with other charts referenced in the same helmfile spec.
	// That is, collocating `chart: incubator/foo` and `chart: stable/foo` isn't allowed. Name them differently for a work-around.
	ChartName string `yaml:"name"`
	// Repository contains the URL for the helm chart repository that hosts the chart identified by ChartName
	Repository string `yaml:"repository"`
	// VersionConstraint is the version constraint of the dependent chart. "*" means the latest version.
	VersionConstraint string `yaml:"version"`
	// Alias differentiates multiple dependencies with the same ChartName.
	// Despite its name, and its optional in Helm's Chart.yaml, we use this as a unique identifier for the dependency.
	// So, every dependency have an alias, even if it's not explicitly set in the helmfile.
	Alias string `yaml:"alias"`
}

type ResolvedChartDependency struct {
	// ChartName identifies the dependant chart. In Helmfile, ChartName for `chart: stable/envoy` would be just `envoy`.
	// It can't be collided with other charts referenced in the same helmfile spec.
	// That is, collocating `chart: incubator/foo` and `chart: stable/foo` isn't allowed. Name them differently for a work-around.
	ChartName string `yaml:"name"`
	// Repository contains the URL for the helm chart repository that hosts the chart identified by ChartName
	Repository string `yaml:"repository"`
	// Version is the version number of the dependent chart.
	// In the context of helmfile this can be omitted. When omitted, it is considered `*` which results helm/helmfile fetching the latest version.
	Version string `yaml:"version"`
}

type UnresolvedDependencies struct {
	deps map[string][]unresolvedChartDependency
}

type ChartRequirements struct {
	UnresolvedDependencies []unresolvedChartDependency `yaml:"dependencies"`
}

type ChartLockedRequirements struct {
	Version              string                    `yaml:"version"`
	ResolvedDependencies []ResolvedChartDependency `yaml:"dependencies"`
	Digest               string                    `yaml:"digest"`
	Generated            string                    `yaml:"generated"`
}

func (d *UnresolvedDependencies) Add(chart, url, versionConstraint, alias string) {
	d.deps[chart] = append(d.deps[chart], unresolvedChartDependency{
		ChartName:         chart,
		Repository:        url,
		VersionConstraint: versionConstraint,
		Alias:             alias,
	})
}

func (d *UnresolvedDependencies) ToChartRequirements() *ChartRequirements {
	deps := []unresolvedChartDependency{}

	for _, ds := range d.deps {
		for _, d := range ds {
			if d.VersionConstraint == "" {
				d.VersionConstraint = "*"
			}
			deps = append(deps, d)
		}
	}

	return &ChartRequirements{UnresolvedDependencies: deps}
}

type ResolvedDependencies struct {
	deps map[string][]ResolvedChartDependency
}

// nolint: unparam
func (d *ResolvedDependencies) add(dep ResolvedChartDependency) error {
	deps := d.deps[dep.ChartName]
	if deps == nil {
		deps = []ResolvedChartDependency{dep}
	} else {
		deps = append(deps, dep)
	}
	d.deps[dep.ChartName] = deps
	return nil
}

func (d *ResolvedDependencies) Get(chart, versionConstraint string) (string, error) {
	if versionConstraint == "" {
		versionConstraint = "*"
	}

	deps, exists := d.deps[chart]
	if exists {
		for _, dep := range deps {
			constraint, err := semver.NewConstraint(versionConstraint)
			if err != nil {
				return "", err
			}
			version, err := semver.NewVersion(dep.Version)
			if err != nil {
				return "", err
			}
			if constraint.Check(version) {
				return dep.Version, nil
			}
		}
	}
	return "", fmt.Errorf("no resolved dependency found for \"%s\", running \"helmfile deps\" may resolve the issue", chart)
}

func dedupResolvedDependencies(deps []ResolvedChartDependency) []ResolvedChartDependency {
	seen := map[string]bool{}
	result := make([]ResolvedChartDependency, 0, len(deps))
	for _, dep := range deps {
		key := dep.ChartName + "|" + dep.Repository + "|" + dep.Version
		if seen[key] {
			continue
		}
		seen[key] = true
		result = append(result, dep)
	}
	return result
}

func (st *HelmState) mergeLockedDependencies() (*HelmState, error) {
	filename, unresolved := getUnresolvedDependenciess(st)

	if len(unresolved.deps) == 0 {
		return st, nil
	}

	lockFile := st.LockFile
	// When basePath is set (e.g. when loaded with baseDir instead of os.Chdir),
	// resolve the lock file path relative to basePath so it can be found
	// without changing the working directory.
	switch {
	case lockFile != "":
		if st.basePath != "" && !filepath.IsAbs(lockFile) {
			lockFile = filepath.Join(st.basePath, lockFile)
		}
	case st.basePath != "":
		// When no custom lockfile is specified, use the default lockfile name
		// joined with basePath to ensure it's found when not changing CWD.
		lockFile = filepath.Join(st.basePath, filename+".lock")
	}

	depMan := NewChartDependencyManager(filename, st.logger, lockFile)

	if st.fs.ReadFile != nil {
		depMan.readFile = st.fs.ReadFile
	}

	return resolveDependencies(st, depMan, unresolved)
}

func resolveDependencies(st *HelmState, depMan *chartDependencyManager, unresolved *UnresolvedDependencies) (*HelmState, error) {
	resolved, lockfileExists, err := depMan.Resolve(unresolved)
	if err != nil {
		return nil, fmt.Errorf("unable to resolve %d deps: %v", len(unresolved.deps), err)
	}
	if !lockfileExists {
		return st, nil
	}

	repoToURL := map[string]string{}

	for _, r := range st.Repositories {
		repoToURL[r.Name] = r.URL
	}

	updated := *st
	for i, r := range updated.Releases {
		repo, chart, ok := resolveRemoteChart(r.Chart)
		if !ok {
			continue
		}

		_, ok = repoToURL[repo]
		// Skip this chart from dependency management, as there's no matching `repository` in the helmfile state,
		// which may imply that this is a local chart within a directory, like `charts/myapp`
		if !ok {
			continue
		}

		ver, err := resolved.Get(chart, r.Version)
		if err != nil {
			return nil, err
		}

		updated.Releases[i].Version = ver
	}

	return &updated, nil
}

func (st *HelmState) updateDependenciesInTempDir(shell helmexec.DependencyUpdater, tempDir func(string, string) (string, error)) (*HelmState, error) {
	filename, unresolved := getUnresolvedDependenciess(st)

	if len(unresolved.deps) == 0 {
		st.logger.Warnf("There are no repositories defined in your helmfile.yaml.\nThis means helmfile cannot update your dependencies or create a lock file.\nSee https://github.com/roboll/helmfile/issues/878 for more information.")
		return st, nil
	}

	d, err := tempDir("", "")
	if err != nil {
		return nil, fmt.Errorf("unable to create dir: %v", err)
	}
	defer func() {
		_ = os.RemoveAll(d)
	}()

	return updateDependencies(st, shell, unresolved, filename, d)
}

// aliasNameFormat = regexp.MustCompile("^[a-zA-Z0-9_-]+$") from helm code
func chartDependenciesAlias(namespace, releaseName string) string {
	return fmt.Sprintf("%s-%s", namespace, releaseName)
}

func getUnresolvedDependenciess(st *HelmState) (string, *UnresolvedDependencies) {
	repoToURL := map[string]RepositorySpec{}

	for _, r := range st.Repositories {
		repoToURL[r.Name] = r
	}

	unresolved := &UnresolvedDependencies{deps: map[string][]unresolvedChartDependency{}}

	for _, r := range st.Releases {
		repo, chart, ok := resolveRemoteChart(r.Chart)
		if !ok {
			continue
		}

		repoSpec, ok := repoToURL[repo]
		// Skip this chart from dependency management, as there's no matching `repository` in the helmfile state,
		// which may imply that this is a local chart within a directory, like `charts/myapp`
		if !ok {
			continue
		}

		url := repoSpec.URL

		if repoSpec.OCI {
			url = fmt.Sprintf("oci://%s", url)
		}

		unresolved.Add(chart, url, r.Version, chartDependenciesAlias(r.Namespace, r.Name))
	}

	filename := filepath.Base(st.FilePath)
	filename = strings.TrimSuffix(filename, ".gotmpl")
	filename = strings.TrimSuffix(filename, ".yaml")
	filename = strings.TrimSuffix(filename, ".yml")

	return filename, unresolved
}

func updateDependencies(st *HelmState, shell helmexec.DependencyUpdater, unresolved *UnresolvedDependencies, filename, wd string) (*HelmState, error) {
	lockFile := st.LockFile
	switch {
	case lockFile != "":
		if st.basePath != "" && !filepath.IsAbs(lockFile) {
			lockFile = filepath.Join(st.basePath, lockFile)
		}
	case st.basePath != "":
		lockFile = filepath.Join(st.basePath, filename+".lock")
	}

	depMan := NewChartDependencyManager(filename, st.logger, lockFile)

	_, err := depMan.Update(shell, wd, unresolved)
	if err != nil {
		return nil, fmt.Errorf("unable to update %d deps: %v", len(unresolved.deps), err)
	}

	return resolveDependencies(st, depMan, unresolved)
}

type chartDependencyManager struct {
	Name string

	lockFilePath string

	logger *zap.SugaredLogger

	readFile  func(string) ([]byte, error)
	writeFile func(string, []byte, os.FileMode) error
}

func NewChartDependencyManager(name string, logger *zap.SugaredLogger, lockFilePath string) *chartDependencyManager {
	return &chartDependencyManager{
		Name:         name,
		readFile:     os.ReadFile,
		writeFile:    os.WriteFile,
		logger:       logger,
		lockFilePath: lockFilePath,
	}
}

func (m *chartDependencyManager) lockFileName() string {
	if m.lockFilePath != "" {
		return m.lockFilePath
	} else {
		return fmt.Sprintf("%s.lock", m.Name)
	}
}

func (m *chartDependencyManager) Update(shell helmexec.DependencyUpdater, wd string, unresolved *UnresolvedDependencies) (*ResolvedDependencies, error) {
	return m.updateHelm(shell, wd, unresolved)
}

func (m *chartDependencyManager) updateHelm(shell helmexec.DependencyUpdater, wd string, unresolved *UnresolvedDependencies) (*ResolvedDependencies, error) {
	// Generate `Chart.yaml` of the temporary local chart
	// Both Helm 3 and Helm 4 use apiVersion: v2 and Chart.lock
	chartMetaContent := fmt.Sprintf("name: %s\nversion: 1.0.0\napiVersion: v2\n", m.Name)

	// Generate `requirements.yaml` of the temporary local chart from the helmfile state
	reqsContent, err := yaml.Marshal(unresolved.ToChartRequirements())
	if err != nil {
		return nil, err
	}
	if err := m.writeBytes(filepath.Join(wd, "Chart.yaml"), []byte(chartMetaContent+string(reqsContent))); err != nil {
		return nil, err
	}

	return m.doUpdate("Chart.lock", unresolved, shell, wd)
}

func (m *chartDependencyManager) doUpdate(chartLockFile string, unresolved *UnresolvedDependencies, shell helmexec.DependencyUpdater, wd string) (*ResolvedDependencies, error) {
	// Generate `requirements.lock` of the temporary local chart by coping `<basename>.lock`
	lockFilePath := m.lockFileName()

	originalLockFileContent, err := m.readBytes(lockFilePath)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	if originalLockFileContent != nil {
		if err := m.writeBytes(filepath.Join(wd, chartLockFile), originalLockFileContent); err != nil {
			return nil, err
		}
	}

	// Update the lock file by running `helm dependency update`
	if err := shell.UpdateDeps(wd); err != nil {
		return nil, err
	}

	updatedLockFileContent, err := m.readBytes(filepath.Join(wd, chartLockFile))
	if err != nil {
		return nil, err
	}

	// Sort requirements alphabetically by name.
	lockedReqs := &ChartLockedRequirements{}
	if err := yaml.Unmarshal(updatedLockFileContent, lockedReqs); err != nil {
		return nil, err
	}

	sort.Slice(lockedReqs.ResolvedDependencies, func(i, j int) bool {
		return lockedReqs.ResolvedDependencies[i].ChartName < lockedReqs.ResolvedDependencies[j].ChartName
	})

	lockedReqs.ResolvedDependencies = dedupResolvedDependencies(lockedReqs.ResolvedDependencies)

	lockedReqs.Version = version.Version()

	updatedLockFileContent, err = yaml.Marshal(lockedReqs)

	if err != nil {
		return nil, err
	}

	// Commit the lock file if and only if everything looks ok
	if err := m.writeBytes(lockFilePath, updatedLockFileContent); err != nil {
		return nil, err
	}

	resolved, _, err := m.Resolve(unresolved)
	return resolved, err
}

func (m *chartDependencyManager) Resolve(unresolved *UnresolvedDependencies) (*ResolvedDependencies, bool, error) {
	updatedLockFileContent, err := m.readBytes(m.lockFileName())
	if err != nil {
		if os.IsNotExist(err) {
			return nil, false, nil
		}
		return nil, false, err
	}

	// Load resolved dependencies into memory
	lockedReqs := &ChartLockedRequirements{}
	if err := yaml.Unmarshal(updatedLockFileContent, lockedReqs); err != nil {
		return nil, false, err
	}

	// Make sure go run main.go works and compatible with old lock files.
	if version.Version() != "" && lockedReqs.Version != "" {
		// Check that the locked version, i.e. the helmfile binary version recorded in the lock file,
		// conforms to semver.
		// This is purely for validation purposes.
		_, err := semver.NewVersion(lockedReqs.Version)
		if err != nil {
			return nil, false, err
		}

		// Note: We no longer validate the version of the lockfile against the version of the helmfile binary.
		// See https://github.com/helmfile/helmfile/issues/1473
	}

	resolved := &ResolvedDependencies{deps: map[string][]ResolvedChartDependency{}}
	for _, d := range lockedReqs.ResolvedDependencies {
		if err := resolved.add(d); err != nil {
			return nil, false, err
		}
	}

	return resolved, true, nil
}

func (m *chartDependencyManager) readBytes(filename string) ([]byte, error) {
	bytes, err := m.readFile(filename)
	if err != nil {
		return nil, err
	}
	m.logger.Debugf("readBytes: read from %s:\n%s", filename, bytes)
	return bytes, nil
}

func (m *chartDependencyManager) writeBytes(filename string, data []byte) error {
	err := m.writeFile(filename, data, 0644)
	if err != nil {
		return err
	}
	m.logger.Debugf("writeBytes: wrote to %s:\n%s", filename, data)
	return nil
}