helmfile/pkg/hcllang/hcl_loader.go

package hcllang

import (
	nativejson "encoding/json"
	"fmt"
	"slices"
	"strings"

	"dario.cat/mergo"
	"github.com/hashicorp/hcl/v2"
	"github.com/hashicorp/hcl/v2/hclparse"
	"github.com/hashicorp/hcl/v2/hclsyntax"
	"github.com/variantdev/dag/pkg/dag"
	"github.com/zclconf/go-cty/cty"
	"github.com/zclconf/go-cty/cty/json"
	"go.uber.org/zap"

	"github.com/helmfile/helmfile/pkg/filesystem"
)

const (
	badIdentifierDetail   = "A name must start with a letter or underscore and may contain only letters, digits, underscores, and dashes."
	ValuesBlockIdentifier = "values"
	LocalsBlockIdentifier = "locals"
	valuesAccessorPrefix  = "hv"
	localsAccessorPrefix  = "local"
)

// ctyMergeValues returns a new cty.Value that is the deep‑merge of a and b.
// a is the “base”, b is the “override”
func ctyMergeValues(a, b cty.Value) cty.Value {
	// If b is null, keep a.
	if b.IsNull() {
		return a
	}

	// If b is unknown (but not null), let the override propagate.
	if !b.IsKnown() {
		return b
	}

	// If a is null or unknown, just return b.
	if a.IsNull() || !a.IsKnown() {
		return b
	}

	// Objects or Maps -> merge per key.
	if (a.Type().IsObjectType() && b.Type().IsObjectType()) ||
		(a.Type().IsMapType() && b.Type().IsMapType()) {
		mergedAttrs := make(map[string]cty.Value)
		// Start with all attrs from a.
		for name, av := range a.AsValueMap() {
			mergedAttrs[name] = av
		}
		// Overlay attrs from b.
		for name, bv := range b.AsValueMap() {
			if av, ok := mergedAttrs[name]; ok {
				mergedAttrs[name] = ctyMergeValues(av, bv)
			} else {
				mergedAttrs[name] = bv
			}
		}
		if a.Type().IsMapType() {
			return cty.MapVal(mergedAttrs)
		}
		return cty.ObjectVal(mergedAttrs)
	}

	// Tuples / Lists -> replace
	if (a.Type().IsTupleType() && b.Type().IsTupleType()) ||
		(a.Type().IsListType() && b.Type().IsListType()) {
		return b
	}

	// Anything else (numbers, strings, bools, etc.) -> right‑hand side wins.
	return b
}

// HelmfileHCLValue represents a single entry from a "values" or "locals" block file.
// The blocks itself is not represented, because it serves only to
// provide context for us to interpret its contents.
type HelmfileHCLValue struct {
	Name  string
	Expr  hcl.Expression
	Range hcl.Range
}

type HCLLoader struct {
	hclFilesPath    []string
	fs              *filesystem.FileSystem
	logger          *zap.SugaredLogger
	allVariableDefs map[string][]*HelmfileHCLValue // Track all definitions for merging
}

func NewHCLLoader(fs *filesystem.FileSystem, logger *zap.SugaredLogger) *HCLLoader {
	return &HCLLoader{
		fs:     fs,
		logger: logger,
	}
}
func (hl *HCLLoader) AddFile(file string) {
	hl.hclFilesPath = append(hl.hclFilesPath, file)
}

func (hl *HCLLoader) AddFiles(files []string) {
	hl.hclFilesPath = append(hl.hclFilesPath, files...)
}

func (hl *HCLLoader) Length() int {
	return len(hl.hclFilesPath)
}

func (hl *HCLLoader) HCLRender() (map[string]any, error) {
	if hl.Length() == 0 {
		return nil, fmt.Errorf("nothing to render")
	}

	HelmfileHCLValues, locals, diags := hl.readHCLs()
	if len(diags) > 0 {
		return nil, diags.Errs()[0]
	}

	// Decode all locals from all files first
	// in order for them to be usable in values blocks
	localsCty := map[string]map[string]cty.Value{}
	for k, local := range locals {
		dagPlan, err := hl.createDAGGraph(local, LocalsBlockIdentifier)
		if err != nil {
			return nil, err
		}
		localFileCty, err := hl.decodeGraph(dagPlan, LocalsBlockIdentifier, locals[k], nil)
		if err != nil {
			return nil, err
		}
		localsCty[k] = make(map[string]cty.Value)
		localsCty[k][localsAccessorPrefix] = localFileCty[localsAccessorPrefix]
	}

	// Decode Values
	dagHelmfileValuePlan, err := hl.createDAGGraph(HelmfileHCLValues, ValuesBlockIdentifier)
	if err != nil {
		return nil, err
	}
	helmfileVarCty, err := hl.decodeGraph(dagHelmfileValuePlan, ValuesBlockIdentifier, HelmfileHCLValues, localsCty)
	if err != nil {
		return nil, err
	}
	nativeGovals, err := hl.convertToGo(helmfileVarCty)
	if err != nil {
		return nil, err
	}
	return nativeGovals, nil
}

func (hl *HCLLoader) createDAGGraph(HelmfileHCLValues map[string]*HelmfileHCLValue, blockType string) (*dag.Topology, error) {
	dagGraph := dag.New()

	for _, hv := range HelmfileHCLValues {
		var traversals []string

		// For values blocks with multiple definitions, collect dependencies from ALL definitions
		// to ensure proper evaluation order even if only earlier definitions have dependencies
		if blockType == ValuesBlockIdentifier && hl.allVariableDefs != nil {
			varDefs := hl.allVariableDefs[hv.Name]
			if len(varDefs) > 1 {
				// Collect traversals from all definitions
				for _, varDef := range varDefs {
					for _, tr := range varDef.Expr.Variables() {
						attr, diags := hl.parseSingleAttrRef(tr, blockType)
						if diags != nil {
							return nil, fmt.Errorf("%s", diags.Errs()[0])
						}
						if attr != "" && !slices.Contains(traversals, attr) {
							traversals = append(traversals, attr)
						}
					}
				}
			} else {
				// Single definition, collect traversals normally
				for _, tr := range hv.Expr.Variables() {
					attr, diags := hl.parseSingleAttrRef(tr, blockType)
					if diags != nil {
						return nil, fmt.Errorf("%s", diags.Errs()[0])
					}
					if attr != "" && !slices.Contains(traversals, attr) {
						traversals = append(traversals, attr)
					}
				}
			}
		} else {
			// For locals or when no tracking available, collect traversals normally
			for _, tr := range hv.Expr.Variables() {
				attr, diags := hl.parseSingleAttrRef(tr, blockType)
				if diags != nil {
					return nil, fmt.Errorf("%s", diags.Errs()[0])
				}
				if attr != "" && !slices.Contains(traversals, attr) {
					traversals = append(traversals, attr)
				}
			}
		}
		hl.logger.Debugf("Adding Dependency : %s  => [%s]", hv.Name, strings.Join(traversals, ", "))
		dagGraph.Add(hv.Name, dag.Dependencies(traversals))
	}

	//Generate Dag Plan which will provide the order from which to interpolate vars
	plan, err := dagGraph.Plan(dag.SortOptions{
		WithDependencies: true,
	})
	if err == nil {
		return &plan, nil
	}

	if ude, ok := err.(*dag.UndefinedDependencyError); ok {
		var quotedVariableNames []string
		for _, d := range ude.Dependents {
			quotedVariableNames = append(quotedVariableNames, fmt.Sprintf("%q", d))
		}
		return nil, fmt.Errorf("variables %s depend(s) on undefined vars %q", strings.Join(quotedVariableNames, ", "), ude.UndefinedNode)
	} else {
		return nil, fmt.Errorf("error while building the DAG variable graph : %s", err.Error())
	}
}

func (hl *HCLLoader) decodeGraph(dagTopology *dag.Topology, blocktype string, vars map[string]*HelmfileHCLValue, additionalLocalContext map[string]map[string]cty.Value) (map[string]cty.Value, error) {
	values := map[string]cty.Value{}
	helmfileHCLValuesValues := map[string]cty.Value{}
	var diags hcl.Diagnostics
	hclFunctions, err := HCLFunctions(nil)
	if err != nil {
		return nil, err
	}
	for groupIndex := 0; groupIndex < len(*dagTopology); groupIndex++ {
		dagNodesInGroup := (*dagTopology)[groupIndex]

		for _, node := range dagNodesInGroup {
			v := vars[node.String()]
			if blocktype != LocalsBlockIdentifier && additionalLocalContext[v.Range.Filename] != nil {
				values[localsAccessorPrefix] = additionalLocalContext[v.Range.Filename][localsAccessorPrefix]
			}
			ctx := &hcl.EvalContext{
				Variables: values,
				Functions: hclFunctions,
			}

			// Check if this variable has multiple definitions (overrides) for values blocks
			if blocktype == ValuesBlockIdentifier && hl.allVariableDefs != nil {
				varDefs := hl.allVariableDefs[node.String()]
				if len(varDefs) > 1 {
					// Evaluate and merge all definitions in file order
					var mergedValue cty.Value
					for i, varDef := range varDefs {
						// Update local context for each file
						if additionalLocalContext[varDef.Range.Filename] != nil {
							ctx.Variables[localsAccessorPrefix] = additionalLocalContext[varDef.Range.Filename][localsAccessorPrefix]
						} else {
							// Ensure locals from a previous definition/file do not leak into this evaluation
							ctx.Variables[localsAccessorPrefix] = cty.NilVal
						}
						evalValue, evalDiags := varDef.Expr.Value(ctx)
						if len(evalDiags) > 0 {
							return nil, fmt.Errorf("error when trying to evaluate variable %s at %s:%d : %s",
								varDef.Name, varDef.Range.Filename, varDef.Range.Start.Line, evalDiags.Errs()[0])
						}
						if i == 0 {
							mergedValue = evalValue
						} else {
							mergedValue = ctyMergeValues(mergedValue, evalValue)
						}
					}
					helmfileHCLValuesValues[node.String()] = mergedValue
					// Reset local context
					values[localsAccessorPrefix] = cty.NilVal
				} else {
					// Single definition, evaluate normally
					helmfileHCLValuesValues[node.String()], diags = v.Expr.Value(ctx)
					if len(diags) > 0 {
						return nil, fmt.Errorf("error when trying to evaluate variable %s : %s", v.Name, diags.Errs()[0])
					}
				}
			} else {
				// For locals or when no tracking available, evaluate normally
				helmfileHCLValuesValues[node.String()], diags = v.Expr.Value(ctx)
				if len(diags) > 0 {
					return nil, fmt.Errorf("error when trying to evaluate variable %s : %s", v.Name, diags.Errs()[0])
				}
			}

			switch blocktype {
			case ValuesBlockIdentifier:
				// Update the eval context for the next value evaluation iteration
				values[valuesAccessorPrefix] = cty.ObjectVal(helmfileHCLValuesValues)
				// Set back local to nil to avoid an unexpected behavior when the next iteration is in another file
				values[localsAccessorPrefix] = cty.NilVal
			case LocalsBlockIdentifier:
				values[localsAccessorPrefix] = cty.ObjectVal(helmfileHCLValuesValues)
			}
		}
	}
	return values, nil
}

func (hl *HCLLoader) readHCLs() (map[string]*HelmfileHCLValue, map[string]map[string]*HelmfileHCLValue, hcl.Diagnostics) {
	var variables map[string]*HelmfileHCLValue
	// Track all definitions for merging during evaluation
	var allVariableDefs map[string][]*HelmfileHCLValue
	var local map[string]*HelmfileHCLValue
	locals := map[string]map[string]*HelmfileHCLValue{}
	var diags hcl.Diagnostics
	for _, file := range hl.hclFilesPath {
		variables, allVariableDefs, local, diags = hl.readHCL(variables, allVariableDefs, file)
		if diags != nil {
			return nil, nil, diags
		}
		locals[file] = make(map[string]*HelmfileHCLValue)
		locals[file] = local
	}

	// Store all definitions in the HCLLoader for use in decodeGraph
	hl.allVariableDefs = allVariableDefs

	return variables, locals, nil
}

func (hl *HCLLoader) readHCL(hvars map[string]*HelmfileHCLValue, allDefs map[string][]*HelmfileHCLValue, file string) (map[string]*HelmfileHCLValue, map[string][]*HelmfileHCLValue, map[string]*HelmfileHCLValue, hcl.Diagnostics) {
	src, err := hl.fs.ReadFile(file)
	if err != nil {
		return nil, nil, nil, hcl.Diagnostics{
			{
				Severity: hcl.DiagError,
				Summary:  fmt.Sprintf("%s", err),
				Detail:   "could not read file",
				Subject:  &hcl.Range{},
			},
		}
	}

	// Parse file as HCL
	p := hclparse.NewParser()
	hclFile, diags := p.ParseHCL(src, file)
	if hclFile == nil || hclFile.Body == nil || diags != nil {
		return nil, nil, nil, diags
	}

	HelmfileHCLValuesSchema := &hcl.BodySchema{
		Blocks: []hcl.BlockHeaderSchema{
			{
				Type: ValuesBlockIdentifier,
			},
			{
				Type: LocalsBlockIdentifier,
			},
		},
	}
	// make sure content has a struct with helmfile_vars Schema defined
	content, diags := hclFile.Body.Content(HelmfileHCLValuesSchema)
	if diags != nil {
		return nil, nil, nil, diags
	}

	var helmfileLocalsVars map[string]*HelmfileHCLValue
	// Decode blocks to return HelmfileHCLValue object => (each var with expr + Name )

	if len(content.Blocks.OfType(LocalsBlockIdentifier)) > 1 {
		return nil, nil, nil, hcl.Diagnostics{
			&hcl.Diagnostic{
				Severity: hcl.DiagError,
				Summary:  "A file can only support exactly 1 `locals` block",
				Subject:  &content.Blocks[0].DefRange,
			}}
	}
	for _, block := range content.Blocks {
		var helmfileBlockVars map[string]*HelmfileHCLValue
		if block.Type == ValuesBlockIdentifier {
			helmfileBlockVars, diags = hl.decodeHelmfileHCLValuesBlock(block)
			if diags != nil {
				return nil, nil, nil, diags
			}
		}

		if block.Type == LocalsBlockIdentifier {
			helmfileLocalsVars, diags = hl.decodeHelmfileHCLValuesBlock(block)
			if diags != nil {
				return nil, nil, nil, diags
			}
		}

		// Track all definitions before merging
		if allDefs == nil {
			allDefs = make(map[string][]*HelmfileHCLValue)
		}
		for k, v := range helmfileBlockVars {
			// Make a copy of v to avoid it being modified by future mergo.Merge calls
			vCopy := *v

			// If this variable already exists in hvars (from a previous file),
			// ensure the old definition is tracked before adding the new one
			if hvars != nil && hvars[k] != nil {
				// Check if hvars[k] (the old definition) is already in allDefs
				oldDefTracked := false
				for _, existing := range allDefs[k] {
					// Compare by values to see if this exact definition is already tracked
					if existing.Range == hvars[k].Range {
						oldDefTracked = true
						break
					}
				}
				if !oldDefTracked {
					// Old definition not yet tracked - add a copy of it
					oldDefCopy := *hvars[k]
					allDefs[k] = append(allDefs[k], &oldDefCopy)
				}
			}
			// Now add the new definition from this file (as a copy)
			allDefs[k] = append(allDefs[k], &vCopy)
		}

		// Allow override of variables across files - last one wins
		// The actual merge will happen in decodeGraph() where we have proper evaluation context
		err = mergo.Merge(&hvars, &helmfileBlockVars, mergo.WithOverride)
		if err != nil {
			var diags hcl.Diagnostics
			diags = append(diags, &hcl.Diagnostic{
				Severity: hcl.DiagError,
				Summary:  "Merge failed",
				Detail:   err.Error(),
				Subject:  nil,
			})
			return nil, nil, nil, diags
		}
	}

	return hvars, allDefs, helmfileLocalsVars, nil
}

func (hl *HCLLoader) decodeHelmfileHCLValuesBlock(block *hcl.Block) (map[string]*HelmfileHCLValue, hcl.Diagnostics) {
	attrs, diags := block.Body.JustAttributes()
	if len(attrs) == 0 || diags != nil {
		return nil, diags
	}

	hfVars := map[string]*HelmfileHCLValue{}
	for name, attr := range attrs {
		if !hclsyntax.ValidIdentifier(name) {
			diags = append(diags, &hcl.Diagnostic{
				Severity: hcl.DiagError,
				Summary:  "Invalid helmfile_vars variable name",
				Detail:   badIdentifierDetail,
				Subject:  &attr.NameRange,
			})
		}

		hfVars[name] = &HelmfileHCLValue{
			Name:  name,
			Expr:  attr.Expr,
			Range: attr.Range,
		}
	}
	return hfVars, diags
}

func (hl *HCLLoader) parseSingleAttrRef(traversal hcl.Traversal, blockType string) (string, hcl.Diagnostics) {
	if len(traversal) == 0 {
		return "", hcl.Diagnostics{
			&hcl.Diagnostic{
				Severity: hcl.DiagError,
				Summary:  "An empty traversal can't be parsed",
			},
		}
	}
	root := traversal.RootName()
	// In `values` blocks, Locals are always precomputed, so they don't need to be in the graph
	if root == localsAccessorPrefix && blockType != LocalsBlockIdentifier {
		return "", nil
	}
	rootRange := traversal[0].SourceRange()

	if len(traversal) < 2 {
		return "", hcl.Diagnostics{
			&hcl.Diagnostic{
				Severity: hcl.DiagError,
				Summary:  "Invalid reference",
				Detail:   fmt.Sprintf("The %q object cannot be accessed directly. Instead, access it from one of its root.", root),
				Subject:  &rootRange,
			},
		}
	}

	if attrTrav, ok := traversal[1].(hcl.TraverseAttr); ok {
		return attrTrav.Name, nil
	}

	return "", hcl.Diagnostics{
		&hcl.Diagnostic{
			Severity: hcl.DiagError,
			Summary:  "Invalid reference",
			Detail:   fmt.Sprintf("The %q object does not support this operation.", root),
			Subject:  traversal[1].SourceRange().Ptr(),
		},
	}
}

func (hl *HCLLoader) convertToGo(src map[string]cty.Value) (map[string]any, error) {
	// Ugly workaround on value conversion
	// CTY conversion to go natives requires much processing and complexity
	// All of this, in our context, can go away because of the CTY capability to dump a cty.Value as Json
	// The Json document outputs 2 keys : "type" and "value" which describe the mapping between the two
	// We only care about the value
	b, err := json.Marshal(src[valuesAccessorPrefix], cty.DynamicPseudoType)
	if err != nil {
		return nil, fmt.Errorf("could not marshal cty value : %s", err.Error())
	}

	var jsonunm map[string]any
	err = nativejson.Unmarshal(b, &jsonunm)
	if err != nil {
		return nil, fmt.Errorf("could not unmarshall json : %s", err.Error())
	}

	if result, ok := jsonunm["value"].(map[string]any); ok {
		return result, nil
	} else {
		return nil, fmt.Errorf("could extract a map object from json \"value\" key")
	}
}