Feature/support env hcl and interpolations (#1423)

* support HCL language for env variables

Signed-off-by: xtphate <65117176+XT-Phate@users.noreply.github.com>

pkg/hcllang/hcl_functions.go

@@ -0,0 +1,82 @@
+package hcllang
+
+import (
+	"github.com/hashicorp/hcl/v2/ext/tryfunc"
+	"github.com/imdario/mergo"
+	ctyyaml "github.com/zclconf/go-cty-yaml"
+	"github.com/zclconf/go-cty/cty/function"
+	"github.com/zclconf/go-cty/cty/function/stdlib"
+)
+
+func HCLFunctions(additionnalFunctions map[string]function.Function) (map[string]function.Function, error) {
+	var hclFunctions = map[string]function.Function{
+		"abs":             stdlib.AbsoluteFunc,
+		"can":             tryfunc.CanFunc,
+		"ceil":            stdlib.CeilFunc,
+		"chomp":           stdlib.ChompFunc,
+		"coalesce":        stdlib.CoalesceFunc,
+		"coalescelist":    stdlib.CoalesceListFunc,
+		"compact":         stdlib.CompactFunc,
+		"concat":          stdlib.ConcatFunc,
+		"contains":        stdlib.ContainsFunc,
+		"csvdecode":       stdlib.CSVDecodeFunc,
+		"distinct":        stdlib.DistinctFunc,
+		"element":         stdlib.ElementFunc,
+		"chunklist":       stdlib.ChunklistFunc,
+		"flatten":         stdlib.FlattenFunc,
+		"floor":           stdlib.FloorFunc,
+		"format":          stdlib.FormatFunc,
+		"formatdate":      stdlib.FormatDateFunc,
+		"formatlist":      stdlib.FormatListFunc,
+		"indent":          stdlib.IndentFunc,
+		"int":             stdlib.IntFunc,
+		"join":            stdlib.JoinFunc,
+		"jsondecode":      stdlib.JSONDecodeFunc,
+		"jsonencode":      stdlib.JSONEncodeFunc,
+		"keys":            stdlib.KeysFunc,
+		"length":          stdlib.LengthFunc,
+		"log":             stdlib.LogFunc,
+		"lookup":          stdlib.LookupFunc,
+		"lower":           stdlib.LowerFunc,
+		"max":             stdlib.MaxFunc,
+		"merge":           stdlib.MergeFunc,
+		"min":             stdlib.MinFunc,
+		"parseint":        stdlib.ParseIntFunc,
+		"pow":             stdlib.PowFunc,
+		"range":           stdlib.RangeFunc,
+		"regex":           stdlib.RegexFunc,
+		"regexall":        stdlib.RegexAllFunc,
+		"reverselist":     stdlib.ReverseListFunc,
+		"setintersection": stdlib.SetIntersectionFunc,
+		"setproduct":      stdlib.SetProductFunc,
+		"setsubtract":     stdlib.SetSubtractFunc,
+		"setunion":        stdlib.SetUnionFunc,
+		"signum":          stdlib.SignumFunc,
+		"slice":           stdlib.SliceFunc,
+		"sort":            stdlib.SortFunc,
+		"split":           stdlib.SplitFunc,
+		"strrev":          stdlib.ReverseFunc,
+		"strlen":          stdlib.StrlenFunc,
+		"substr":          stdlib.SubstrFunc,
+		"timeadd":         stdlib.TimeAddFunc,
+		"title":           stdlib.TitleFunc,
+		"trim":            stdlib.TrimFunc,
+		"trimprefix":      stdlib.TrimPrefixFunc,
+		"trimspace":       stdlib.TrimSpaceFunc,
+		"trimsuffix":      stdlib.TrimSuffixFunc,
+		"try":             tryfunc.TryFunc,
+		"upper":           stdlib.UpperFunc,
+		"values":          stdlib.ValuesFunc,
+		"yamldecode":      ctyyaml.YAMLDecodeFunc,
+		"yamlencode":      ctyyaml.YAMLEncodeFunc,
+		"zipmap":          stdlib.ZipmapFunc,
+	}
+
+	if additionnalFunctions != nil {
+		err := mergo.MapWithOverwrite(hclFunctions, additionnalFunctions)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return hclFunctions, nil
+}

pkg/hcllang/hcl_loader.go

@@ -0,0 +1,378 @@
+package hcllang
+
+import (
+	nativejson "encoding/json"
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclparse"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+	"github.com/imdario/mergo"
+	"github.com/variantdev/dag/pkg/dag"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/json"
+	"go.uber.org/zap"
+
+	"github.com/helmfile/helmfile/pkg/filesystem"
+)
+
+const (
+	badIdentifierDetail   = "A name must start with a letter or underscore and may contain only letters, digits, underscores, and dashes."
+	ValuesBlockIdentifier = "values"
+	LocalsBlockIdentifier = "locals"
+	valuesAccessorPrefix  = "hv"
+	localsAccessorPrefix  = "local"
+)
+
+// HelmfileHCLValue represents a single entry from a "values" or "locals" block file.
+// The blocks itself is not represented, because it serves only to
+// provide context for us to interpret its contents.
+type HelmfileHCLValue struct {
+	Name  string
+	Expr  hcl.Expression
+	Range hcl.Range
+}
+
+type HCLLoader struct {
+	hclFilesPath []string
+	fs           *filesystem.FileSystem
+	logger       *zap.SugaredLogger
+}
+
+func NewHCLLoader(fs *filesystem.FileSystem, logger *zap.SugaredLogger) *HCLLoader {
+	return &HCLLoader{
+		fs:     fs,
+		logger: logger,
+	}
+}
+func (hl *HCLLoader) AddFile(file string) {
+	hl.hclFilesPath = append(hl.hclFilesPath, file)
+}
+
+func (hl *HCLLoader) AddFiles(files []string) {
+	hl.hclFilesPath = append(hl.hclFilesPath, files...)
+}
+
+func (hl *HCLLoader) Length() int {
+	return len(hl.hclFilesPath)
+}
+
+func (hl *HCLLoader) HCLRender() (map[string]any, error) {
+	if hl.Length() == 0 {
+		return nil, fmt.Errorf("nothing to render")
+	}
+
+	HelmfileHCLValues, locals, diags := hl.readHCLs()
+	if len(diags) > 0 {
+		return nil, diags.Errs()[0]
+	}
+
+	// Decode all locals from all files first
+	// in order for them to be usable in values blocks
+	localsCty := map[string]map[string]cty.Value{}
+	for k, local := range locals {
+		dagPlan, err := hl.createDAGGraph(local, LocalsBlockIdentifier)
+		if err != nil {
+			return nil, err
+		}
+		localFileCty, err := hl.decodeGraph(dagPlan, LocalsBlockIdentifier, locals[k], nil)
+		if err != nil {
+			return nil, err
+		}
+		localsCty[k] = make(map[string]cty.Value)
+		localsCty[k][localsAccessorPrefix] = localFileCty[localsAccessorPrefix]
+	}
+
+	// Decode Values
+	dagHelmfileValuePlan, err := hl.createDAGGraph(HelmfileHCLValues, ValuesBlockIdentifier)
+	if err != nil {
+		return nil, err
+	}
+	helmfileVarCty, err := hl.decodeGraph(dagHelmfileValuePlan, ValuesBlockIdentifier, HelmfileHCLValues, localsCty)
+	if err != nil {
+		return nil, err
+	}
+	nativeGovals, err := hl.convertToGo(helmfileVarCty)
+	if err != nil {
+		return nil, err
+	}
+	return nativeGovals, nil
+}
+
+func (hl *HCLLoader) createDAGGraph(HelmfileHCLValues map[string]*HelmfileHCLValue, blockType string) (*dag.Topology, error) {
+	dagGraph := dag.New()
+
+	for _, hv := range HelmfileHCLValues {
+		var traversals []string
+		for _, tr := range hv.Expr.Variables() {
+			attr, diags := hl.parseSingleAttrRef(tr, blockType)
+			if diags != nil {
+				return nil, fmt.Errorf("%s", diags.Errs()[0])
+			}
+			if attr != "" && !slices.Contains(traversals, attr) {
+				traversals = append(traversals, attr)
+			}
+		}
+		hl.logger.Debugf("Adding Dependency : %s  => [%s]", hv.Name, strings.Join(traversals, ", "))
+		dagGraph.Add(hv.Name, dag.Dependencies(traversals))
+	}
+
+	//Generate Dag Plan which will provide the order from which to interpolate vars
+	plan, err := dagGraph.Plan(dag.SortOptions{
+		WithDependencies: true,
+	})
+	if err == nil {
+		return &plan, nil
+	}
+
+	if ude, ok := err.(*dag.UndefinedDependencyError); ok {
+		var quotedVariableNames []string
+		for _, d := range ude.Dependents {
+			quotedVariableNames = append(quotedVariableNames, fmt.Sprintf("%q", d))
+		}
+		return nil, fmt.Errorf("variables %s depend(s) on undefined vars %q", strings.Join(quotedVariableNames, ", "), ude.UndefinedNode)
+	} else {
+		return nil, fmt.Errorf("error while building the DAG variable graph : %s", err.Error())
+	}
+}
+
+func (hl *HCLLoader) decodeGraph(dagTopology *dag.Topology, blocktype string, vars map[string]*HelmfileHCLValue, additionalLocalContext map[string]map[string]cty.Value) (map[string]cty.Value, error) {
+	values := map[string]cty.Value{}
+	helmfileHCLValuesValues := map[string]cty.Value{}
+	var diags hcl.Diagnostics
+	hclFunctions, err := HCLFunctions(nil)
+	if err != nil {
+		return nil, err
+	}
+	for groupIndex := 0; groupIndex < len(*dagTopology); groupIndex++ {
+		dagNodesInGroup := (*dagTopology)[groupIndex]
+
+		for _, node := range dagNodesInGroup {
+			v := vars[node.String()]
+			if blocktype != LocalsBlockIdentifier && additionalLocalContext[v.Range.Filename] != nil {
+				values[localsAccessorPrefix] = additionalLocalContext[v.Range.Filename][localsAccessorPrefix]
+			}
+			ctx := &hcl.EvalContext{
+				Variables: values,
+				Functions: hclFunctions,
+			}
+			// Decode Value
+			helmfileHCLValuesValues[node.String()], diags = v.Expr.Value(ctx)
+			if len(diags) > 0 {
+				return nil, fmt.Errorf("error when trying to evaluate variable %s : %s", v.Name, diags.Errs()[0])
+			}
+			switch blocktype {
+			case ValuesBlockIdentifier:
+				// Update the eval context for the next value evaluation iteration
+				values[valuesAccessorPrefix] = cty.ObjectVal(helmfileHCLValuesValues)
+				// Set back local to nil to avoid an unexpected behavior when the next iteration is in another file
+				values[localsAccessorPrefix] = cty.NilVal
+			case LocalsBlockIdentifier:
+				values[localsAccessorPrefix] = cty.ObjectVal(helmfileHCLValuesValues)
+			}
+		}
+	}
+	return values, nil
+}
+
+func (hl *HCLLoader) readHCLs() (map[string]*HelmfileHCLValue, map[string]map[string]*HelmfileHCLValue, hcl.Diagnostics) {
+	var variables map[string]*HelmfileHCLValue
+	var local map[string]*HelmfileHCLValue
+	locals := map[string]map[string]*HelmfileHCLValue{}
+	var diags hcl.Diagnostics
+	for _, file := range hl.hclFilesPath {
+		variables, local, diags = hl.readHCL(variables, file)
+		if diags != nil {
+			return nil, nil, diags
+		}
+		locals[file] = make(map[string]*HelmfileHCLValue)
+		locals[file] = local
+	}
+	return variables, locals, nil
+}
+
+func (hl *HCLLoader) readHCL(hvars map[string]*HelmfileHCLValue, file string) (map[string]*HelmfileHCLValue, map[string]*HelmfileHCLValue, hcl.Diagnostics) {
+	src, err := hl.fs.ReadFile(file)
+	if err != nil {
+		return nil, nil, hcl.Diagnostics{
+			{
+				Severity: hcl.DiagError,
+				Summary:  fmt.Sprintf("%s", err),
+				Detail:   "could not read file",
+				Subject:  &hcl.Range{},
+			},
+		}
+	}
+
+	// Parse file as HCL
+	p := hclparse.NewParser()
+	hclFile, diags := p.ParseHCL(src, file)
+	if hclFile == nil || hclFile.Body == nil || diags != nil {
+		return nil, nil, diags
+	}
+
+	HelmfileHCLValuesSchema := &hcl.BodySchema{
+		Blocks: []hcl.BlockHeaderSchema{
+			{
+				Type: ValuesBlockIdentifier,
+			},
+			{
+				Type: LocalsBlockIdentifier,
+			},
+		},
+	}
+	// make sure content has a struct with helmfile_vars Schema defined
+	content, diags := hclFile.Body.Content(HelmfileHCLValuesSchema)
+	if diags != nil {
+		return nil, nil, diags
+	}
+
+	var helmfileLocalsVars map[string]*HelmfileHCLValue
+	// Decode blocks to return HelmfileHCLValue object => (each var with expr + Name )
+
+	if len(content.Blocks.OfType(LocalsBlockIdentifier)) > 1 {
+		return nil, nil, hcl.Diagnostics{
+			&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "A file can only support exactly 1 `locals` block",
+				Subject:  &content.Blocks[0].DefRange,
+			}}
+	}
+	for _, block := range content.Blocks {
+		var helmfileBlockVars map[string]*HelmfileHCLValue
+		if block.Type == ValuesBlockIdentifier {
+			helmfileBlockVars, diags = hl.decodeHelmfileHCLValuesBlock(block)
+			if diags != nil {
+				return nil, nil, diags
+			}
+		}
+
+		if block.Type == LocalsBlockIdentifier {
+			helmfileLocalsVars, diags = hl.decodeHelmfileHCLValuesBlock(block)
+			if diags != nil {
+				return nil, nil, diags
+			}
+		}
+
+		// make sure vars are unique across blocks
+		for k := range helmfileBlockVars {
+			if hvars[k] != nil {
+				var diags hcl.Diagnostics
+				diags = append(diags, &hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Duplicate helmfile_vars definition",
+					Detail: fmt.Sprintf("The helmfile_var %q was already defined at %s:%d",
+						k, hvars[k].Range.Filename, hvars[k].Range.Start.Line),
+					Subject: &helmfileBlockVars[k].Range,
+				})
+				return nil, nil, diags
+			}
+		}
+		err = mergo.Merge(&hvars, &helmfileBlockVars)
+		if err != nil {
+			var diags hcl.Diagnostics
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Merge failed",
+				Detail:   err.Error(),
+				Subject:  nil,
+			})
+			return nil, nil, diags
+		}
+	}
+
+	return hvars, helmfileLocalsVars, nil
+}
+
+func (hl *HCLLoader) decodeHelmfileHCLValuesBlock(block *hcl.Block) (map[string]*HelmfileHCLValue, hcl.Diagnostics) {
+	attrs, diags := block.Body.JustAttributes()
+	if len(attrs) == 0 || diags != nil {
+		return nil, diags
+	}
+
+	hfVars := map[string]*HelmfileHCLValue{}
+	for name, attr := range attrs {
+		if !hclsyntax.ValidIdentifier(name) {
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid helmfile_vars variable name",
+				Detail:   badIdentifierDetail,
+				Subject:  &attr.NameRange,
+			})
+		}
+
+		hfVars[name] = &HelmfileHCLValue{
+			Name:  name,
+			Expr:  attr.Expr,
+			Range: attr.Range,
+		}
+	}
+	return hfVars, diags
+}
+
+func (hl *HCLLoader) parseSingleAttrRef(traversal hcl.Traversal, blockType string) (string, hcl.Diagnostics) {
+	if len(traversal) == 0 {
+		return "", hcl.Diagnostics{
+			&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "An empty traversal can't be parsed",
+			},
+		}
+	}
+	root := traversal.RootName()
+	// In `values` blocks, Locals are always precomputed, so they don't need to be in the graph
+	if root == localsAccessorPrefix && blockType != LocalsBlockIdentifier {
+		return "", nil
+	}
+	rootRange := traversal[0].SourceRange()
+
+	if len(traversal) < 2 {
+		return "", hcl.Diagnostics{
+			&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid reference",
+				Detail:   fmt.Sprintf("The %q object cannot be accessed directly. Instead, access it from one of its root.", root),
+				Subject:  &rootRange,
+			},
+		}
+	}
+
+	if attrTrav, ok := traversal[1].(hcl.TraverseAttr); ok {
+		return attrTrav.Name, nil
+	}
+
+	return "", hcl.Diagnostics{
+		&hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Invalid reference",
+			Detail:   fmt.Sprintf("The %q object does not support this operation.", root),
+			Subject:  traversal[1].SourceRange().Ptr(),
+		},
+	}
+}
+
+func (hl *HCLLoader) convertToGo(src map[string]cty.Value) (map[string]any, error) {
+	// Ugly workaround on value conversion
+	// CTY conversion to go natives requires much processing and complexity
+	// All of this, in our context, can go away because of the CTY capability to dump a cty.Value as Json
+	// The Json document outputs 2 keys : "type" and "value" which describe the mapping between the two
+	// We only care about the value
+	b, err := json.Marshal(src[valuesAccessorPrefix], cty.DynamicPseudoType)
+	if err != nil {
+		return nil, fmt.Errorf("could not marshal cty value : %s", err.Error())
+	}
+
+	var jsonunm map[string]any
+	err = nativejson.Unmarshal(b, &jsonunm)
+	if err != nil {
+		return nil, fmt.Errorf("could not unmarshall json : %s", err.Error())
+	}
+
+	if result, ok := jsonunm["value"].(map[string]any); ok {
+		return result, nil
+	} else {
+		return nil, fmt.Errorf("could extract a map object from json \"value\" key")
+	}
+}

pkg/hcllang/hcl_loader_test.go

@@ -0,0 +1,163 @@
+package hcllang
+
+import (
+	"io"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+
+	ffs "github.com/helmfile/helmfile/pkg/filesystem"
+	"github.com/helmfile/helmfile/pkg/helmexec"
+)
+
+func newHCLLoader() *HCLLoader {
+	log := helmexec.NewLogger(io.Discard, "debug")
+	return &HCLLoader{
+		fs:     ffs.DefaultFileSystem(),
+		logger: log,
+	}
+}
+
+func TestHCL_localsTraversalsParser(t *testing.T) {
+	l := newHCLLoader()
+	files := []string{"testdata/values.1.hcl"}
+	l.AddFiles(files)
+
+	_, filesLocals, diags := l.readHCLs()
+	if diags != nil {
+		t.Errorf("Test file parsing error : %s", diags.Errs()[0].Error())
+	}
+
+	actual := make(map[string]map[string]int)
+	for file, locals := range filesLocals {
+		actual[file] = make(map[string]int)
+		for k, v := range locals {
+			actual[file][k] = len(v.Expr.Variables())
+		}
+	}
+
+	expected := map[string]map[string]int{
+		"testdata/values.1.hcl": {
+			"myLocal":    0,
+			"myLocalRef": 1,
+		},
+	}
+
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf(diff)
+	}
+}
+
+func TestHCL_localsTraversalsAttrParser(t *testing.T) {
+	l := newHCLLoader()
+	files := []string{"testdata/values.1.hcl"}
+	l.AddFiles(files)
+
+	_, filesLocals, diags := l.readHCLs()
+	if diags != nil {
+		t.Errorf("Test file parsing error : %s", diags.Errs()[0].Error())
+	}
+
+	actual := make(map[string]map[string]string)
+	for file, locals := range filesLocals {
+		actual[file] = make(map[string]string)
+		for k, v := range locals {
+			str := ""
+			for _, v := range v.Expr.Variables() {
+				tr, _ := l.parseSingleAttrRef(v, LocalsBlockIdentifier)
+				str += tr
+			}
+			actual[file][k] = str
+		}
+	}
+
+	expected := map[string]map[string]string{
+		"testdata/values.1.hcl": {
+			"myLocal":    "",
+			"myLocalRef": "myLocal",
+		},
+	}
+
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf(diff)
+	}
+}
+func TestHCL_valuesTraversalsParser(t *testing.T) {
+	l := newHCLLoader()
+	files := []string{"testdata/values.1.hcl"}
+	l.AddFiles(files)
+
+	fileValues, _, diags := l.readHCLs()
+	if diags != nil {
+		t.Errorf("Test file parsing error : %s", diags.Errs()[0].Error())
+	}
+
+	actual := make(map[string]int)
+	for k, v := range fileValues {
+		actual[k] = len(v.Expr.Variables())
+	}
+
+	expected := map[string]int{
+		"val1": 0,
+		"val2": 1,
+		"val3": 2,
+		"val4": 1,
+	}
+
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf(diff)
+	}
+}
+
+func TestHCL_valuesTraversalsAttrParser(t *testing.T) {
+	l := newHCLLoader()
+	files := []string{"testdata/values.1.hcl"}
+	l.AddFiles(files)
+
+	fileValues, _, diags := l.readHCLs()
+	if diags != nil {
+		t.Errorf("Test file parsing error : %s", diags.Errs()[0].Error())
+	}
+
+	actual := make(map[string]string)
+	for k, v := range fileValues {
+		str := ""
+		for _, v := range v.Expr.Variables() {
+			tr, _ := l.parseSingleAttrRef(v, ValuesBlockIdentifier)
+			str += tr
+		}
+		actual[k] = str
+	}
+
+	expected := map[string]string{
+		"val1": "",
+		"val2": "",
+		"val3": "val1",
+		"val4": "val1",
+	}
+
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf(diff)
+	}
+}
+
+func TestHCL_resultValidate(t *testing.T) {
+	l := newHCLLoader()
+	files := []string{"testdata/values.1.hcl"}
+	l.AddFiles(files)
+	actual, err := l.HCLRender()
+	if err != nil {
+		t.Errorf("Render error : %s", err.Error())
+	}
+
+	expected := map[string]any{
+		"val1": float64(1),
+		"val2": "LOCAL",
+		"val3": "local1",
+		"val4": float64(-1),
+	}
+
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf(diff)
+	}
+}

pkg/hcllang/testdata/values.1.hcl

@@ -0,0 +1,11 @@
+locals {
+  myLocal = "local"
+  myLocalRef = local.myLocal
+}
+
+values {
+  val1 = 1
+  val2 = upper(local.myLocal)
+  val3 = "${local.myLocal}${hv.val1}"
+  val4 = min(hv.val1, 10, -1)
+}

pkg/helmexec/exec.go

@@ -354,6 +354,22 @@ func (helm *execer) DecryptSecret(context HelmContext, name string, flags ...str
 			return "", err
 		}
 
+		// When the source encrypted file is not a yaml file AND helm secrets < 4
+		// secrets plugin returns a yaml file with all the content in a yaml `data` key
+		// which isn't parsable from an hcl perspective
+		if strings.HasSuffix(name, ".hcl") && pluginVersion.Major() < 4 {
+			type helmSecretDataV3 struct {
+				Data string `yaml:"data"`
+			}
+			var data helmSecretDataV3
+			err := yaml.Unmarshal(secretBytes, &data)
+			if err != nil {
+				return "", fmt.Errorf("Could not unmarshall helm secret plugin V3 decrypted file to a yaml string\n"+
+					"You may consider upgrading your helm secrets plugin to >4.0.\n %s", err.Error())
+			}
+			secretBytes = []byte(data.Data)
+		}
+
 		secret.bytes = secretBytes
 	} else {
 		// Cache hit

pkg/state/create.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"slices"
+	"strings"
 
 	"github.com/helmfile/vals"
 	"github.com/imdario/mergo"
@@ -18,8 +20,9 @@ import (
 )
 
 const (
-	DefaultHelmBinary      = "helm"
-	DefaultKustomizeBinary = "kustomize"
+	DefaultHelmBinary       = "helm"
+	DefaultKustomizeBinary  = "kustomize"
+	DefaultHCLFileExtension = ".hcl"
 )
 
 type StateLoadError struct {
@@ -233,17 +236,19 @@ func (c *StateCreator) loadBases(envValues, overrodeEnv *environment.Environment
 
 // nolint: unparam
 func (c *StateCreator) loadEnvValues(st *HelmState, name string, failOnMissingEnv bool, ctxEnv, overrode *environment.Environment) (*environment.Environment, error) {
-	envVals := map[string]any{}
+	secretVals := map[string]any{}
+	valuesVals := map[string]any{}
 	envSpec, ok := st.Environments[name]
+	decryptedFiles := []string{}
 	if ok {
 		var err error
-		envVals, err = st.loadValuesEntries(envSpec.MissingFileHandler, envSpec.Values, c.remote, ctxEnv, name)
-		if err != nil {
-			return nil, err
-		}
-
+		// To keep supporting the secrets entries having precedence over the values
+		// This require to be done in 2 steps for HCL encrypted file support :
+		// 1. Get the Secrets
+		// 2. Merge the secrets with the envValues after
+		// Also makes the fail +- faster as it's trying to decrypt before loading values
+		var envSecretFiles []string
 		if len(envSpec.Secrets) > 0 {
-			var envSecretFiles []string
 			for _, urlOrPath := range envSpec.Secrets {
 				resolved, skipped, err := st.storage().resolveFile(envSpec.MissingFileHandler, "environment values", urlOrPath, envSpec.MissingFileHandlerConfig.resolveFileOptions()...)
 				if err != nil {
@@ -252,18 +257,40 @@ func (c *StateCreator) loadEnvValues(st *HelmState, name string, failOnMissingEn
 				if skipped {
 					continue
 				}
-
 				envSecretFiles = append(envSecretFiles, resolved...)
 			}
-			if err = c.scatterGatherEnvSecretFiles(st, envSecretFiles, envVals); err != nil {
+			keepSecretFilesExtensions := []string{DefaultHCLFileExtension}
+			decryptedFiles, err = c.scatterGatherEnvSecretFiles(st, envSecretFiles, secretVals, keepSecretFilesExtensions)
+			if err != nil {
 				return nil, err
 			}
+
+			defer func() {
+				for _, file := range decryptedFiles {
+					if err := c.fs.DeleteFile(file); err != nil {
+						c.logger.Warnf("failed removing decrypted file %s: %w", file, err)
+					}
+				}
+			}()
+		}
+		var valuesFiles []any
+		for _, f := range decryptedFiles {
+			valuesFiles = append(valuesFiles, f)
+		}
+		envValuesEntries := append(valuesFiles, envSpec.Values...)
+		valuesVals, err = st.loadValuesEntries(envSpec.MissingFileHandler, envValuesEntries, c.remote, ctxEnv, name)
+		if err != nil {
+			return nil, err
+		}
+
+		if err = mergo.Merge(&valuesVals, &secretVals, mergo.WithOverride); err != nil {
+			return nil, err
 		}
 	} else if ctxEnv == nil && overrode == nil && name != DefaultEnv && failOnMissingEnv {
 		return nil, &UndefinedEnvError{Env: name}
 	}
 
-	newEnv := &environment.Environment{Name: name, Values: envVals, KubeContext: envSpec.KubeContext}
+	newEnv := &environment.Environment{Name: name, Values: valuesVals, KubeContext: envSpec.KubeContext}
 
 	if ctxEnv != nil {
 		intCtxEnv := *ctxEnv
@@ -288,9 +315,14 @@ func (c *StateCreator) loadEnvValues(st *HelmState, name string, failOnMissingEn
 	return newEnv, nil
 }
 
-func (c *StateCreator) scatterGatherEnvSecretFiles(st *HelmState, envSecretFiles []string, envVals map[string]any) error {
+// For all keepFileExtensions, the decrypted files will be retained
+// with the specified extensions
+// They will not be parsed nor added to the envVals.
+// Only their decrypted filePath will be returned
+// Up to the caller to remove them
+func (c *StateCreator) scatterGatherEnvSecretFiles(st *HelmState, envSecretFiles []string, envVals map[string]any, keepFileExtensions []string) ([]string, error) {
 	var errs []error
-
+	var decryptedFilesKeeper []string
 	helm := c.getHelm(st)
 	inputs := envSecretFiles
 	inputsSize := len(inputs)
@@ -325,31 +357,39 @@ func (c *StateCreator) scatterGatherEnvSecretFiles(st *HelmState, envSecretFiles
 					results <- secretResult{secret.id, nil, err, secret.path}
 					continue
 				}
-
+				for _, ext := range keepFileExtensions {
+					if strings.HasSuffix(secret.path, ext) {
+						decryptedFilesKeeper = append(decryptedFilesKeeper, decFile)
+					}
+				}
 				// nolint: staticcheck
 				defer func() {
-					if err := c.fs.DeleteFile(decFile); err != nil {
-						c.logger.Warnf("removing decrypted file %s: %w", decFile, err)
+					if !slices.Contains(decryptedFilesKeeper, decFile) {
+						if err := c.fs.DeleteFile(decFile); err != nil {
+							c.logger.Warnf("removing decrypted file %s: %w", decFile, err)
+						}
 					}
 				}()
-
-				bytes, err := c.fs.ReadFile(decFile)
-				if err != nil {
-					results <- secretResult{secret.id, nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", secret.path, err), secret.path}
-					continue
-				}
-				m := map[string]any{}
-				if err := yaml.Unmarshal(bytes, &m); err != nil {
-					results <- secretResult{secret.id, nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", secret.path, err), secret.path}
-					continue
-				}
-				// All the nested map key should be string. Otherwise we get strange errors due to that
-				// mergo or reflect is unable to merge map[any]any with map[string]any or vice versa.
-				// See https://github.com/roboll/helmfile/issues/677
-				vals, err := maputil.CastKeysToStrings(m)
-				if err != nil {
-					results <- secretResult{secret.id, nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", secret.path, err), secret.path}
-					continue
+				var vals map[string]any
+				if !slices.Contains(decryptedFilesKeeper, decFile) {
+					bytes, err := c.fs.ReadFile(decFile)
+					if err != nil {
+						results <- secretResult{secret.id, nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", secret.path, err), secret.path}
+						continue
+					}
+					m := map[string]any{}
+					if err := yaml.Unmarshal(bytes, &m); err != nil {
+						results <- secretResult{secret.id, nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", secret.path, err), secret.path}
+						continue
+					}
+					// All the nested map key should be string. Otherwise we get strange errors due to that
+					// mergo or reflect is unable to merge map[any]any with map[string]any or vice versa.
+					// See https://github.com/roboll/helmfile/issues/677
+					vals, err = maputil.CastKeysToStrings(m)
+					if err != nil {
+						results <- secretResult{secret.id, nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", secret.path, err), secret.path}
+						continue
+					}
 				}
 				results <- secretResult{secret.id, vals, nil, secret.path}
 			}
@@ -379,9 +419,9 @@ func (c *StateCreator) scatterGatherEnvSecretFiles(st *HelmState, envSecretFiles
 		for _, err := range errs {
 			st.logger.Error(err)
 		}
-		return fmt.Errorf("failed loading environment secrets with %d errors", len(errs))
+		return decryptedFilesKeeper, fmt.Errorf("failed loading environment secrets with %d errors", len(errs))
 	}
-	return nil
+	return decryptedFilesKeeper, nil
 }
 
 func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []any, remote *remote.Remote, ctxEnv *environment.Environment, envName string) (map[string]any, error) {

pkg/state/envvals_loader.go

@@ -3,12 +3,14 @@ package state
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 
 	"github.com/imdario/mergo"
 	"go.uber.org/zap"
 
 	"github.com/helmfile/helmfile/pkg/environment"
 	"github.com/helmfile/helmfile/pkg/filesystem"
+	"github.com/helmfile/helmfile/pkg/hcllang"
 	"github.com/helmfile/helmfile/pkg/maputil"
 	"github.com/helmfile/helmfile/pkg/remote"
 	"github.com/helmfile/helmfile/pkg/tmpl"
@@ -35,7 +37,11 @@ func NewEnvironmentValuesLoader(storage *Storage, fs *filesystem.FileSystem, log
 }
 
 func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *string, valuesEntries []any, ctxEnv *environment.Environment, envName string) (map[string]any, error) {
-	result := map[string]any{}
+	var (
+		result    = map[string]any{}
+		hclLoader = hcllang.NewHCLLoader(ld.fs, ld.logger)
+		err       error
+	)
 
 	for _, entry := range valuesEntries {
 		maps := []any{}
@@ -49,7 +55,6 @@ func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *str
 			if skipped {
 				continue
 			}
-
 			for _, f := range files {
 				var env environment.Environment
 				if ctxEnv == nil {
@@ -57,38 +62,61 @@ func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *str
 				} else {
 					env = *ctxEnv
 				}
-
-				tmplData := NewEnvironmentTemplateData(env, "", map[string]any{})
-				r := tmpl.NewFileRenderer(ld.fs, filepath.Dir(f), tmplData)
-				bytes, err := r.RenderToBytes(f)
-				if err != nil {
-					return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", f, err)
+				if strings.HasSuffix(f, ".hcl") {
+					hclLoader.AddFile(f)
+				} else {
+					tmplData := NewEnvironmentTemplateData(env, "", map[string]any{})
+					r := tmpl.NewFileRenderer(ld.fs, filepath.Dir(f), tmplData)
+					bytes, err := r.RenderToBytes(f)
+					if err != nil {
+						return nil, fmt.Errorf("failed to load environment values file \"%s\": %v", f, err)
+					}
+					m := map[string]any{}
+					if err := yaml.Unmarshal(bytes, &m); err != nil {
+						return nil, fmt.Errorf("failed to load environment values file \"%s\": %v\n\nOffending YAML:\n%s", f, err, bytes)
+					}
+					maps = append(maps, m)
+					ld.logger.Debugf("envvals_loader: loaded %s:%v", strOrMap, m)
 				}
-				m := map[string]any{}
-				if err := yaml.Unmarshal(bytes, &m); err != nil {
-					return nil, fmt.Errorf("failed to load environment values file \"%s\": %v\n\nOffending YAML:\n%s", f, err, bytes)
-				}
-				maps = append(maps, m)
-				ld.logger.Debugf("envvals_loader: loaded %s:%v", strOrMap, m)
 			}
 		case map[any]any, map[string]any:
 			maps = append(maps, strOrMap)
 		default:
 			return nil, fmt.Errorf("unexpected type of value: value=%v, type=%T", strOrMap, strOrMap)
 		}
-		for _, m := range maps {
-			// All the nested map key should be string. Otherwise we get strange errors due to that
-			// mergo or reflect is unable to merge map[any]any with map[string]any or vice versa.
-			// See https://github.com/roboll/helmfile/issues/677
-			vals, err := maputil.CastKeysToStrings(m)
-			if err != nil {
-				return nil, err
-			}
-			if err := mergo.Merge(&result, &vals, mergo.WithOverride); err != nil {
-				return nil, fmt.Errorf("failed to merge %v: %v", m, err)
-			}
+
+		result, err = mapMerge(result, maps)
+		if err != nil {
+			return nil, err
 		}
 	}
-
+	maps := []any{}
+	if hclLoader.Length() > 0 {
+		m, err := hclLoader.HCLRender()
+		if err != nil {
+			return nil, err
+		}
+		maps = append(maps, m)
+	}
+	result, err = mapMerge(result, maps)
+	if err != nil {
+		return nil, err
+	}
 	return result, nil
 }
+
+func mapMerge(dest map[string]any, maps []any) (map[string]any, error) {
+	for _, m := range maps {
+		// All the nested map key should be string. Otherwise we get strange errors due to that
+		// mergo or reflect is unable to merge map[any]any with map[string]any or vice versa.
+		// See https://github.com/roboll/helmfile/issues/677
+		vals, err := maputil.CastKeysToStrings(m)
+		if err != nil {
+			return nil, err
+		}
+		if err := mergo.Merge(&dest, &vals, mergo.WithOverride); err != nil {
+			return nil, fmt.Errorf("failed to merge %v: %v", m, err)
+		}
+	}
+	return dest, nil
+}

pkg/state/envvals_loader_test.go

@@ -187,3 +187,44 @@ func TestEnvValsLoad_OverwriteEmptyValue_Issue1168(t *testing.T) {
 		t.Errorf(diff)
 	}
 }
+
+func TestEnvValsLoad_MultiHCL(t *testing.T) {
+	l := newLoader()
+
+	actual, err := l.LoadEnvironmentValues(nil, []any{"testdata/values.7.hcl", "testdata/values.8.hcl"}, nil, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := map[string]any{
+		"a": "a",
+		"b": "b",
+		"c": "ab",
+		"map": map[string]any{
+			"a": "a",
+		},
+		"list": []any{
+			"b",
+		},
+		"nestedmap": map[string]any{
+			"submap": map[string]any{
+				"subsubmap": map[string]any{
+					"hello": "ab",
+				},
+			},
+		},
+		"ternary":          true,
+		"fromMap":          "aab",
+		"expressionInText": "yes",
+		"insideFor":        "b",
+		"multi_block":      "block",
+		"block":            "block",
+		"crossfile":        "crossfile var",
+		"crossfile_var":    "crossfile var",
+		"localRef":         "localInValues7",
+	}
+
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf(diff)
+	}
+}

pkg/state/testdata/values.7.hcl

@@ -0,0 +1,42 @@
+locals {
+  localInValues7File = "localInValues7"
+}
+
+values {
+  a = "a"
+  b = "b"
+  c = "${hv.a}${hv.b}"
+
+  map = {
+    "a" = "${hv.a}"
+  } 
+
+  list = [
+    hv.b
+  ]
+
+  nestedmap = {
+    submap = {
+      subsubmap = {
+        hello = hv.c
+      }
+    }
+  }
+
+  ternary = true ? true : false
+
+  fromMap = "${hv.map.a}${hv.nestedmap.submap.subsubmap.hello}"
+
+  expressionInText = "%{if hv.ternary }yes%{else}no%{endif}"
+  insideFor = "%{for i in hv.list }${i}%{endfor}"
+
+  multi_block = hv.block
+
+  crossfile = hv.crossfile_var
+
+  localRef = local.localInValues7File
+}
+
+values {
+  block = "block"
+}

pkg/state/testdata/values.8.hcl

@@ -0,0 +1,3 @@
+values {
+  crossfile_var = "crossfile var"
+}