Add the ability to specify a lock file (#432)

Allow configuring the lockfile in the state. This makes it possible for
example maintain a lock per environment.

Signed-off-by: Lassi Pölönen <lassi.polonen@iki.fi>

Signed-off-by: Lassi Pölönen <lassi.polonen@iki.fi>

docs/advanced-features.md

@@ -312,3 +312,21 @@ releases:
   - chart: oci://my-oci-registry/helm-repo/envoy
     version: 1.5
 ```
+
+### Lockfile per environment
+
+In some cases it can be handy for CI/CD pipelines to be able to roll out updates gradually for environments, such as staging and production while using the same
+set of charts. This can be achieved by using `lockFilePath` in combination with environments, such as:
+
+```yaml
+environments:
+  staging:
+  production
+
+---
+lockFilePath: .helmfile.{{ .Environment.Name}}.lock
+
+releases:
+- name: myapp
+  chart: charts/myapp
+```

docs/index.md

@@ -169,6 +169,10 @@ repositories:
 # Path to alternative helm binary (--helm-binary)
 helmBinary: path/to/helm3
 
+
+# Path to alternative lock file. The default is <state file name>.lock, i.e for helmfile.yaml it's helmfile.lock.
+lockFilePath: path/to/lock.file
+
 # Default values to set for args along with dedicated keys that can be set by contributors, cli args take precedence over these.
 # In other words, unset values results in no flags passed to helm.
 # See the helm usage (helm SUBCOMMAND -h) for more info on default values when those flags aren't provided.
@@ -568,6 +572,8 @@ All the other `helmfile` sub-commands like `sync` use chart versions recorded in
 
 For example, the lock file for a helmfile state file named `helmfile.1.yaml` will be `helmfile.1.lock`. The lock file for a local chart would be `requirements.lock`, which is the same as `helm`.
 
+The lock file can be changed using `lockFilePath` in helm state, which makes it possible to for example have a different lock file per environment via templating.
+
 It is recommended to version-control all the lock files, so that they can be used in the production deployment pipeline for extra reproducibility.
 
 To bring in chart updates systematically, it would also be a good idea to run `helmfile deps` regularly, test it, and then update the lock files in the version-control system.

pkg/app/desired_state_file_loader.go

@@ -36,6 +36,8 @@ type desiredStateLoader struct {
 	remote      *remote.Remote
 	logger      *zap.SugaredLogger
 	valsRuntime vals.Evaluator
+
+	lockFilePath string
 }
 
 func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, error) {
@@ -163,7 +165,7 @@ func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *e
 }
 
 func (a *desiredStateLoader) underlying() *state.StateCreator {
-	c := state.NewCreator(a.logger, a.fs, a.valsRuntime, a.getHelm, a.overrideHelmBinary, a.remote, a.enableLiveOutput)
+	c := state.NewCreator(a.logger, a.fs, a.valsRuntime, a.getHelm, a.overrideHelmBinary, a.remote, a.enableLiveOutput, a.lockFilePath)
 	c.LoadFile = a.loadFile
 	return c
 }

pkg/state/chart_dependency.go

@@ -145,7 +145,7 @@ func (st *HelmState) mergeLockedDependencies() (*HelmState, error) {
 		return st, nil
 	}
 
-	depMan := NewChartDependencyManager(filename, st.logger)
+	depMan := NewChartDependencyManager(filename, st.logger, st.LockFile)
 
 	if st.fs.ReadFile != nil {
 		depMan.readFile = st.fs.ReadFile
@@ -258,7 +258,7 @@ func getUnresolvedDependenciess(st *HelmState) (string, *UnresolvedDependencies,
 }
 
 func updateDependencies(st *HelmState, shell helmexec.DependencyUpdater, unresolved *UnresolvedDependencies, filename, wd string) (*HelmState, error) {
-	depMan := NewChartDependencyManager(filename, st.logger)
+	depMan := NewChartDependencyManager(filename, st.logger, st.LockFile)
 
 	_, err := depMan.Update(shell, wd, unresolved)
 	if err != nil {
@@ -271,6 +271,8 @@ func updateDependencies(st *HelmState, shell helmexec.DependencyUpdater, unresol
 type chartDependencyManager struct {
 	Name string
 
+	lockFilePath string
+
 	logger *zap.SugaredLogger
 
 	readFile  func(string) ([]byte, error)
@@ -278,17 +280,22 @@ type chartDependencyManager struct {
 }
 
 // nolint: golint
-func NewChartDependencyManager(name string, logger *zap.SugaredLogger) *chartDependencyManager {
+func NewChartDependencyManager(name string, logger *zap.SugaredLogger, lockFilePath string) *chartDependencyManager {
 	return &chartDependencyManager{
 		Name:         name,
 		readFile:     os.ReadFile,
 		writeFile:    os.WriteFile,
 		logger:       logger,
+		lockFilePath: lockFilePath,
 	}
 }
 
 func (m *chartDependencyManager) lockFileName() string {
+	if m.lockFilePath != "" {
+		return m.lockFilePath
+	} else {
 		return fmt.Sprintf("%s.lock", m.Name)
+	}
 }
 
 func (m *chartDependencyManager) Update(shell helmexec.DependencyUpdater, wd string, unresolved *UnresolvedDependencies) (*ResolvedDependencies, error) {
@@ -334,9 +341,9 @@ func (m *chartDependencyManager) updateHelm2(shell helmexec.DependencyUpdater, w
 
 func (m *chartDependencyManager) doUpdate(chartLockFile string, unresolved *UnresolvedDependencies, shell helmexec.DependencyUpdater, wd string) (*ResolvedDependencies, error) {
 	// Generate `requirements.lock` of the temporary local chart by coping `<basename>.lock`
-	lockFile := m.lockFileName()
+	lockFilePath := m.lockFileName()
 
-	originalLockFileContent, err := m.readBytes(lockFile)
+	originalLockFileContent, err := m.readBytes(lockFilePath)
 	if err != nil && !os.IsNotExist(err) {
 		return nil, err
 	}
@@ -394,7 +401,7 @@ func (m *chartDependencyManager) doUpdate(chartLockFile string, unresolved *Unre
 	}
 
 	// Commit the lock file if and only if everything looks ok
-	if err := m.writeBytes(lockFile, updatedLockFileContent); err != nil {
+	if err := m.writeBytes(lockFilePath, updatedLockFileContent); err != nil {
 		return nil, err
 	}
 

pkg/state/create.go

@@ -58,9 +58,11 @@ type StateCreator struct {
 	enableLiveOutput bool
 
 	remote *remote.Remote
+
+	lockFile string
 }
 
-func NewCreator(logger *zap.SugaredLogger, fs *filesystem.FileSystem, valsRuntime vals.Evaluator, getHelm func(*HelmState) helmexec.Interface, overrideHelmBinary string, remote *remote.Remote, enableLiveOutput bool) *StateCreator {
+func NewCreator(logger *zap.SugaredLogger, fs *filesystem.FileSystem, valsRuntime vals.Evaluator, getHelm func(*HelmState) helmexec.Interface, overrideHelmBinary string, remote *remote.Remote, enableLiveOutput bool, lockFile string) *StateCreator {
 	return &StateCreator{
 		logger: logger,
 
@@ -73,6 +75,8 @@ func NewCreator(logger *zap.SugaredLogger, fs *filesystem.FileSystem, valsRuntim
 		enableLiveOutput:   enableLiveOutput,
 
 		remote: remote,
+
+		lockFile: lockFile,
 	}
 }
 
@@ -84,6 +88,8 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState,
 	state.FilePath = file
 	state.basePath = baseDir
 
+	state.LockFile = c.lockFile
+
 	decoder := yaml.NewDecoder(bytes.NewReader(content))
 
 	decoder.KnownFields(c.Strict)

pkg/state/create_test.go

@@ -83,7 +83,7 @@ func (testEnv stateTestEnv) MustLoadStateWithEnableLiveOutput(t *testing.T, file
 	}
 
 	r := remote.NewRemote(logger, testFs.Cwd, testFs.ToFileSystem())
-	state, err := NewCreator(logger, testFs.ToFileSystem(), nil, nil, "", r, enableLiveOutput).
+	state, err := NewCreator(logger, testFs.ToFileSystem(), nil, nil, "", r, enableLiveOutput, "").
 		ParseAndLoad([]byte(yamlContent), filepath.Dir(file), file, envName, true, nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -153,7 +153,7 @@ releaseNamespace: mynamespace
 	env := environment.Environment{
 		Name: "production",
 	}
-	state, err := NewCreator(logger, testFs.ToFileSystem(), nil, nil, "", r, false).
+	state, err := NewCreator(logger, testFs.ToFileSystem(), nil, nil, "", r, false, "").
 		ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", true, &env)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -240,7 +240,7 @@ overrideNamespace: myns
 	testFs.Cwd = "/example/path/to"
 
 	r := remote.NewRemote(logger, testFs.Cwd, testFs.ToFileSystem())
-	state, err := NewCreator(logger, testFs.ToFileSystem(), nil, nil, "", r, false).
+	state, err := NewCreator(logger, testFs.ToFileSystem(), nil, nil, "", r, false, "").
 		ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", true, nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)

pkg/state/state.go

@@ -79,6 +79,8 @@ type ReleaseSetSpec struct {
 	// non-existent path. The default behavior is to print a warning. Note the
 	// differing default compared to other MissingFileHandlers.
 	MissingFileHandler string `yaml:"missingFileHandler,omitempty"`
+
+	LockFile string `yaml:"lockFilePath,omitempty"`
 }
 
 // HelmState structure for the helmfile

pkg/state/state_test.go

@@ -2030,6 +2030,56 @@ func TestHelmState_ResolveDeps_NoLockFile(t *testing.T) {
 	}
 }
 
+func TestHelmState_ResolveDeps_NoLockFile_WithCustomLockFile(t *testing.T) {
+	logger := helmexec.NewLogger(io.Discard, "debug")
+	state := &HelmState{
+		basePath: "/src",
+		FilePath: "/src/helmfile.yaml",
+		ReleaseSetSpec: ReleaseSetSpec{
+			LockFile: "custom-lock-file",
+			Releases: []ReleaseSpec{
+				{
+					Chart: "./..",
+				},
+				{
+					Chart: "../examples",
+				},
+				{
+					Chart: "../../helmfile",
+				},
+				{
+					Chart: "published",
+				},
+				{
+					Chart: "published/deeper",
+				},
+				{
+					Chart: "stable/envoy",
+				},
+			},
+			Repositories: []RepositorySpec{
+				{
+					Name: "stable",
+					URL:  "https://kubernetes-charts.storage.googleapis.com",
+				},
+			},
+		},
+		logger: logger,
+		fs: &filesystem.FileSystem{
+			ReadFile: func(f string) ([]byte, error) {
+				if f != "custom-lock-file" {
+					return nil, fmt.Errorf("stub: unexpected file: %s", f)
+				}
+				return nil, os.ErrNotExist
+			},
+		},
+	}
+
+	_, err := state.ResolveDeps()
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+}
 func TestHelmState_ReleaseStatuses(t *testing.T) {
 	tests := []struct {
 		name     string

test/e2e/template/helmfile/snapshot_test.go

@@ -134,13 +134,8 @@ func TestHelmfileTemplateWithBuildCommand(t *testing.T) {
 					if !c.IsDir() {
 						t.Fatalf("%s is not a directory", c)
 					}
+					chartName, chartVersion := execHelmShowChart(t, chartPath)
 					tgzFile := execHelmPackage(t, chartPath)
-
-					// Extract chart version from the name of chart package archival
-					chartName := c.Name()
-					chartNameWithVersion := strings.TrimSuffix(filepath.Base(tgzFile), filepath.Ext(tgzFile))
-					chartVersion := strings.TrimPrefix(chartNameWithVersion, fmt.Sprintf("%s-", chartName))
-
 					chartDigest, err := execHelmPush(t, tgzFile, fmt.Sprintf("oci://localhost:%d/myrepo", hostPort))
 					require.NoError(t, err, "Unable to run helm push to local registry: %v", err)
 
@@ -235,6 +230,23 @@ func execDocker(t *testing.T, args ...string) {
 	}
 }
 
+func execHelmShowChart(t *testing.T, localChart string) (string, string) {
+	t.Helper()
+
+	name, version := "", ""
+	out := execHelm(t, "show", "chart", localChart)
+	sc := bufio.NewScanner(strings.NewReader(out))
+	for sc.Scan() {
+		if strings.HasPrefix(sc.Text(), "name:") {
+			name = strings.TrimPrefix(sc.Text(), "name: ")
+		}
+		if strings.HasPrefix(sc.Text(), "version:") {
+			version = strings.TrimPrefix(sc.Text(), "version: ")
+		}
+	}
+	return name, version
+}
+
 func execHelmPackage(t *testing.T, localChart string) string {
 	t.Helper()
 

test/e2e/template/helmfile/testdata/charts/raw-0.0.1/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: raw
+description: A Helm chart for Kubernetes
+type: application
+version: 0.0.1
+appVersion: "1.16.0"

test/e2e/template/helmfile/testdata/charts/raw-0.0.1/README.md

@@ -0,0 +1 @@
+A copy of chart ../raw, but older version to test locking.

test/e2e/template/helmfile/testdata/charts/raw-0.1.0/.gitignore

@@ -0,0 +1 @@
+*.tgz

test/e2e/template/helmfile/testdata/charts/raw-0.1.0/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

test/e2e/template/helmfile/testdata/charts/raw-0.1.0/templates/resources.yaml

@@ -0,0 +1,6 @@
+{{- range $i, $r := $.Values.templates }}
+{{- if gt $i 0 }}
+---
+{{- end }}
+{{- (tpl $r $) }}
+{{- end }}

test/e2e/template/helmfile/testdata/charts/raw-0.1.0/values.yaml

@@ -0,0 +1,48 @@
+templates: []
+
+##
+## Example: Uncomment the below and run `helm template ./`:
+##
+#
+# templates:
+# - |
+#   apiVersion: v1
+#   kind: ConfigMap
+#   metadata:
+#     name: {{ .Release.Name }}-1
+#     namespace: {{ .Release.Namespace }}
+#   data:
+#     foo: {{ .Values.foo }}
+# - |
+#   apiVersion: v1
+#   kind: ConfigMap
+#   metadata:
+#     name: {{ .Release.Name }}-2
+#     namespace: {{ .Release.Namespace }}
+#   data:
+#     foo: {{ .Values.foo }}
+# values:
+#   foo: FOO
+#
+##
+## Expected Output:
+##
+#
+# ---
+# # Source: raw/templates/resources.yaml
+# apiVersion: v1
+# kind: ConfigMap
+# metadata:
+#   name: release-name-1
+#   namespace: default
+# data:
+#   foo:
+# ---
+# # Source: raw/templates/resources.yaml
+# apiVersion: v1
+# kind: ConfigMap
+# metadata:
+#   name: release-name-2
+#   namespace: default
+# data:
+#   foo:

test/e2e/template/helmfile/testdata/snapshot/chart_need/input.yaml

@@ -4,7 +4,7 @@ repositories:
 
 releases:
 - name: foo
-  chart: ../../charts/raw
+  chart: ../../charts/raw-0.1.0
   values:
   - templates:
     - |

test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/input.yaml

@@ -4,7 +4,7 @@ repositories:
 
 releases:
 - name: foo
-  chart: ../../charts/raw
+  chart: ../../charts/raw-0.1.0
   values:
   - templates:
     - |

test/e2e/template/helmfile/testdata/snapshot/oci_need/input.yaml

@@ -1,6 +1,6 @@
 releases:
 - name: foo
-  chart: ../../charts/raw
+  chart: ../../charts/raw-0.1.0
   values:
   - templates:
     - |

test/e2e/template/helmfile/testdata/snapshot/templated_lockfile/config.yaml

@@ -0,0 +1,8 @@
+localChartRepoServer:
+  enabled: true
+  port: 18080
+chartifyTempDir: temp1
+helmfileArgs:
+- --environment
+- prod
+- template

test/e2e/template/helmfile/testdata/snapshot/templated_lockfile/input.yaml

@@ -0,0 +1,18 @@
+repositories:
+- name: myrepo
+  url: http://localhost:18080/
+
+environments:
+  prod:
+  staging:
+
+---
+lockFilePath: test-lock-file-{{ .Environment.Name }}
+
+releases:
+- name: raw
+  chart: myrepo/raw
+  values:
+  - templates:
+    - |
+      chartVersion: {{`{{ .Chart.Version }}`}}

test/e2e/template/helmfile/testdata/snapshot/templated_lockfile/output.yaml

@@ -0,0 +1,8 @@
+Adding repo myrepo http://localhost:18080/
+"myrepo" has been added to your repositories
+
+Templating release=raw, chart=myrepo/raw
+---
+# Source: raw/templates/resources.yaml
+chartVersion: 0.0.1
+

test/e2e/template/helmfile/testdata/snapshot/templated_lockfile/test-lock-file-prod

@@ -0,0 +1,7 @@
+version: 0.0.0-dev
+dependencies:
+  - name: raw
+    repository: http://localhost:18080/
+    version: 0.0.1
+digest: sha256:5401817b653c4eeb186cbfbb8d77dda6b72f84a548fc9cd128cbd478d5b2e705
+generated: "2022-10-12T20:17:15.98786845+03:00"