Merge pull request #239 from helmfile/local-oci-integration-test

Add integration test for #238 with local docker registry as a OCI-based helm chart repo

test/e2e/template/helmfile/README.md

@@ -0,0 +1,10 @@
+This directory contains a set of Go test source and testdata
+to test the helmfile template's rendering result by calling `helmfile build` or `helmfile template` on test input
+and comparing the output against the snapshot.
+
+The `testdata` directory is composed of:
+
+- `charts`: The Helm charts used from within test helmfile configs (`snapshpt/*/input.yaml`) as local charts and remote charts
+- `snapshot/$NAME/input.yaml`: The input helmfile config for the test case of `$NAME`
+- `snapshot/$NAME/output.yaml`: The expected output of the helmfile command
+- `snapshot/$NAME/config.yaml`: The snapshot test configuration file. See the `Config` struct defined in `snapshot_test.go` for more information

test/e2e/template/helmfile/snapshot_test.go

@@ -2,21 +2,34 @@ package helmfile
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
 )
 
 func TestHelmfileTemplateWithBuildCommand(t *testing.T) {
+	type Config struct {
+		LocalDockerRegistry struct {
+			Enabled bool `yaml:"enabled"`
+			Port    int  `yaml:"port"`
+		} `yaml:"localDockerRegistry"`
+		ChartifyTempDir string   `yaml:"chartifyTempDir"`
+		HelmfileArgs    []string `yaml:"helmfileArgs"`
+	}
+
 	_, filename, _, _ := runtime.Caller(0)
 	projectRoot := filepath.Join(filepath.Dir(filename), "..", "..", "..", "..")
 	helmfileBin := filepath.Join(projectRoot, "helmfile")
 	testdataDir := "testdata/snapshot"
+	chartsDir := "testdata/charts"
 
 	entries, err := os.ReadDir(testdataDir)
 	require.NoError(t, err)
@@ -28,23 +41,152 @@ func TestHelmfileTemplateWithBuildCommand(t *testing.T) {
 
 		name := e.Name()
 
-		t.Run(name, func(t *testing.T) {
+		wd, err := os.Getwd()
-			inputFile := filepath.Join(testdataDir, name, "input.yaml")
-
-			want, err := os.ReadFile(filepath.Join(testdataDir, name, "output.yaml"))
 		require.NoError(t, err)
 
+		// We read the config from `testdata/snapshot/$CASE_NAME/config.yaml`.
+		// It's optional so the test won't fail even if the config file does not exist.
+
+		var config Config
+
+		configFile := filepath.Join(testdataDir, name, "config.yaml")
+		if configData, err := os.ReadFile(configFile); err == nil {
+			if err := yaml.Unmarshal(configData, &config); err != nil {
+				t.Fatalf("Unable to load %s: %v", configFile, err)
+			}
+		}
+
+		// We run `helmfile build` by default.
+		// If you want to test `helmfile template`, set the following in the config.yaml:
+		//
+		// helmfileArgs:
+		// - template
+		helmfileArgs := config.HelmfileArgs
+		if len(helmfileArgs) == 0 {
+			helmfileArgs = append(helmfileArgs, "build")
+		}
+
+		t.Run(name, func(t *testing.T) {
+			// Use the specific chartify tempdir for easy debugging and the test reproducibility.
+			// We do snapshot testing in this test. The default chartify tempdir is a random directory created within the os temp dir.
+			// Without making it a static path, it's unnecessarily hard to snapshot test it, as the dir path embedded in the output changes
+			// on each test run.
+			chartifyTempDir := config.ChartifyTempDir
+			if chartifyTempDir == "" {
+				chartifyTempDir = "chartify_temp"
+			}
+
+			// We set the envvar probided by chartify, CHARTIFY_TEMPDIR, to make the tempdir static.
+			chartifyTempDir = filepath.Join(wd, chartifyTempDir)
+			t.Setenv("CHARTIFY_TEMPDIR", chartifyTempDir)
+			// Ensure there's no dangling and remaining tempdir from the previous run
+			if err := os.RemoveAll(chartifyTempDir); err != nil {
+				t.Fatalf("unable to remove chartify temp dir %q: %v", chartifyTempDir, err)
+			}
+			// Ensure it's removed on test completion
+			t.Cleanup(func() {
+				if err := os.RemoveAll(chartifyTempDir); err != nil {
+					t.Fatalf("unable to remove chartify temp dir %q: %v", chartifyTempDir, err)
+				}
+			})
+
+			// If localDockerRegistry.enabled is set to `true`,
+			// run the docker registry v2 and push the test charts to the registry
+			// so that it can be accessed by helm and helmfile as a oci registry based chart repository.
+			if config.LocalDockerRegistry.Enabled {
+				containerName := "helmfile_docker_registry"
+
+				hostPort := config.LocalDockerRegistry.Port
+				if hostPort < 0 {
+					hostPort = 5000
+				}
+
+				execDocker(t, "run", "-d", "-p", fmt.Sprintf("%d:5000", hostPort), "--restart=always", "--name", containerName, "registry:2")
+				t.Cleanup(func() {
+					execDocker(t, "stop", containerName)
+					execDocker(t, "rm", containerName)
+				})
+
+				// We helm-package and helm-push every test chart saved in the ./testdata/charts directory
+				// to the local registry, so that they can be accessed by helmfile and helm invoked while testing.
+				charts, err := os.ReadDir(chartsDir)
+				require.NoError(t, err)
+
+				for _, c := range charts {
+					chartPath := filepath.Join(chartsDir, c.Name())
+					if !c.IsDir() {
+						t.Fatalf("%s is not a directory", c)
+					}
+					tgzFile := execHelmPackage(t, chartPath)
+					_ = execHelm(t, "push", tgzFile, fmt.Sprintf("oci://localhost:%d/myrepo", hostPort))
+				}
+			}
+
+			inputFile := filepath.Join(testdataDir, name, "input.yaml")
+			outputFile := filepath.Join(testdataDir, name, "output.yaml")
+
 			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 			defer cancel()
 
-			cmd := exec.CommandContext(ctx, helmfileBin, "-f", inputFile, "build")
+			args := []string{"-f", inputFile}
+			args = append(args, helmfileArgs...)
+			cmd := exec.CommandContext(ctx, helmfileBin, args...)
 			got, err := cmd.CombinedOutput()
 			if err != nil {
-				t.Logf("%s", string(got))
+				t.Logf("Output from %v: %s", args, string(got))
 			}
-			require.NoError(t, err)
 
-			require.Equal(t, string(want), string(got))
+			gotStr := string(got)
+			gotStr = strings.ReplaceAll(gotStr, fmt.Sprintf("chart=%s", wd), "chart=$WD")
+
+			require.NoError(t, err, "Unable to run helmfile with args %v", args)
+
+			if stat, _ := os.Stat(outputFile); stat != nil {
+				want, err := os.ReadFile(outputFile)
+				require.NoError(t, err)
+				require.Equal(t, string(want), gotStr)
+			} else {
+				// To update the test golden image(output.yaml), just remove it and rerun this test.
+				// We automatically capture the output to `output.yaml` in the test case directory
+				// when the output.yaml doesn't exist.
+				require.NoError(t, os.WriteFile(outputFile, []byte(gotStr), 0664))
+			}
 		})
 	}
 }
+
+func execDocker(t *testing.T, args ...string) {
+	t.Helper()
+
+	docker := exec.Command("docker", args...)
+	out, err := docker.CombinedOutput()
+	if err != nil {
+		t.Logf("Docker output: %s", string(out))
+		t.Fatalf("Unable to run docker: %v", err)
+	}
+}
+
+func execHelmPackage(t *testing.T, localChart string) string {
+	t.Helper()
+
+	out := execHelm(t, "package", localChart)
+	msg := strings.Split(out, " ")
+	tgzAbsPath := msg[len(msg)-1]
+	return strings.TrimSpace(tgzAbsPath)
+}
+
+func execHelm(t *testing.T, args ...string) string {
+	t.Helper()
+
+	cmd := []string{"helm"}
+	cmd = append(cmd, args...)
+	c := strings.Join(cmd, " ")
+	docker := exec.Command("helm", args...)
+	out, err := docker.CombinedOutput()
+	if err != nil {
+		t.Logf("%s: %s", c, string(out))
+		t.Fatalf("Unable to run %s: %v", c, err)
+	}
+
+	return string(out)
+}

test/e2e/template/helmfile/testdata/charts/raw/.gitignore

@@ -0,0 +1 @@
+*.tgz

test/e2e/template/helmfile/testdata/charts/raw/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

test/e2e/template/helmfile/testdata/charts/raw/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: raw
+description: A Helm chart for Kubernetes
+type: application
+version: 0.1.0
+appVersion: "1.16.0"

test/e2e/template/helmfile/testdata/charts/raw/README.md

@@ -0,0 +1,10 @@
+You should be able to test pushing this chart to a local registry with:
+
+```
+$ helm package .
+Successfully packaged chart and saved it to: /home/mumoshu/p/helmfile/test/e2e/template/helmfile/testdata/charts/raw/raw-0.1.0.tgz
+
+$ helm push raw-0.1.0.tgz oci://localhost:5000/myrepo/raw
+Pushed: localhost:5000/myrepo/raw/raw:0.1.0
+Digest: sha256:9b7c9633b519b024fdbec1db795bc2dd8b0009149135908a3aafc55280146ad9
+```

test/e2e/template/helmfile/testdata/charts/raw/templates/resources.yaml

@@ -0,0 +1,6 @@
+{{- range $i, $r := $.Values.templates }}
+{{- if gt $i 0 }}
+---
+{{- end }}
+{{- (tpl $r $) }}
+{{- end }}

test/e2e/template/helmfile/testdata/charts/raw/values.yaml

@@ -0,0 +1,48 @@
+templates: []
+
+##
+## Example: Uncomment the below and run `helm template ./`:
+##
+#
+# templates:
+# - |
+#   apiVersion: v1
+#   kind: ConfigMap
+#   metadata:
+#     name: {{ .Release.Name }}-1
+#     namespace: {{ .Release.Namespace }}
+#   data:
+#     foo: {{ .Values.foo }}
+# - |
+#   apiVersion: v1
+#   kind: ConfigMap
+#   metadata:
+#     name: {{ .Release.Name }}-2
+#     namespace: {{ .Release.Namespace }}
+#   data:
+#     foo: {{ .Values.foo }}
+# values:
+#   foo: FOO
+#
+##
+## Expected Output:
+##
+#
+# ---
+# # Source: raw/templates/resources.yaml
+# apiVersion: v1
+# kind: ConfigMap
+# metadata:
+#   name: release-name-1
+#   namespace: default
+# data:
+#   foo:
+# ---
+# # Source: raw/templates/resources.yaml
+# apiVersion: v1
+# kind: ConfigMap
+# metadata:
+#   name: release-name-2
+#   namespace: default
+# data:
+#   foo:

test/e2e/template/helmfile/testdata/snapshot/oci_need/config.yaml

@@ -0,0 +1,6 @@
+localDockerRegistry:
+  enabled: true
+  port: 5000
+chartifyTempDir: temp1
+helmfileArgs:
+- template

test/e2e/template/helmfile/testdata/snapshot/oci_need/input.yaml

@@ -0,0 +1,27 @@
+releases:
+- name: foo
+  chart: ../../charts/raw
+  values:
+  - templates:
+    - |
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: {{`{{ .Release.Name }}`}}-1
+        namespace: {{`{{ .Release.Namespace }}`}}
+      data:
+        foo: FOO
+    dep:
+      templates:
+      - |
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: {{`{{ .Release.Name }}`}}-2
+          namespace: {{`{{ .Release.Namespace }}`}}
+        data:
+          bar: BAR
+  dependencies:
+  - alias: dep
+    chart: oci://localhost:5000/myrepo/raw
+    version: 0.1.0

test/e2e/template/helmfile/testdata/snapshot/oci_need/output.yaml

@@ -0,0 +1,23 @@
+Building dependency release=foo, chart=$WD/temp1/foo
+Templating release=foo, chart=$WD/temp1/foo
+---
+# Source: raw/templates/charts/dep/templates/resources.yaml
+# Source: raw/charts/dep/templates/resources.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: foo-2
+  namespace: default
+data:
+  bar: BAR
+---
+# Source: raw/templates/resources.yaml
+# Source: raw/templates/resources.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: foo-1
+  namespace: default
+data:
+  foo: FOO
+