Refactor integration tests (#531)

* refactor integrations

Signed-off-by: yxxhero <aiopsclub@163.com>

* Update the integration test directory structure to better correlate the test script with testdata

Signed-off-by: Yusuke Kuoka <ykuoka@gmail.com>

* Undo the test-cases directory renaming to make the diff more easy to understand

Signed-off-by: Yusuke Kuoka <ykuoka@gmail.com>

* fix ci

Signed-off-by: yxxhero <aiopsclub@163.com>

Signed-off-by: yxxhero <aiopsclub@163.com>
Signed-off-by: Yusuke Kuoka <ykuoka@gmail.com>
Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>

test/integration/run.sh

@@ -21,6 +21,7 @@ helmfile="./helmfile ${EXTRA_HELMFILE_FLAGS} --namespace=${test_ns}"
 helm="helm --kube-context=minikube"
 kubectl="kubectl --context=minikube --namespace=${test_ns}"
 helm_dir="${PWD}/${dir}/.helm"
+cases_dir="${dir}/test-cases"
 export HELM_DATA_HOME="${helm_dir}/data"
 export HELM_HOME="${HELM_DATA_HOME}"
 export HELM_PLUGINS="${HELM_DATA_HOME}/plugins"
@@ -79,7 +80,7 @@ fi
 ${helm} plugin ls | grep diff || ${helm} plugin install https://github.com/databus23/helm-diff --version v${HELM_DIFF_VERSION}
 info "Using Kustomize version: $(kustomize version --short | grep -o 'v[0-9.]\+')"
 ${kubectl} get namespace ${test_ns} &> /dev/null && warn "Namespace ${test_ns} exists, from a previous test run?"
-$kubectl create namespace ${test_ns} || fail "Could not create namespace ${test_ns}"
+${kubectl} create namespace ${test_ns} || fail "Could not create namespace ${test_ns}"
 
 
 # TEST CASES----------------------------------------------------------------------------------------------------------

test/integration/test-cases/chart-needs.sh

@@ -1,15 +1,17 @@
 if [[ helm_major_version -eq 3 ]]; then
+  chart_need_case_input_dir="${cases_dir}/chart-needs/input"
+  chart_need_case_output_dir="${cases_dir}/chart-needs/output"
+
   chart_needs_tmp=$(mktemp -d)
-  chart_needs_golden_dir=${dir}/chart-needs-golden
   chart_needs_template_reverse=${chart_needs_tmp}/chart.needs.template.log
   chart_needs_lint_reverse=${chart_needs_tmp}/chart.needs.lint.log
   chart_needs_diff_reverse=${chart_needs_tmp}/chart.needs.diff.log
 
-  lint_out_file=${chart_needs_golden_dir}/lint
+  lint_out_file=${chart_need_case_output_dir}/lint
-  diff_out_file=${chart_needs_golden_dir}/diff
+  diff_out_file=${chart_need_case_output_dir}/diff
   if [[ $EXTRA_HELMFILE_FLAGS == *--enable-live-output* ]]; then
-      lint_out_file=${chart_needs_golden_dir}/lint-live
+      lint_out_file=${chart_need_case_output_dir}/lint-live
-      diff_out_file=${chart_needs_golden_dir}/diff-live
+      diff_out_file=${chart_need_case_output_dir}/diff-live
   fi
 
   test_start "chart prepare when helmfile template with needs"
@@ -18,46 +20,46 @@ if [[ helm_major_version -eq 3 ]]; then
 
   for i in $(seq 10); do
       info "Comparing template/chart-needs #$i"
-      ${helmfile} -f ${dir}/issue.455/helmfile.yaml template --include-needs > ${chart_needs_template_reverse} || fail "\"helmfile template\" shouldn't fail"
+      ${helmfile} -f ${chart_need_case_input_dir}/helmfile.yaml template --include-needs > ${chart_needs_template_reverse} || fail "\"helmfile template\" shouldn't fail"
-      ./yamldiff ${chart_needs_golden_dir}/template ${chart_needs_template_reverse} || fail "\"helmfile template\" should be consistent"
+      ./yamldiff ${chart_need_case_output_dir}/template ${chart_needs_template_reverse} || fail "\"helmfile template\" should be consistent"
       echo code=$?
   done
 
   for i in $(seq 10); do
       info "Comparing lint/chart-needs #$i"
-      ${helmfile} -f ${dir}/issue.455/helmfile.yaml lint --include-needs | grep -v Linting > ${chart_needs_lint_reverse} || fail "\"helmfile lint\" shouldn't fail"
+      ${helmfile} -f ${chart_need_case_input_dir}/helmfile.yaml lint --include-needs | grep -v Linting > ${chart_needs_lint_reverse} || fail "\"helmfile lint\" shouldn't fail"
       diff -u ${lint_out_file} ${chart_needs_lint_reverse} || fail "\"helmfile lint\" should be consistent"
       echo code=$?
   done
 
   for i in $(seq 10); do
       info "Comparing diff/chart-needs #$i"
-      ${helmfile} -f ${dir}/issue.455/helmfile.yaml diff --include-needs | grep -Ev "Comparing release=azuredisk-csi-storageclass, chart=/tmp/.*/azuredisk-csi-storageclass" > ${chart_needs_diff_reverse} || fail "\"helmfile diff\" shouldn't fail"
+      ${helmfile} -f ${chart_need_case_input_dir}/helmfile.yaml diff --include-needs | grep -Ev "Comparing release=azuredisk-csi-storageclass, chart=/tmp/.*/azuredisk-csi-storageclass" > ${chart_needs_diff_reverse} || fail "\"helmfile diff\" shouldn't fail"
       diff -u ${diff_out_file} ${chart_needs_diff_reverse} || fail "\"helmfile diff\" should be consistent"
       echo code=$?
   done
 
-  info "Applying ${dir}/issue.455/helmfile.yaml"
+  info "Applying ${chart_need_case_input_dir}/helmfile.yaml"
-  ${helmfile} -f ${dir}/issue.455/helmfile.yaml  apply --include-needs
+  ${helmfile} -f ${chart_need_case_input_dir}/helmfile.yaml  apply --include-needs
   code=$?
   [ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile apply: want 0, got ${code}"
 
   ${kubectl} get storageclass managed-csi -o yaml | grep -q "provisioner: disk.csi.azure.com" || fail "storageclass managed-csi should be created when applying helmfile.yaml"
 
-  info "Destroying ${dir}/issue.455/helmfile.yaml"
+  info "Destroying ${chart_need_case_input_dir}/helmfile.yaml"
-  ${helmfile} -f ${dir}/issue.455/helmfile.yaml destroy
+  ${helmfile} -f ${chart_need_case_input_dir}/helmfile.yaml destroy
   code=$?
   [ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile destroy: want 0, got ${code}"
 
-  info "Syncing ${dir}/issue.455/helmfile.yaml"
+  info "Syncing ${chart_need_case_input_dir}/helmfile.yaml"
-  ${helmfile} -f ${dir}/issue.455/helmfile.yaml  sync --include-needs
+  ${helmfile} -f ${chart_need_case_input_dir}/helmfile.yaml  sync --include-needs
   code=$?
   [ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile apply: want 0, got ${code}"
 
   ${kubectl} get storageclass managed-csi -o yaml | grep -q "provisioner: disk.csi.azure.com" || fail "storageclass managed-csi should be created when syncing helmfile.yaml"
 
-  info "Destroying ${dir}/issue.455/helmfile.yaml"
+  info "Destroying ${chart_need_case_input_dir}/helmfile.yaml"
-  ${helmfile} -f ${dir}/issue.455/helmfile.yaml destroy
+  ${helmfile} -f ${chart_need_case_input_dir}/helmfile.yaml destroy
   code=$?
   [ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile destroy: want 0, got ${code}"
 

test/integration/test-cases/happypath.sh

@@ -1,89 +1,68 @@
 test_start "happypath - simple rollout of httpbin chart"
 
-info "Diffing ${dir}/happypath.yaml"
+happypath_case_input_dir="${cases_dir}/happypath/input"
-bash -c "${helmfile} -f ${dir}/happypath.yaml diff --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
+happypath_case_output_dir="${cases_dir}/happypath/output"
 
-info "Diffing ${dir}/happypath.yaml without color"
+info "Diffing ${happypath_case_input_dir}/happypath.yaml"
-bash -c "${helmfile} -f ${dir}/happypath.yaml --no-color diff --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
+bash -c "${helmfile} -f ${happypath_case_input_dir}/happypath.yaml diff --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
 
-info "Diffing ${dir}/happypath.yaml with limited context"
+info "Diffing ${happypath_case_input_dir}/happypath.yaml without color"
-bash -c "${helmfile} -f ${dir}/happypath.yaml diff --context 3 --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
+bash -c "${helmfile} -f ${happypath_case_input_dir}/happypath.yaml --no-color diff --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
 
-info "Diffing ${dir}/happypath.yaml with altered output"
+info "Diffing ${happypath_case_input_dir}/happypath.yaml with limited context"
-bash -c "${helmfile} -f ${dir}/happypath.yaml diff --output simple --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
+bash -c "${helmfile} -f ${happypath_case_input_dir}/happypath.yaml diff --context 3 --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
 
-info "Templating ${dir}/happypath.yaml"
+info "Diffing ${happypath_case_input_dir}/happypath.yaml with altered output"
+bash -c "${helmfile} -f ${happypath_case_input_dir}/happypath.yaml diff --output simple --detailed-exitcode; code="'$?'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile diff"
+
+info "Templating ${happypath_case_input_dir}/happypath.yaml"
 rm -rf ${dir}/tmp
-${helmfile} -f ${dir}/happypath.yaml --debug template --output-dir tmp
+${helmfile} -f ${happypath_case_input_dir}/happypath.yaml --debug template --output-dir tmp
 code=$?
 [ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile template: ${code}"
 for output in $(ls -d ${dir}/tmp/*); do
     # e.g. test/integration/tmp/happypath-877c0dd4-helmx/helmx
     for release_dir in $(ls -d ${output}/*); do
         release_name=$(basename ${release_dir})
-        golden_dir=${dir}/templates-golden/v${helm_major_version}/${release_name}
+        golden_dir=${happypath_case_output_dir}/v${helm_major_version}/${release_name}
         info "Comparing template output ${release_dir}/templates with ${golden_dir}"
         ./diff-yamls ${golden_dir} ${release_dir}/templates || fail "unexpected diff in template result for ${release_name}"
     done
 done
 
-info "Templating ${dir}/happypath.yaml from stdin"
+info "Applying ${happypath_case_input_dir}/happypath.yaml"
-pushd ${dir}
+bash -c "${helmfile} -f ${happypath_case_input_dir}/happypath.yaml apply --detailed-exitcode; code="'$?'"; echo Code: "'$code'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile apply"
-rm -rf ./tmp
-cat ./happypath.yaml | ../../${helmfile} -f - --debug template --output-dir tmp
-code=$?
-[ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile template: ${code}"
-for output in $(ls -d ./tmp/*); do
-    # e.g. test/integration/tmp/happypath-877c0dd4-helmx/helmx
-    for release_dir in $(ls -d ${output}/*); do
-        release_name=$(basename ${release_dir})
-        golden_dir=./templates-golden/v${helm_major_version}/${release_name}
-        info "Comparing template output ${release_dir}/templates with ${golden_dir}"
-        ../../diff-yamls ${golden_dir} ${release_dir}/templates || fail "unexpected diff in template result for ${release_name}"
-    done
-done
-popd
 
-info "Applying ${dir}/happypath.yaml"
+info "Syncing ${happypath_case_input_dir}/happypath.yaml"
-bash -c "${helmfile} -f ${dir}/happypath.yaml apply --detailed-exitcode; code="'$?'"; echo Code: "'$code'"; [ "'${code}'" -eq 2 ]" || fail "unexpected exit code returned by helmfile apply"
+${helmfile} -f ${happypath_case_input_dir}/happypath.yaml sync
-
-info "Syncing ${dir}/happypath.yaml"
-${helmfile} -f ${dir}/happypath.yaml sync
 wait_deploy_ready httpbin-httpbin
 retry 5 "curl --fail $(minikube service --url --namespace=${test_ns} httpbin-httpbin)/status/200"
 [ ${retry_result} -eq 0 ] || fail "httpbin failed to return 200 OK"
 
-info "Applying ${dir}/happypath.yaml"
+info "Applying ${happypath_case_input_dir}/happypath.yaml"
-${helmfile} -f ${dir}/happypath.yaml apply --detailed-exitcode
+${helmfile} -f ${happypath_case_input_dir}/happypath.yaml apply --detailed-exitcode
 code=$?
 [ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile apply: want 0, got ${code}"
 
-info "Applying ${dir}/happypath.yaml from stdin"
-pushd ${dir}
-cat ./happypath.yaml | ../../${helmfile} -f - apply --detailed-exitcode
-code=$?
-[ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile apply: want 0, got ${code}"
-popd
-
 info "Locking dependencies"
-${helmfile} -f ${dir}/happypath.yaml deps
+${helmfile} -f ${happypath_case_input_dir}/happypath.yaml deps
 code=$?
 [ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile deps: ${code}"
 
-info "Applying ${dir}/happypath.yaml with locked dependencies"
+info "Applying ${happypath_case_input_dir}/happypath.yaml with locked dependencies"
-${helmfile} -f ${dir}/happypath.yaml apply
+${helmfile} -f ${happypath_case_input_dir}/happypath.yaml apply
 code=$?
 [ ${code} -eq 0 ] || fail "unexpected exit code returned by helmfile apply: ${code}"
 ${helm} list --namespace=${test_ns} || fail "unable to list releases"
 
 info "Deleting release"
-${helmfile} -f ${dir}/happypath.yaml delete
+${helmfile} -f ${happypath_case_input_dir}/happypath.yaml delete
 ${helm} status --namespace=${test_ns} httpbin &> /dev/null && fail "release should not exist anymore after a delete"
 
 info "Ensuring \"helmfile delete\" doesn't fail when no releases installed"
-${helmfile} -f ${dir}/happypath.yaml delete || fail "\"helmfile delete\" shouldn't fail when there are no installed releases"
+${helmfile} -f ${happypath_case_input_dir}/happypath.yaml delete || fail "\"helmfile delete\" shouldn't fail when there are no installed releases"
 
 info "Ensuring \"helmfile template\" output does contain only YAML docs"
-(${helmfile} -f ${dir}/happypath.yaml template | kubectl apply -f -) || fail "\"helmfile template | kubectl apply -f -\" shouldn't fail"
+(${helmfile} -f ${happypath_case_input_dir}/happypath.yaml template | kubectl apply -f -) || fail "\"helmfile template | kubectl apply -f -\" shouldn't fail"
 
 test_pass "happypath"

test/integration/test-cases/happypath/input/happypath.yaml

@@ -13,7 +13,7 @@ helmDefaults:
 releases:
 
   - name: httpbin
-    chart: ./charts/httpbin
+    chart: ../../../charts/httpbin
     set:
       - name: ingress.enabled
         value: false
@@ -34,7 +34,7 @@ releases:
       - values.yaml
 
   - name: helmx
-    chart: ./charts/helmx
+    chart: ../../../charts/helmx
     namespace: helmx-system
     jsonPatches:
       - target:

test/integration/test-cases/regression.sh

@@ -2,15 +2,16 @@
 test_start "regression tests"
 
 if [[ helm_major_version -eq 3 ]]; then
+  regression_case_input_dir="${cases_dir}/regression/input"
   info "https://github.com/roboll/helmfile/issues/1857"
-  (${helmfile} -f ${dir}/issue.1857.yaml --state-values-set grafanaEnabled=true template | grep grafana 1>/dev/null) || fail "\"helmfile template\" shouldn't include grafana"
+  (${helmfile} -f ${regression_case_input_dir}/issue.1857.yaml --state-values-set grafanaEnabled=true template | grep grafana 1>/dev/null) || fail "\"helmfile template\" shouldn't include grafana"
-  ! (${helmfile} -f ${dir}/issue.1857.yaml --state-values-set grafanaEnabled=false template | grep grafana) || fail "\"helmfile template\" shouldn't include grafana"
+  ! (${helmfile} -f ${regression_case_input_dir}/issue.1857.yaml --state-values-set grafanaEnabled=false template | grep grafana) || fail "\"helmfile template\" shouldn't include grafana"
 
   info "https://github.com/roboll/helmfile/issues/1867"
-  (${helmfile} -f ${dir}/issue.1867.yaml template 1>/dev/null) || fail "\"helmfile template\" shouldn't fail"
+  (${helmfile} -f ${regression_case_input_dir}/issue.1867.yaml template 1>/dev/null) || fail "\"helmfile template\" shouldn't fail"
 
   info "https://github.com/roboll/helmfile/issues/2118"
-  (${helmfile} -f ${dir}/issue.2118.yaml template 1>/dev/null) || fail "\"helmfile template\" shouldn't fail"
+  (${helmfile} -f ${regression_case_input_dir}/issue.2118.yaml template 1>/dev/null) || fail "\"helmfile template\" shouldn't fail"
 else
   info "There are no regression tests for helm 2 because all the target charts have dropped helm 2 support."
 fi

test/integration/test-cases/regression/input/issue.1867.yaml

@@ -15,4 +15,3 @@ releases:
       - op: add
         path: /spec/template/spec/containers/0/volumeMounts/0/subPathExpr
         value: elasticsearch/$(MY_POD_NAME)
-  

test/integration/test-cases/secretssops.sh

@@ -2,11 +2,15 @@ if [[ helm_major_version -eq 3 ]]; then
   export VAULT_ADDR=http://127.0.0.1:8200
   export VAULT_TOKEN=toor
   sops="sops --hc-vault-transit $VAULT_ADDR/v1/sops/keys/key"
-  mkdir -p ${dir}/tmp
+
+  secretssops_case_input_dir="${cases_dir}/secretssops/input"
+  secretssops_case_output_dir="${cases_dir}/secretssops/output"
+
+  mkdir -p ${secretssops_case_input_dir}/tmp
 
   info "Encrypt secrets"
-  ${sops} -e ${dir}/env-1.secrets.yaml > ${dir}/tmp/env-1.secrets.sops.yaml || fail "${sops} failed at ${dir}/env-1.secrets.yaml"
+  ${sops} -e ${secretssops_case_input_dir}/env-1.secrets.yaml > ${secretssops_case_input_dir}/tmp/env-1.secrets.sops.yaml || fail "${sops} failed at ${secretssops_case_input_dir}/env-1.secrets.yaml"
-  ${sops} -e ${dir}/env-2.secrets.yaml > ${dir}/tmp/env-2.secrets.sops.yaml || fail "${sops} failed at ${dir}/env-2.secrets.yaml"
+  ${sops} -e ${secretssops_case_input_dir}/env-2.secrets.yaml > ${secretssops_case_input_dir}/tmp/env-2.secrets.sops.yaml || fail "${sops} failed at ${secretssops_case_input_dir}/env-2.secrets.yaml"
 
   test_start "secretssops.1 - should fail without secrets plugin"
 
@@ -15,7 +19,7 @@ if [[ helm_major_version -eq 3 ]]; then
 
   info "Ensure helmfile fails when no helm-secrets is installed"
   unset code
-  ${helmfile} -f ${dir}/secretssops.yaml -e direct build || code="$?"; code="${code:-0}"
+  ${helmfile} -f ${secretssops_case_input_dir}/secretssops.yaml -e direct build || code="$?"; code="${code:-0}"
   echo Code: "${code}"
   [ "${code}" -ne 0 ] || fail "\"helmfile build\" should fail without secrets plugin"
 
@@ -27,7 +31,7 @@ if [[ helm_major_version -eq 3 ]]; then
   ${helm} plugin install https://github.com/jkroepke/helm-secrets --version v${HELM_SECRETS_VERSION}
 
   info "Ensure helmfile succeed when helm-secrets is installed"
-  ${helmfile} -f ${dir}/secretssops.yaml -e direct build || fail "\"helmfile build\" shouldn't fail"
+  ${helmfile} -f ${secretssops_case_input_dir}/secretssops.yaml -e direct build || fail "\"helmfile build\" shouldn't fail"
 
   test_pass "secretssops.2"
 
@@ -36,23 +40,22 @@ if [[ helm_major_version -eq 3 ]]; then
   secretssops_tmp=$(mktemp -d)
   direct=${secretssops_tmp}/direct.build.yaml
   reverse=${secretssops_tmp}/reverse.build.yaml
-  secrets_golden_dir=${dir}/secrets-golden
 
   info "Building secrets output"
 
-  info "Comparing build/direct output ${direct} with ${secrets_golden_dir}"
+  info "Comparing build/direct output ${direct} with ${secretssops_case_output_dir}"
   for i in $(seq 10); do
       info "Comparing build/direct #$i"
-      ${helmfile} -f ${dir}/secretssops.yaml -e direct template --skip-deps > ${direct} || fail "\"helmfile template\" shouldn't fail"
+      ${helmfile} -f ${secretssops_case_input_dir}/secretssops.yaml -e direct template --skip-deps > ${direct} || fail "\"helmfile template\" shouldn't fail"
-      ./yamldiff ${secrets_golden_dir}/direct.build.yaml ${direct} || fail "\"helmfile template\" should be consistent"
+      ./yamldiff ${secretssops_case_output_dir}/direct.build.yaml ${direct} || fail "\"helmfile template\" should be consistent"
       echo code=$?
   done
 
-  info "Comparing build/reverse output ${direct} with ${secrets_golden_dir}"
+  info "Comparing build/reverse output ${direct} with ${secretssops_case_output_dir}"
   for i in $(seq 10); do
       info "Comparing build/reverse #$i"
-      ${helmfile} -f ${dir}/secretssops.yaml -e reverse template --skip-deps > ${reverse} || fail "\"helmfile template\" shouldn't fail"
+      ${helmfile} -f ${secretssops_case_input_dir}/secretssops.yaml -e reverse template --skip-deps > ${reverse} || fail "\"helmfile template\" shouldn't fail"
-      ./yamldiff ${secrets_golden_dir}/reverse.build.yaml ${reverse} || fail "\"helmfile template\" should be consistent"
+      ./yamldiff ${secretssops_case_output_dir}/reverse.build.yaml ${reverse} || fail "\"helmfile template\" should be consistent"
       echo code=$?
   done
 

test/integration/test-cases/yaml-overwrite.sh

@@ -1,14 +1,16 @@
 if [[ helm_major_version -eq 3 ]]; then
+  yaml_overwrite_case_input_dir="${cases_dir}/yaml-overwrite/input"
+  yaml_overwrite_case_output_dir="${cases_dir}/yaml-overwrite/output"
+
   yaml_overwrite_tmp=$(mktemp -d)
-  yaml_feature_golden_dir=${dir}/yaml-features-golden
   yaml_overwrite_reverse=${yaml_overwrite_tmp}/yaml.override.build.yaml
 
   test_start "yaml overwrite feature"
-  info "Comparing yaml overwrite feature output ${yaml_overwrite_reverse} with ${yaml_feature_golden_dir}/overwritten.yaml"
+  info "Comparing yaml overwrite feature output ${yaml_overwrite_reverse} with ${yaml_overwrite_case_output_dir}/overwritten.yaml"
   for i in $(seq 10); do
       info "Comparing build/yaml-overwrite #$i"
-      ${helmfile} -f ${dir}/issue.657.yaml template --skip-deps > ${yaml_overwrite_reverse} || fail "\"helmfile template\" shouldn't fail"
+      ${helmfile} -f ${yaml_overwrite_case_input_dir}/issue.657.yaml template --skip-deps > ${yaml_overwrite_reverse} || fail "\"helmfile template\" shouldn't fail"
-      ./yamldiff ${yaml_feature_golden_dir}/overwritten.yaml ${yaml_overwrite_reverse} || fail "\"helmfile template\" should be consistent"
+      ./yamldiff ${yaml_overwrite_case_output_dir}/overwritten.yaml ${yaml_overwrite_reverse} || fail "\"helmfile template\" should be consistent"
       echo code=$?
   done
   test_pass "yaml overwrite feature"

test/integration/test-cases/yaml-overwrite/input/issue.657.yaml

@@ -8,7 +8,7 @@ templates:
 
 releases:
   - name: httpbin
-    chart: ./charts/httpbin
+    chart: ../../../charts/httpbin
     <<: *default
     values:
       - image: