From 453b498ccbd96451b3468d1fba3ecd805c56e01f Mon Sep 17 00:00:00 2001
From: Thomas Loubiou <xabufr@users.noreply.github.com>
Date: Thu, 4 Mar 2021 01:24:22 +0100
Subject: [PATCH] fix: pull OCI charts one by one (#1706)

* fix: pull OCI charts one by one

* fix: gofmt
---
 pkg/state/state.go | 40 +++++++++++++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/pkg/state/state.go b/pkg/state/state.go
index 74396ab0..b5393a06 100644
--- a/pkg/state/state.go
+++ b/pkg/state/state.go
@@ -73,6 +73,11 @@ type ReleaseSetSpec struct {
 	MissingFileHandler string `yaml:"missingFileHandler,omitempty"`
 }
 
+type PullCommand struct {
+	ChartRef     string
+	responseChan chan error
+}
+
 // HelmState structure for the helmfile
 type HelmState struct {
 	basePath string
@@ -961,6 +966,11 @@ func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurre
 	}
 
 	var builds []*chartPrepareResult
+	pullChan := make(chan *PullCommand)
+	defer func() {
+		pullChan <- nil
+	}()
+	go st.pullChartWorker(pullChan, helm)
 
 	st.scatterGather(
 		concurrency,
@@ -993,7 +1003,7 @@ func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurre
 				chartFetchedByGoGetter := chartPath != chartName
 
 				if !chartFetchedByGoGetter {
-					ociChartPath, err := st.getOCIChart(release, dir, helm)
+					ociChartPath, err := st.getOCIChart(pullChan, release, dir, helm)
 					if err != nil {
 						results <- &chartPrepareResult{err: fmt.Errorf("release %q: %w", release.Name, err)}
 
@@ -2991,7 +3001,7 @@ func (st *HelmState) Reverse() {
 	}
 }
 
-func (st *HelmState) getOCIChart(release *ReleaseSpec, tempDir string, helm helmexec.Interface) (*string, error) {
+func (st *HelmState) getOCIChart(pullChan chan *PullCommand, release *ReleaseSpec, tempDir string, helm helmexec.Interface) (*string, error) {
 	repo, name := st.GetRepositoryAndNameFromChartName(release.Chart)
 	if repo == nil {
 		return nil, nil
@@ -3008,7 +3018,7 @@ func (st *HelmState) getOCIChart(release *ReleaseSpec, tempDir string, helm helm
 
 	qualifiedChartName := fmt.Sprintf("%s/%s:%s", repo.URL, name, chartVersion)
 
-	err := helm.ChartPull(qualifiedChartName)
+	err := st.pullChart(pullChan, qualifiedChartName)
 	if err != nil {
 		return nil, err
 	}
@@ -3039,3 +3049,27 @@ func (st *HelmState) getOCIChart(release *ReleaseSpec, tempDir string, helm helm
 
 	return &chartPath, nil
 }
+
+// Pull charts one by one to prevent concurrent pull problems with Helm
+func (st *HelmState) pullChartWorker(pullChan chan *PullCommand, helm helmexec.Interface) {
+	for {
+		pullCmd := <-pullChan
+		if pullCmd != nil {
+			err := helm.ChartPull(pullCmd.ChartRef)
+			pullCmd.responseChan <- err
+		} else {
+			return
+		}
+	}
+}
+
+// Send a pull command to the pull worker
+func (st *HelmState) pullChart(pullChan chan *PullCommand, chartRef string) error {
+	response := make(chan error, 1)
+	cmd := &PullCommand{
+		responseChan: response,
+		ChartRef:     chartRef,
+	}
+	pullChan <- cmd
+	return <-response
+}