feat: Add `needs: [NS/NAME]` for controlling installation/deletion order declaratively (#914)

Introduces DAG-aware installation/deletion ordering to Helmfile.

`needs` controls the order of the installation/deletion of the release:

```yaml
relesaes:
- name: somerelease
  needs:
  - [TILLER_NAMESPACE/][NAMESPACE/]anotherelease
```

All the releases listed under `needs` are installed before(or deleted after) the release itself.

For the following example, `helmfile [sync|apply]` installs releases in this order:

1. logging
2. servicemesh
3. myapp1 and myapp2

```yaml
  - name: myapp1
    chart: charts/myapp
    needs:
    - servicemesh
    - logging
  - name: myapp2
    chart: charts/myapp
    needs:
    - servicemesh
    - logging
  - name: servicemesh
    chart: charts/istio
    needs:
    - logging
  - name: logging
    chart: charts/fluentd
```

Note that all the releases in a same group is installed concurrently. That is, myapp1 and myapp2 are installed concurrently.

On `helmdile [delete|destroy]`, deleations happen in the reverse order.

That is, `myapp1` and `myapp2` are deleted first, then `servicemesh`, and finally `logging`.

Resolves #715
This commit is contained in:
KUOKA Yusuke 2019-10-28 12:57:25 +09:00 committed by GitHub
parent b8f24948bb
commit 7666e95690
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 316 additions and 8 deletions

View File

@ -723,6 +723,50 @@ With the [helm-tiller](https://github.com/rimusz/helm-tiller) plugin installed,
To enable this mode, you need to define `tillerless: true` and set the `tillerNamespace` in the `helmDefaults` section
or in the `releases` entries.
## DAG-aware installation/deletion ordering
`needs` controls the order of the installation/deletion of the release:
```yaml
relesaes:
- name: somerelease
needs:
- [TILLER_NAMESPACE/][NAMESPACE/]anotherelease
```
All the releases listed under `needs` are installed before(or deleted after) the release itself.
For the following example, `helmfile [sync|apply]` installs releases in this order:
1. logging
2. servicemesh
3. myapp1 and myapp2
```yaml
- name: myapp1
chart: charts/myapp
needs:
- servicemesh
- logging
- name: myapp2
chart: charts/myapp
needs:
- servicemesh
- logging
- name: servicemesh
chart: charts/istio
needs:
- logging
- name: logging
chart: charts/fluentd
```
Note that all the releases in a same group is installed concurrently. That is, myapp1 and myapp2 are installed concurrently.
On `helmdile [delete|destroy]`, deleations happen in the reverse order.
That is, `myapp1` and `myapp2` are deleted first, then `servicemesh`, and finally `logging`.
## Separating helmfile.yaml into multiple independent files
Once your `helmfile.yaml` got to contain too many releases,

1
go.mod
View File

@ -25,6 +25,7 @@ require (
github.com/r3labs/diff v0.0.0-20190801153147-a71de73c46ad
github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939
github.com/urfave/cli v1.20.0
github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363
github.com/variantdev/vals v0.0.0-20191026125821-5d18b16cf30a
go.mozilla.org/sops v0.0.0-20190912205235-14a22d7a7060 // indirect
go.opencensus.io v0.22.1 // indirect

9
go.sum
View File

@ -18,6 +18,7 @@ cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFx
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/storage v1.0.0 h1:VV2nUM3wwLLGh9lSABFgZMjInyUbJeaRSE64WuAIQ+4=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f/go.mod h1:sk5LnIjB/nIEU7yP5sDQExVm62wu0pBh3yrElngUisI=
contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA=
@ -85,6 +86,7 @@ github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RP
github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver v1.4.1 h1:CaDA1wAoM3rj9sAFyyZP37LloExUzxFGYt+DqJ870JA=
github.com/Masterminds/semver v1.4.1/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60=
github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
@ -228,6 +230,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@ -240,6 +243,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c=
github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU=
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
@ -313,6 +317,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0=
github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@ -577,7 +582,10 @@ github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28 h1:Yf7/DcGM+61oLBGXQV2Q+ztEGBcOe3EUnbKSOn4fQuE=
github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363 h1:KrfQBEUn+wEOQ/6UIfoqRDvn+Q/wZridQ7t0G1vQqKE=
github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363/go.mod h1:pH1TQsNSLj2uxMo9NNl9zdGy01Wtn+/2MT96BrKmVyE=
github.com/variantdev/vals v0.0.0-20191025124021-e86de6f8cd7d h1:od9EUts72ELxOSrqJxtbxAU/c30XCaA+xnq0jDWMQrA=
github.com/variantdev/vals v0.0.0-20191025124021-e86de6f8cd7d/go.mod h1:7Gb8avEj7D9rMIqn4Nn9RBKXo/0rZ7Z0e+3bmHIs420=
github.com/variantdev/vals v0.0.0-20191026125821-5d18b16cf30a h1:zrV+XXPXniLy9ZVHUN7DLVVKnMjActASxwsMKIohfLA=
@ -608,6 +616,7 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.8.0 h1:r6Za1Rii8+EGOYRDLvpooNOF6kP3iyDnkpzbw67gCQ8=
go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=

View File

@ -20,6 +20,7 @@ import (
"github.com/roboll/helmfile/pkg/helmexec"
"github.com/roboll/helmfile/pkg/remote"
"github.com/roboll/helmfile/pkg/tmpl"
"github.com/variantdev/dag/pkg/dag"
"regexp"
@ -144,6 +145,8 @@ type ReleaseSpec struct {
// MissingFileHandler is set to either "Error" or "Warn". "Error" instructs helmfile to fail when unable to find a values or secrets file. When "Warn", it prints the file and continues.
// The default value for MissingFileHandler is "Error".
MissingFileHandler *string `yaml:"missingFileHandler,omitempty"`
// Needs is the [TILLER_NS/][NS/]NAME representations of releases that this release depends on.
Needs []string `yaml:"needs,omitempty"`
// Hooks is a list of extension points paired with operations, that are executed in specific points of the lifecycle of releases defined in helmfile
Hooks []event.Hook `yaml:"hooks,omitempty"`
@ -408,12 +411,89 @@ func (st *HelmState) SyncReleases(affectedReleases *AffectedReleases, helm helme
return prepErrs
}
availableIds := make([]string, len(preps))
idToPrep := map[string]syncPrepareResult{}
d := dag.New()
for i, p := range preps {
r := p.release
id := releaseToID(r)
idToPrep[id] = p
availableIds[i] = id
d.Add(id, dag.Dependencies(r.Needs))
}
plan, err := d.Plan()
if err != nil {
return []error{err}
}
groupsTotal := len(plan)
st.logger.Debugf("syncing %d groups of releases in this order: %s", groupsTotal, plan)
for id, prep := range idToPrep {
for _, need := range prep.release.Needs {
if _, ok := idToPrep[need]; !ok {
return []error{fmt.Errorf("%q needs %q, but it must be one of %s", id, need, strings.Join(availableIds, ", "))}
}
}
}
for groupIndex, dagNodesInGroup := range plan {
var idsInGroup []string
var prepsInGroup []syncPrepareResult
for _, node := range dagNodesInGroup {
prepareResult, ok := idToPrep[node.Id]
if !ok {
panic(fmt.Sprintf("[bug] no release found for dag node id %q", node.Id))
}
prepsInGroup = append(prepsInGroup, prepareResult)
idsInGroup = append(idsInGroup, node.Id)
}
st.logger.Debugf("syncing releases in group %d/%d: %s", groupIndex+1, groupsTotal, strings.Join(idsInGroup, ", "))
errs := st.syncReleaseGroup(affectedReleases, helm, prepsInGroup)
if len(errs) > 0 {
return errs
}
}
return nil
}
func releaseToID(r *ReleaseSpec) string {
var id string
tns := r.TillerNamespace
if tns != "" {
id += tns + "/"
}
ns := r.Namespace
if ns != "" {
id += ns + "/"
}
id += r.Name
return id
}
func (st *HelmState) syncReleaseGroup(affectedReleases *AffectedReleases, helm helmexec.Interface, preps []syncPrepareResult) []error {
errs := []error{}
jobQueue := make(chan *syncPrepareResult, len(preps))
results := make(chan syncResult, len(preps))
concurrency := len(preps)
st.scatterGather(
workerLimit,
concurrency,
len(preps),
func() {
for i := 0; i < len(preps); i++ {
@ -1010,8 +1090,9 @@ func (st *HelmState) ReleaseStatuses(helm helmexec.Interface, workerLimit int) [
}
// DeleteReleases wrapper for executing helm delete on the releases
// This function traverses the DAG of the releases in the reverse order, so that the releases that are NOT depended by any others are deleted first.
func (st *HelmState) DeleteReleases(affectedReleases *AffectedReleases, helm helmexec.Interface, concurrency int, purge bool) []error {
return st.scatterGatherReleases(helm, concurrency, func(release ReleaseSpec, workerIndex int) error {
return st.dagAwareReverseIterateOnReleases(helm, concurrency, func(release ReleaseSpec, workerIndex int) error {
if !release.Desired() {
return nil
}

View File

@ -2,9 +2,11 @@ package state
import (
"fmt"
"strings"
"sync"
"github.com/roboll/helmfile/pkg/helmexec"
"github.com/variantdev/dag/pkg/dag"
)
type result struct {
@ -51,9 +53,14 @@ func (st *HelmState) scatterGather(concurrency int, items int, produceInputs fun
func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency int,
do func(ReleaseSpec, int) error) []error {
return st.iterateOnReleases(helm, concurrency, st.Releases, do)
}
func (st *HelmState) iterateOnReleases(helm helmexec.Interface, concurrency int, inputs []ReleaseSpec,
do func(ReleaseSpec, int) error) []error {
var errs []error
inputs := st.Releases
inputsSize := len(inputs)
releases := make(chan ReleaseSpec)
@ -96,3 +103,52 @@ func (st *HelmState) scatterGatherReleases(helm helmexec.Interface, concurrency
return nil
}
func (st *HelmState) dagAwareReverseIterateOnReleases(helm helmexec.Interface, concurrency int,
do func(ReleaseSpec, int) error) []error {
idToRelease := map[string]ReleaseSpec{}
preps := st.Releases
d := dag.New()
for _, r := range preps {
id := releaseToID(&r)
idToRelease[id] = r
d.Add(id, dag.Dependencies(r.Needs))
}
plan, err := d.Plan()
if err != nil {
return []error{err}
}
groupsTotal := len(plan)
st.logger.Debugf("processing %d groups of releases in this order: %s", groupsTotal, plan)
for groupIndex := len(plan) - 1; groupIndex >= 0; groupIndex-- {
dagNodesInGroup := plan[groupIndex]
var idsInGroup []string
var releasesInGroup []ReleaseSpec
for _, node := range dagNodesInGroup {
releasesInGroup = append(releasesInGroup, idToRelease[node.Id])
idsInGroup = append(idsInGroup, node.Id)
}
st.logger.Debugf("processing releases in group %d/%d: %s", groupIndex+1, groupsTotal, strings.Join(idsInGroup, ", "))
errs := st.iterateOnReleases(helm, concurrency, releasesInGroup, do)
if len(errs) > 0 {
return errs
}
}
return nil
}

View File

@ -867,10 +867,11 @@ func TestHelmState_SyncRepos(t *testing.T) {
func TestHelmState_SyncReleases(t *testing.T) {
tests := []struct {
name string
releases []ReleaseSpec
helm *mockHelmExec
wantReleases []mockRelease
name string
releases []ReleaseSpec
helm *mockHelmExec
wantReleases []mockRelease
wantErrorMsgs []string
}{
{
name: "normal release",
@ -961,6 +962,106 @@ func TestHelmState_SyncReleases(t *testing.T) {
helm: &mockHelmExec{},
wantReleases: []mockRelease{{"releaseName", []string{"--set", "foo.bar[0]={A,B}"}}},
},
{
name: "foo needs bar",
releases: []ReleaseSpec{
{
Name: "foo",
Chart: "charts/foo",
Needs: []string{
"bar",
},
},
{
Name: "bar",
Chart: "charts/bar",
},
},
helm: &mockHelmExec{},
wantReleases: []mockRelease{{"bar", []string{}}, {"foo", []string{}}},
},
{
name: "bar needs foo",
releases: []ReleaseSpec{
{
Name: "foo",
Chart: "charts/foo",
},
{
Name: "bar",
Chart: "charts/bar",
Needs: []string{
"foo",
},
},
},
helm: &mockHelmExec{},
wantReleases: []mockRelease{{"foo", []string{}}, {"bar", []string{}}},
},
{
name: "ns2/bar needs ns1/foo",
releases: []ReleaseSpec{
{
Name: "foo",
Namespace: "ns1",
Chart: "charts/foo",
},
{
Name: "bar",
Namespace: "ns2",
Chart: "charts/bar",
Needs: []string{
"ns1/foo",
},
},
},
helm: &mockHelmExec{},
wantReleases: []mockRelease{{"foo", []string{"--namespace", "ns1"}}, {"bar", []string{"--namespace", "ns2"}}},
},
{
name: "tillerns1/ns1/foo needs tillerns2/ns2/bar",
releases: []ReleaseSpec{
{
Name: "foo",
Chart: "charts/foo",
Namespace: "ns1",
TillerNamespace: "tillerns1",
Needs: []string{
"tillerns2/ns2/bar",
},
},
{
Name: "bar",
Namespace: "ns2",
TillerNamespace: "tillerns2",
Chart: "charts/bar",
},
},
helm: &mockHelmExec{},
wantReleases: []mockRelease{{"bar", []string{"--tiller-namespace", "tillerns2", "--namespace", "ns2"}}, {"foo", []string{"--tiller-namespace", "tillerns1", "--namespace", "ns1"}}},
},
{
name: "tillerns1/ns1/foo needs tillerns2/ns2/bar",
releases: []ReleaseSpec{
{
Name: "foo",
Chart: "charts/foo",
Namespace: "ns1",
TillerNamespace: "tillerns1",
Needs: []string{
"bar",
},
},
{
Name: "bar",
Namespace: "ns2",
TillerNamespace: "tillerns2",
Chart: "charts/bar",
},
},
helm: &mockHelmExec{},
wantErrorMsgs: []string{`"tillerns1/ns1/foo" needs "bar", but it must be one of tillerns1/ns1/foo, tillerns2/ns2/bar`},
},
}
for i := range tests {
tt := tests[i]
@ -970,7 +1071,23 @@ func TestHelmState_SyncReleases(t *testing.T) {
logger: logger,
valsRuntime: valsRuntime,
}
if _ = state.SyncReleases(&AffectedReleases{}, tt.helm, []string{}, 1); !reflect.DeepEqual(tt.helm.releases, tt.wantReleases) {
if errs := state.SyncReleases(&AffectedReleases{}, tt.helm, []string{}, 1); errs != nil && len(errs) > 0 {
if len(errs) != len(tt.wantErrorMsgs) {
t.Fatalf("Unexpected errors: %v\nExpected: %v", errs, tt.wantErrorMsgs)
}
var mismatch int
for i := range tt.wantErrorMsgs {
expected := tt.wantErrorMsgs[i]
actual := errs[i].Error()
if !reflect.DeepEqual(actual, expected) {
t.Errorf("Unexpected error: expected=%v, got=%v", expected, actual)
}
}
if mismatch > 0 {
t.Fatalf("%d unexpected errors detected", mismatch)
}
}
if !reflect.DeepEqual(tt.helm.releases, tt.wantReleases) {
t.Errorf("HelmState.SyncReleases() for [%s] = %v, want %v", tt.name, tt.helm.releases, tt.wantReleases)
}
})