diff --git a/cmd/apply.go b/cmd/apply.go index f7a603da..d9e8e224 100644 --- a/cmd/apply.go +++ b/cmd/apply.go @@ -72,7 +72,9 @@ func NewApplyCmd(globalCfg *config.GlobalImpl) *cobra.Command { f.StringArrayVar(&applyOptions.SuppressOutputLineRegex, "suppress-output-line-regex", nil, "a list of regex patterns to suppress output lines from diff output") f.StringVar(&applyOptions.TrackMode, "track-mode", "", "Track mode for releases: 'helm' (default), 'helm-legacy' (Helm v4 only), or 'kubedog'") f.IntVar(&applyOptions.TrackTimeout, "track-timeout", 0, `Timeout in seconds for kubedog tracking (0 to use default 300s timeout)`) - f.BoolVar(&applyOptions.TrackLogs, "track-logs", false, "Enable log streaming with kubedog tracking") + f.BoolVar(&applyOptions.TrackLogs, "track-logs", false, "Enable log streaming with kubedog tracking (all pods)") + f.BoolVar(&applyOptions.TrackFailedLogs, "track-failed-logs", false, "Enable log streaming with kubedog tracking, but only emit logs for pods that enter a failed state. Overridden by --track-logs when both are set") + f.IntVar(&applyOptions.HelmStuckGrace, "helm-stuck-grace", 0, "When using --track-mode kubedog: if the cluster confirms all tracked resources have converged but the helm subprocess is still running, wait this many seconds before sending SIGINT to helm. Recovers from helm v4 hook waiter wedges. May leave the release secret in pending-install state requiring manual cleanup. 0 disables.") f.BoolVar(&applyOptions.TrackFailOnError, "track-fail-on-error", false, "Fail with non-zero exit code when kubedog tracking fails") f.StringVar(&applyOptions.Description, "description", "", `Set description for all releases. If set, overridesdescriptions in helmfile.yaml. Will be passed to "helm upgrade --description"`) diff --git a/cmd/sync.go b/cmd/sync.go index 0f398682..5f1ed686 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -57,7 +57,9 @@ func NewSyncCmd(globalCfg *config.GlobalImpl) *cobra.Command { f.StringVar(&syncOptions.Cascade, "cascade", "", "pass cascade to helm exec, default: background") f.StringVar(&syncOptions.TrackMode, "track-mode", "", "Track mode for releases: 'helm' (default), 'helm-legacy' (Helm v4 only), or 'kubedog'") f.IntVar(&syncOptions.TrackTimeout, "track-timeout", 0, `Timeout in seconds for kubedog tracking (0 to use default 300s timeout)`) - f.BoolVar(&syncOptions.TrackLogs, "track-logs", false, "Enable log streaming with kubedog tracking") + f.BoolVar(&syncOptions.TrackLogs, "track-logs", false, "Enable log streaming with kubedog tracking (all pods)") + f.BoolVar(&syncOptions.TrackFailedLogs, "track-failed-logs", false, "Enable log streaming with kubedog tracking, but only emit logs for pods that enter a failed state. Overridden by --track-logs when both are set") + f.IntVar(&syncOptions.HelmStuckGrace, "helm-stuck-grace", 0, "When using --track-mode kubedog: if the cluster confirms all tracked resources have converged but the helm subprocess is still running, wait this many seconds before sending SIGINT to helm. Recovers from helm v4 hook waiter wedges. May leave the release secret in pending-install state requiring manual cleanup. 0 disables.") f.BoolVar(&syncOptions.TrackFailOnError, "track-fail-on-error", false, "Fail with non-zero exit code when kubedog tracking fails") f.StringVar(&syncOptions.Description, "description", "", `Set description for all releases. If set, overrides descriptions in helmfile.yaml. Will be passed to "helm upgrade --description"`) diff --git a/go.mod b/go.mod index 31366a03..b3dc351c 100644 --- a/go.mod +++ b/go.mod @@ -189,6 +189,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect + github.com/chanced/caps v1.0.2 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.3 // indirect @@ -201,6 +202,7 @@ require ( github.com/cyberark/conjur-api-go v0.14.1 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/danieljoos/wincred v1.2.2 // indirect + github.com/dominikbraun/graph v0.23.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect @@ -317,6 +319,8 @@ require ( github.com/werf/logboek v0.6.1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yandex-cloud/go-genproto v0.79.0 // indirect diff --git a/go.sum b/go.sum index e344e315..07cc2bb3 100644 --- a/go.sum +++ b/go.sum @@ -217,6 +217,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/chanced/caps v1.0.2 h1:RELvNN4lZajqSXJGzPaU7z8B4LK2+o2Oc/upeWdgMOA= +github.com/chanced/caps v1.0.2/go.mod h1:SJhRzeYLKJ3OmzyQXhdZ7Etj7lqqWoPtQ1zcSJRtQjs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= @@ -280,6 +282,8 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo= +github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a h1:UwSIFv5g5lIvbGgtf3tVwC7Ky9rmMFBp0RMs+6f6YqE= @@ -789,6 +793,7 @@ github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcY github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= diff --git a/pkg/app/app.go b/pkg/app/app.go index d64eced9..e2c0b0f9 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -1943,8 +1943,12 @@ Do you really want to apply? TrackMode: c.TrackMode(), TrackTimeout: c.TrackTimeout(), TrackLogs: c.TrackLogs(), + TrackFailedLogs: c.TrackFailedLogs(), + HelmStuckGrace: c.HelmStuckGrace(), TrackFailOnError: c.TrackFailOnError(), Description: c.Description(), + Color: c.Color(), + NoColor: c.NoColor(), } return subst.SyncReleases(&affectedReleases, helm, c.Values(), c.Concurrency(), syncOpts) })) @@ -1955,7 +1959,7 @@ Do you really want to apply? } } - affectedReleases.DisplayAffectedReleases(c.Logger()) + affectedReleases.DisplayAffectedReleases(c.Logger(), !c.NoColor()) for id := range releasesWithNoChange { r := releasesWithNoChange[id] @@ -2041,7 +2045,7 @@ Do you really want to delete? } } } - affectedReleases.DisplayAffectedReleases(c.Logger()) + affectedReleases.DisplayAffectedReleases(c.Logger(), !c.NoColor()) return true, errs } @@ -2432,8 +2436,12 @@ Do you really want to sync? TrackMode: c.TrackMode(), TrackTimeout: c.TrackTimeout(), TrackLogs: c.TrackLogs(), + TrackFailedLogs: c.TrackFailedLogs(), + HelmStuckGrace: c.HelmStuckGrace(), TrackFailOnError: c.TrackFailOnError(), Description: c.Description(), + Color: c.Color(), + NoColor: c.NoColor(), } return subst.SyncReleases(&affectedReleases, helm, c.Values(), c.Concurrency(), syncOpts) })) @@ -2444,7 +2452,7 @@ Do you really want to sync? } } - affectedReleases.DisplayAffectedReleases(c.Logger()) + affectedReleases.DisplayAffectedReleases(c.Logger(), !c.NoColor()) for id := range releasesWithNoChange { r := releasesWithNoChange[id] diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 98c92bf9..abb8b157 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -2776,6 +2776,14 @@ func (a applyConfig) TrackLogs() bool { return a.trackLogs } +func (a applyConfig) TrackFailedLogs() bool { + return false +} + +func (a applyConfig) HelmStuckGrace() int { + return 0 +} + func (a applyConfig) TrackFailOnError() bool { return a.trackFailOnError } diff --git a/pkg/app/config.go b/pkg/app/config.go index 61aab00b..70da6907 100644 --- a/pkg/app/config.go +++ b/pkg/app/config.go @@ -92,6 +92,8 @@ type ApplyConfigProvider interface { TrackMode() string TrackTimeout() int TrackLogs() bool + TrackFailedLogs() bool + HelmStuckGrace() int TrackFailOnError() bool Description() string @@ -133,8 +135,13 @@ type SyncConfigProvider interface { TrackMode() string TrackTimeout() int TrackLogs() bool + TrackFailedLogs() bool + HelmStuckGrace() int TrackFailOnError() bool + Color() bool + NoColor() bool + Description() string DAGConfig @@ -193,6 +200,7 @@ type DestroyConfigProvider interface { SkipCharts() bool DeleteWait() bool DeleteTimeout() int + NoColor() bool interactive loggingConfig diff --git a/pkg/app/destroy_test.go b/pkg/app/destroy_test.go index 45908f02..1a136230 100644 --- a/pkg/app/destroy_test.go +++ b/pkg/app/destroy_test.go @@ -90,6 +90,10 @@ func (d destroyConfig) DeleteTimeout() int { return d.deleteTimeout } +func (d destroyConfig) NoColor() bool { + return true +} + func TestDestroy(t *testing.T) { type testcase struct { ns string diff --git a/pkg/app/testdata/testapply/delete_bar_when_bar_needs_foo/log b/pkg/app/testdata/testapply/delete_bar_when_bar_needs_foo/log index 3adeaeab..40f95e3d 100644 --- a/pkg/app/testdata/testapply/delete_bar_when_bar_needs_foo/log +++ b/pkg/app/testdata/testapply/delete_bar_when_bar_needs_foo/log @@ -23,12 +23,12 @@ GROUP RELEASES processing releases in group 1/1: default//foo -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo stable/mychart1 3.1.0 0s -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION bar 0s diff --git a/pkg/app/testdata/testapply/delete_bar_when_foo_needs_bar/log b/pkg/app/testdata/testapply/delete_bar_when_foo_needs_bar/log index c4da6734..e3bf1491 100644 --- a/pkg/app/testdata/testapply/delete_bar_when_foo_needs_bar/log +++ b/pkg/app/testdata/testapply/delete_bar_when_foo_needs_bar/log @@ -26,12 +26,12 @@ GROUP RELEASES processing releases in group 1/1: default//foo WARNING: release foo needs bar, but bar is not installed due to installed: false. Either mark bar as installed or remove bar from foo's needs -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo stable/mychart1 3.1.0 0s -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION bar 0s diff --git a/pkg/app/testdata/testapply/delete_foo_and_bar_when_bar_needs_foo/log b/pkg/app/testdata/testapply/delete_foo_and_bar_when_bar_needs_foo/log index 4cf0f4cb..72e1968b 100644 --- a/pkg/app/testdata/testapply/delete_foo_and_bar_when_bar_needs_foo/log +++ b/pkg/app/testdata/testapply/delete_foo_and_bar_when_bar_needs_foo/log @@ -20,7 +20,7 @@ GROUP RELEASES processing releases in group 1/2: default//bar processing releases in group 2/2: default//foo -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION bar 0s foo 0s diff --git a/pkg/app/testdata/testapply/delete_foo_and_bar_when_foo_needs_bar/log b/pkg/app/testdata/testapply/delete_foo_and_bar_when_foo_needs_bar/log index a15ac481..2f5a6c29 100644 --- a/pkg/app/testdata/testapply/delete_foo_and_bar_when_foo_needs_bar/log +++ b/pkg/app/testdata/testapply/delete_foo_and_bar_when_foo_needs_bar/log @@ -20,7 +20,7 @@ GROUP RELEASES processing releases in group 1/2: default//foo processing releases in group 2/2: default//bar -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION foo 0s bar 0s diff --git a/pkg/app/testdata/testapply/delete_foo_when_bar_needs_foo/log b/pkg/app/testdata/testapply/delete_foo_when_bar_needs_foo/log index 5b557587..56fbd0dc 100644 --- a/pkg/app/testdata/testapply/delete_foo_when_bar_needs_foo/log +++ b/pkg/app/testdata/testapply/delete_foo_when_bar_needs_foo/log @@ -26,12 +26,12 @@ GROUP RELEASES processing releases in group 1/1: default//bar WARNING: release bar needs foo, but foo is not installed due to installed: false. Either mark foo as installed or remove foo from bar's needs -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION bar stable/mychart2 3.1.0 0s -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION foo 0s diff --git a/pkg/app/testdata/testapply/delete_foo_when_foo_needs_bar/log b/pkg/app/testdata/testapply/delete_foo_when_foo_needs_bar/log index 659b1fd2..63cc1b4b 100644 --- a/pkg/app/testdata/testapply/delete_foo_when_foo_needs_bar/log +++ b/pkg/app/testdata/testapply/delete_foo_when_foo_needs_bar/log @@ -23,12 +23,12 @@ GROUP RELEASES processing releases in group 1/1: default//bar -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION bar stable/mychart2 3.1.0 0s -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION foo 0s diff --git a/pkg/app/testdata/testapply/install-with-upgrade-with-reinstallifforbidden/log b/pkg/app/testdata/testapply/install-with-upgrade-with-reinstallifforbidden/log index 19044a1e..fcf491dc 100644 --- a/pkg/app/testdata/testapply/install-with-upgrade-with-reinstallifforbidden/log +++ b/pkg/app/testdata/testapply/install-with-upgrade-with-reinstallifforbidden/log @@ -24,7 +24,7 @@ update strategy - sync success processing releases in group 2/2: default//foo getting deployed release version failed: Failed to get the version for: mychart1 -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION baz stable/mychart3 3.1.0 0s bar stable/mychart2 3.1.0 0s diff --git a/pkg/app/testdata/testapply/install-with-upgrade-with-skip-diff-on-install-with-reinstallifforbidden/log b/pkg/app/testdata/testapply/install-with-upgrade-with-skip-diff-on-install-with-reinstallifforbidden/log index 19044a1e..fcf491dc 100644 --- a/pkg/app/testdata/testapply/install-with-upgrade-with-skip-diff-on-install-with-reinstallifforbidden/log +++ b/pkg/app/testdata/testapply/install-with-upgrade-with-skip-diff-on-install-with-reinstallifforbidden/log @@ -24,7 +24,7 @@ update strategy - sync success processing releases in group 2/2: default//foo getting deployed release version failed: Failed to get the version for: mychart1 -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION baz stable/mychart3 3.1.0 0s bar stable/mychart2 3.1.0 0s diff --git a/pkg/app/testdata/testapply/install-with-upgrade-with-skip-diff-on-install/log b/pkg/app/testdata/testapply/install-with-upgrade-with-skip-diff-on-install/log index f3becb67..bfe160cb 100644 --- a/pkg/app/testdata/testapply/install-with-upgrade-with-skip-diff-on-install/log +++ b/pkg/app/testdata/testapply/install-with-upgrade-with-skip-diff-on-install/log @@ -22,7 +22,7 @@ processing releases in group 1/2: default//baz, default//bar processing releases in group 2/2: default//foo getting deployed release version failed: Failed to get the version for: mychart1 -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION baz stable/mychart3 3.1.0 0s bar stable/mychart2 3.1.0 0s diff --git a/pkg/app/testdata/testapply/install-with-upgrade-with-validation-control/log b/pkg/app/testdata/testapply/install-with-upgrade-with-validation-control/log index f3becb67..bfe160cb 100644 --- a/pkg/app/testdata/testapply/install-with-upgrade-with-validation-control/log +++ b/pkg/app/testdata/testapply/install-with-upgrade-with-validation-control/log @@ -22,7 +22,7 @@ processing releases in group 1/2: default//baz, default//bar processing releases in group 2/2: default//foo getting deployed release version failed: Failed to get the version for: mychart1 -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION baz stable/mychart3 3.1.0 0s bar stable/mychart2 3.1.0 0s diff --git a/pkg/app/testdata/testapply/install/log b/pkg/app/testdata/testapply/install/log index 56bf3acf..f742e710 100644 --- a/pkg/app/testdata/testapply/install/log +++ b/pkg/app/testdata/testapply/install/log @@ -24,7 +24,7 @@ getting deployed release version failed: unexpected list key: listkey(filter=^ba processing releases in group 2/2: default//foo getting deployed release version failed: unexpected list key: listkey(filter=^foo$,flags=--kube-context default --uninstalling --deployed --failed --pending) not found in -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION baz stable/mychart3 0s bar stable/mychart2 0s diff --git a/pkg/app/testdata/testapply/smoke/log b/pkg/app/testdata/testapply/smoke/log index 2daeed71..d35748d1 100644 --- a/pkg/app/testdata/testapply/smoke/log +++ b/pkg/app/testdata/testapply/smoke/log @@ -46,7 +46,7 @@ processing releases in group 3/5: default//anotherbackend processing releases in group 4/5: default//backend-v2 processing releases in group 5/5: default//frontend-v3 -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION logging charts/fluent-bit 3.1.0 0s front-proxy stable/envoy 3.1.0 0s @@ -57,7 +57,7 @@ backend-v2 charts/backend 3.1.0 0s frontend-v3 charts/frontend 3.1.0 0s -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION frontend-v1 0s backend-v1 0s diff --git a/pkg/app/testdata/testapply/upgrade_when_bar_needs_foo,_with_ns_override/log b/pkg/app/testdata/testapply/upgrade_when_bar_needs_foo,_with_ns_override/log index f8951d02..3d997670 100644 --- a/pkg/app/testdata/testapply/upgrade_when_bar_needs_foo,_with_ns_override/log +++ b/pkg/app/testdata/testapply/upgrade_when_bar_needs_foo,_with_ns_override/log @@ -22,7 +22,7 @@ getting deployed release version failed: Failed to get the version for: mychart1 processing releases in group 2/2: default/testNamespace/bar getting deployed release version failed: Failed to get the version for: mychart2 -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo testNamespace stable/mychart1 0s bar testNamespace stable/mychart2 0s diff --git a/pkg/app/testdata/testapply/upgrade_when_bar_needs_foo/log b/pkg/app/testdata/testapply/upgrade_when_bar_needs_foo/log index 46a172b1..a7ebee8a 100644 --- a/pkg/app/testdata/testapply/upgrade_when_bar_needs_foo/log +++ b/pkg/app/testdata/testapply/upgrade_when_bar_needs_foo/log @@ -22,7 +22,7 @@ getting deployed release version failed: Failed to get the version for: mychart1 processing releases in group 2/2: default//bar getting deployed release version failed: Failed to get the version for: mychart2 -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo stable/mychart1 0s bar stable/mychart2 0s diff --git a/pkg/app/testdata/testapply/upgrade_when_foo_needs_bar,_with_ns_override/log b/pkg/app/testdata/testapply/upgrade_when_foo_needs_bar,_with_ns_override/log index dc4f817a..b3c0302d 100644 --- a/pkg/app/testdata/testapply/upgrade_when_foo_needs_bar,_with_ns_override/log +++ b/pkg/app/testdata/testapply/upgrade_when_foo_needs_bar,_with_ns_override/log @@ -22,7 +22,7 @@ getting deployed release version failed: Failed to get the version for: mychart2 processing releases in group 2/2: default/testNamespace/foo getting deployed release version failed: Failed to get the version for: mychart1 -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION bar testNamespace stable/mychart2 0s foo testNamespace stable/mychart1 0s diff --git a/pkg/app/testdata/testapply/upgrade_when_foo_needs_bar/log b/pkg/app/testdata/testapply/upgrade_when_foo_needs_bar/log index e4a563ea..a9385727 100644 --- a/pkg/app/testdata/testapply/upgrade_when_foo_needs_bar/log +++ b/pkg/app/testdata/testapply/upgrade_when_foo_needs_bar/log @@ -22,7 +22,7 @@ getting deployed release version failed: Failed to get the version for: mychart2 processing releases in group 2/2: default//foo getting deployed release version failed: Failed to get the version for: mychart1 -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION bar stable/mychart2 0s foo stable/mychart1 0s diff --git a/pkg/app/testdata/testapply/upgrade_when_ns1/foo_needs_ns1/bar_and_ns2/bar_is_disabled/log b/pkg/app/testdata/testapply/upgrade_when_ns1/foo_needs_ns1/bar_and_ns2/bar_is_disabled/log index a8776915..987bcef0 100644 --- a/pkg/app/testdata/testapply/upgrade_when_ns1/foo_needs_ns1/bar_and_ns2/bar_is_disabled/log +++ b/pkg/app/testdata/testapply/upgrade_when_ns1/foo_needs_ns1/bar_and_ns2/bar_is_disabled/log @@ -28,13 +28,13 @@ getting deployed release version failed: Failed to get the version for: mychart2 processing releases in group 2/2: default/ns1/foo getting deployed release version failed: Failed to get the version for: mychart1 -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION bar ns1 stable/mychart2 0s foo ns1 stable/mychart1 0s -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION bar ns2 0s diff --git a/pkg/app/testdata/testapply/upgrade_when_ns1/foo_needs_ns2/bar/log b/pkg/app/testdata/testapply/upgrade_when_ns1/foo_needs_ns2/bar/log index de9d660f..897b48dd 100644 --- a/pkg/app/testdata/testapply/upgrade_when_ns1/foo_needs_ns2/bar/log +++ b/pkg/app/testdata/testapply/upgrade_when_ns1/foo_needs_ns2/bar/log @@ -22,7 +22,7 @@ getting deployed release version failed: Failed to get the version for: mychart2 processing releases in group 2/2: default/ns1/foo getting deployed release version failed: Failed to get the version for: mychart1 -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION bar ns2 stable/mychart2 0s foo ns1 stable/mychart1 0s diff --git a/pkg/app/testdata/testapply/upgrade_when_ns2/bar_needs_ns1/foo/log b/pkg/app/testdata/testapply/upgrade_when_ns2/bar_needs_ns1/foo/log index 2a5224ce..3be04c0e 100644 --- a/pkg/app/testdata/testapply/upgrade_when_ns2/bar_needs_ns1/foo/log +++ b/pkg/app/testdata/testapply/upgrade_when_ns2/bar_needs_ns1/foo/log @@ -22,7 +22,7 @@ getting deployed release version failed: Failed to get the version for: mychart1 processing releases in group 2/2: default/ns2/bar getting deployed release version failed: Failed to get the version for: mychart2 -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo ns1 stable/mychart1 0s bar ns2 stable/mychart2 0s diff --git a/pkg/app/testdata/testapply/upgrades_with_good_selector_with_--skip-needs=true/log b/pkg/app/testdata/testapply/upgrades_with_good_selector_with_--skip-needs=true/log index 6af67687..7cf8a83d 100644 --- a/pkg/app/testdata/testapply/upgrades_with_good_selector_with_--skip-needs=true/log +++ b/pkg/app/testdata/testapply/upgrades_with_good_selector_with_--skip-needs=true/log @@ -47,7 +47,7 @@ GROUP RELEASES processing releases in group 1/2: default/default/external-secrets processing releases in group 2/2: default/default/my-release -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION external-secrets default incubator/raw 3.1.0 0s my-release default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_2/deduplicate_by_--selector/log b/pkg/app/testdata/testapply_2/deduplicate_by_--selector/log index e545c63c..3982cefa 100644 --- a/pkg/app/testdata/testapply_2/deduplicate_by_--selector/log +++ b/pkg/app/testdata/testapply_2/deduplicate_by_--selector/log @@ -15,7 +15,7 @@ GROUP RELEASES processing releases in group 1/1: default/default/foo -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_2/include-transitive-needs=true/log b/pkg/app/testdata/testapply_2/include-transitive-needs=true/log index a4405c6a..60bb4ec0 100644 --- a/pkg/app/testdata/testapply_2/include-transitive-needs=true/log +++ b/pkg/app/testdata/testapply_2/include-transitive-needs=true/log @@ -48,7 +48,7 @@ processing releases in group 1/3: default//serviceC processing releases in group 2/3: default//serviceB processing releases in group 3/3: default//serviceA -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION serviceC my/chart 3.1.0 0s serviceB my/chart 3.1.0 0s diff --git a/pkg/app/testdata/testapply_2/select_single_release_from_helmfile_with_two_duplicates/log b/pkg/app/testdata/testapply_2/select_single_release_from_helmfile_with_two_duplicates/log index 7fec524f..9d96a630 100644 --- a/pkg/app/testdata/testapply_2/select_single_release_from_helmfile_with_two_duplicates/log +++ b/pkg/app/testdata/testapply_2/select_single_release_from_helmfile_with_two_duplicates/log @@ -15,7 +15,7 @@ GROUP RELEASES processing releases in group 1/1: default/default/foo -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true/log b/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true/log index 62337d68..727cdb1e 100644 --- a/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true/log +++ b/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true/log @@ -52,7 +52,7 @@ processing releases in group 1/3: default/kube-system/kubernetes-external-secret processing releases in group 2/3: default/default/external-secrets processing releases in group 3/3: default/default/my-release -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION kubernetes-external-secrets kube-system incubator/raw 3.1.0 0s external-secrets default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true_but_no_diff_on_needed_release/log b/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true_but_no_diff_on_needed_release/log index 77069b44..83cd53b3 100644 --- a/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true_but_no_diff_on_needed_release/log +++ b/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true_but_no_diff_on_needed_release/log @@ -49,7 +49,7 @@ GROUP RELEASES processing releases in group 1/2: default/default/external-secrets processing releases in group 2/2: default/default/my-release -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION external-secrets default incubator/raw 3.1.0 0s my-release default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true_with_installed_but_disabled_release/log b/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true_with_installed_but_disabled_release/log index 1a00d70c..ec811507 100644 --- a/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true_with_installed_but_disabled_release/log +++ b/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true_with_installed_but_disabled_release/log @@ -59,13 +59,13 @@ processing releases in group 1/2: default/default/external-secrets WARNING: release external-secrets needs kubernetes-external-secrets, but kubernetes-external-secrets is not installed due to installed: false. Either mark kubernetes-external-secrets as installed or remove kubernetes-external-secrets from external-secrets's needs processing releases in group 2/2: default/default/my-release -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION external-secrets default incubator/raw 3.1.0 0s my-release default incubator/raw 3.1.0 0s -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION kubernetes-external-secrets kube-system 0s diff --git a/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true_with_not_installed_and_disabled_release/log b/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true_with_not_installed_and_disabled_release/log index 58c602da..5abca8a4 100644 --- a/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true_with_not_installed_and_disabled_release/log +++ b/pkg/app/testdata/testapply_2/skip-needs=false_include-needs=true_with_not_installed_and_disabled_release/log @@ -53,7 +53,7 @@ processing releases in group 1/2: default/default/external-secrets WARNING: release external-secrets needs kubernetes-external-secrets, but kubernetes-external-secrets is not installed due to installed: false. Either mark kubernetes-external-secrets as installed or remove kubernetes-external-secrets from external-secrets's needs processing releases in group 2/2: default/default/my-release -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION external-secrets default incubator/raw 3.1.0 0s my-release default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_2/skip-needs=true/log b/pkg/app/testdata/testapply_2/skip-needs=true/log index 6af67687..7cf8a83d 100644 --- a/pkg/app/testdata/testapply_2/skip-needs=true/log +++ b/pkg/app/testdata/testapply_2/skip-needs=true/log @@ -47,7 +47,7 @@ GROUP RELEASES processing releases in group 1/2: default/default/external-secrets processing releases in group 2/2: default/default/my-release -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION external-secrets default incubator/raw 3.1.0 0s my-release default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_2/skip-needs=true_with_no_diff_on_a_release/log b/pkg/app/testdata/testapply_2/skip-needs=true_with_no_diff_on_a_release/log index 17ac4bf2..2f5b078b 100644 --- a/pkg/app/testdata/testapply_2/skip-needs=true_with_no_diff_on_a_release/log +++ b/pkg/app/testdata/testapply_2/skip-needs=true_with_no_diff_on_a_release/log @@ -44,7 +44,7 @@ GROUP RELEASES processing releases in group 1/1: default/default/external-secrets -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION external-secrets default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_2/timeout_flag_is_passed_to_helm/log b/pkg/app/testdata/testapply_2/timeout_flag_is_passed_to_helm/log index 5e22123e..31e950ae 100644 --- a/pkg/app/testdata/testapply_2/timeout_flag_is_passed_to_helm/log +++ b/pkg/app/testdata/testapply_2/timeout_flag_is_passed_to_helm/log @@ -15,7 +15,7 @@ GROUP RELEASES processing releases in group 1/1: default/default/my-release -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION my-release default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true/log b/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true/log index 3706c59d..d713d85f 100644 --- a/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true/log +++ b/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true/log @@ -52,7 +52,7 @@ processing releases in group 1/3: kube-system/kubernetes-external-secrets processing releases in group 2/3: default/external-secrets processing releases in group 3/3: default/my-release -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION kubernetes-external-secrets kube-system incubator/raw 3.1.0 0s external-secrets default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true_but_no_diff_on_needed_release/log b/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true_but_no_diff_on_needed_release/log index 7f9b3818..ed93f7b8 100644 --- a/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true_but_no_diff_on_needed_release/log +++ b/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true_but_no_diff_on_needed_release/log @@ -49,7 +49,7 @@ GROUP RELEASES processing releases in group 1/2: default/external-secrets processing releases in group 2/2: default/my-release -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION external-secrets default incubator/raw 3.1.0 0s my-release default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true_with_installed_but_disabled_release/log b/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true_with_installed_but_disabled_release/log index 0144f44a..60d7dcc0 100644 --- a/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true_with_installed_but_disabled_release/log +++ b/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true_with_installed_but_disabled_release/log @@ -59,13 +59,13 @@ processing releases in group 1/2: default/external-secrets WARNING: release external-secrets needs kubernetes-external-secrets, but kubernetes-external-secrets is not installed due to installed: false. Either mark kubernetes-external-secrets as installed or remove kubernetes-external-secrets from external-secrets's needs processing releases in group 2/2: default/my-release -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION external-secrets default incubator/raw 3.1.0 0s my-release default incubator/raw 3.1.0 0s -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION kubernetes-external-secrets kube-system 0s diff --git a/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true_with_not_installed_and_disabled_release/log b/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true_with_not_installed_and_disabled_release/log index b67a15d7..339ad7bf 100644 --- a/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true_with_not_installed_and_disabled_release/log +++ b/pkg/app/testdata/testapply_3/skip-needs=false_include-needs=true_with_not_installed_and_disabled_release/log @@ -53,7 +53,7 @@ processing releases in group 1/2: default/external-secrets WARNING: release external-secrets needs kubernetes-external-secrets, but kubernetes-external-secrets is not installed due to installed: false. Either mark kubernetes-external-secrets as installed or remove kubernetes-external-secrets from external-secrets's needs processing releases in group 2/2: default/my-release -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION external-secrets default incubator/raw 3.1.0 0s my-release default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_3/skip-needs=true/log b/pkg/app/testdata/testapply_3/skip-needs=true/log index a01ac5c9..c72bf97c 100644 --- a/pkg/app/testdata/testapply_3/skip-needs=true/log +++ b/pkg/app/testdata/testapply_3/skip-needs=true/log @@ -47,7 +47,7 @@ GROUP RELEASES processing releases in group 1/2: default/external-secrets processing releases in group 2/2: default/my-release -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION external-secrets default incubator/raw 3.1.0 0s my-release default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_3/skip-needs=true_with_no_diff_on_a_release/log b/pkg/app/testdata/testapply_3/skip-needs=true_with_no_diff_on_a_release/log index 47aff850..f8192f63 100644 --- a/pkg/app/testdata/testapply_3/skip-needs=true_with_no_diff_on_a_release/log +++ b/pkg/app/testdata/testapply_3/skip-needs=true_with_no_diff_on_a_release/log @@ -44,7 +44,7 @@ GROUP RELEASES processing releases in group 1/1: default/external-secrets -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION external-secrets default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_hooks/apply_release_with_preapply_hook#01/log b/pkg/app/testdata/testapply_hooks/apply_release_with_preapply_hook#01/log index fb797d3c..d34e77b1 100644 --- a/pkg/app/testdata/testapply_hooks/apply_release_with_preapply_hook#01/log +++ b/pkg/app/testdata/testapply_hooks/apply_release_with_preapply_hook#01/log @@ -8,7 +8,7 @@ hook[preapply] logs | hook[presync] logs | foo hook[presync] logs | -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo default incubator/raw 0s diff --git a/pkg/app/testdata/testapply_hooks/apply_release_with_preapply_hook#02/log b/pkg/app/testdata/testapply_hooks/apply_release_with_preapply_hook#02/log index fcd71882..df25affd 100644 --- a/pkg/app/testdata/testapply_hooks/apply_release_with_preapply_hook#02/log +++ b/pkg/app/testdata/testapply_hooks/apply_release_with_preapply_hook#02/log @@ -2,7 +2,7 @@ hook[presync] logs | foo hook[presync] logs | -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo default incubator/raw 0s diff --git a/pkg/app/testdata/testapply_hooks/apply_release_with_preapply_hook/log b/pkg/app/testdata/testapply_hooks/apply_release_with_preapply_hook/log index 6eb18713..f5a509ef 100644 --- a/pkg/app/testdata/testapply_hooks/apply_release_with_preapply_hook/log +++ b/pkg/app/testdata/testapply_hooks/apply_release_with_preapply_hook/log @@ -2,7 +2,7 @@ hook[preapply] logs | foo hook[preapply] logs | -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo default incubator/raw 0s diff --git a/pkg/app/testdata/testapply_hooks/hooks_are_not_run_on_alreadyd_uninstalled_release/log b/pkg/app/testdata/testapply_hooks/hooks_are_not_run_on_alreadyd_uninstalled_release/log index 1b743d1b..1521407a 100644 --- a/pkg/app/testdata/testapply_hooks/hooks_are_not_run_on_alreadyd_uninstalled_release/log +++ b/pkg/app/testdata/testapply_hooks/hooks_are_not_run_on_alreadyd_uninstalled_release/log @@ -11,7 +11,7 @@ hook[preapply] logs | hook[presync] logs | foo hook[presync] logs | -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo default incubator/raw 3.1.0 0s diff --git a/pkg/app/testdata/testapply_hooks/hooks_are_not_run_on_disabled_release/log b/pkg/app/testdata/testapply_hooks/hooks_are_not_run_on_disabled_release/log index fb797d3c..d34e77b1 100644 --- a/pkg/app/testdata/testapply_hooks/hooks_are_not_run_on_disabled_release/log +++ b/pkg/app/testdata/testapply_hooks/hooks_are_not_run_on_disabled_release/log @@ -8,7 +8,7 @@ hook[preapply] logs | hook[presync] logs | foo hook[presync] logs | -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo default incubator/raw 0s diff --git a/pkg/app/testdata/testapply_hooks/hooks_are_run_on_enabled_release/log b/pkg/app/testdata/testapply_hooks/hooks_are_run_on_enabled_release/log index 8f04cb37..a6a34570 100644 --- a/pkg/app/testdata/testapply_hooks/hooks_are_run_on_enabled_release/log +++ b/pkg/app/testdata/testapply_hooks/hooks_are_run_on_enabled_release/log @@ -17,7 +17,7 @@ hook[presync] logs | hook[presync] logs | bar hook[presync] logs | -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo default incubator/raw 0s bar default incubator/raw 0s diff --git a/pkg/app/testdata/testapply_hooks/hooks_are_run_on_to-be-uninstalled_release/log b/pkg/app/testdata/testapply_hooks/hooks_are_run_on_to-be-uninstalled_release/log index 66c1277b..7d719ccd 100644 --- a/pkg/app/testdata/testapply_hooks/hooks_are_run_on_to-be-uninstalled_release/log +++ b/pkg/app/testdata/testapply_hooks/hooks_are_run_on_to-be-uninstalled_release/log @@ -14,12 +14,12 @@ hook[presync] logs | hook[presync] logs | foo hook[presync] logs | -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION foo default incubator/raw 3.1.0 0s -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION bar default 0s diff --git a/pkg/app/testdata/testdestroy/destroy_installed_but_disabled_release/log b/pkg/app/testdata/testdestroy/destroy_installed_but_disabled_release/log index a7ec424b..affd218f 100644 --- a/pkg/app/testdata/testdestroy/destroy_installed_but_disabled_release/log +++ b/pkg/app/testdata/testdestroy/destroy_installed_but_disabled_release/log @@ -13,7 +13,7 @@ release "frontend-v1" processed processing releases in group 2/2: default//backend-v1 release "backend-v1" processed -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION frontend-v1 0s backend-v1 0s diff --git a/pkg/app/testdata/testdestroy/destroy_only_one_release_with_selector/log b/pkg/app/testdata/testdestroy/destroy_only_one_release_with_selector/log index 52a67e23..488f95eb 100644 --- a/pkg/app/testdata/testdestroy/destroy_only_one_release_with_selector/log +++ b/pkg/app/testdata/testdestroy/destroy_only_one_release_with_selector/log @@ -8,7 +8,7 @@ GROUP RELEASES processing releases in group 1/1: default//logging release "logging" processed -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION logging 0s diff --git a/pkg/app/testdata/testdestroy/helm3/log b/pkg/app/testdata/testdestroy/helm3/log index a7ec424b..affd218f 100644 --- a/pkg/app/testdata/testdestroy/helm3/log +++ b/pkg/app/testdata/testdestroy/helm3/log @@ -13,7 +13,7 @@ release "frontend-v1" processed processing releases in group 2/2: default//backend-v1 release "backend-v1" processed -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION frontend-v1 0s backend-v1 0s diff --git a/pkg/app/testdata/testdestroy/smoke/log b/pkg/app/testdata/testdestroy/smoke/log index d133d3d4..a2397c48 100644 --- a/pkg/app/testdata/testdestroy/smoke/log +++ b/pkg/app/testdata/testdestroy/smoke/log @@ -25,7 +25,7 @@ processing releases in group 5/5: default//front-proxy, default//logging release "front-proxy" processed release "logging" processed -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION frontend-v3 0s frontend-v2 0s diff --git a/pkg/app/testdata/testdestroy_2/destroy_installed_but_disabled_release/log b/pkg/app/testdata/testdestroy_2/destroy_installed_but_disabled_release/log index 9e44026f..e962caa0 100644 --- a/pkg/app/testdata/testdestroy_2/destroy_installed_but_disabled_release/log +++ b/pkg/app/testdata/testdestroy_2/destroy_installed_but_disabled_release/log @@ -13,7 +13,7 @@ release "frontend-v1" processed processing releases in group 2/2: backend-v1 release "backend-v1" processed -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION frontend-v1 0s backend-v1 0s diff --git a/pkg/app/testdata/testdestroy_2/destroy_only_one_release_with_selector/log b/pkg/app/testdata/testdestroy_2/destroy_only_one_release_with_selector/log index 06806aed..b8b3bd50 100644 --- a/pkg/app/testdata/testdestroy_2/destroy_only_one_release_with_selector/log +++ b/pkg/app/testdata/testdestroy_2/destroy_only_one_release_with_selector/log @@ -8,7 +8,7 @@ GROUP RELEASES processing releases in group 1/1: logging release "logging" processed -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION logging 0s diff --git a/pkg/app/testdata/testdestroy_2/helm3/log b/pkg/app/testdata/testdestroy_2/helm3/log index 9e44026f..e962caa0 100644 --- a/pkg/app/testdata/testdestroy_2/helm3/log +++ b/pkg/app/testdata/testdestroy_2/helm3/log @@ -13,7 +13,7 @@ release "frontend-v1" processed processing releases in group 2/2: backend-v1 release "backend-v1" processed -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION frontend-v1 0s backend-v1 0s diff --git a/pkg/app/testdata/testdestroy_2/smoke/log b/pkg/app/testdata/testdestroy_2/smoke/log index a317059f..12b4f1d0 100644 --- a/pkg/app/testdata/testdestroy_2/smoke/log +++ b/pkg/app/testdata/testdestroy_2/smoke/log @@ -25,7 +25,7 @@ processing releases in group 5/5: front-proxy, logging release "front-proxy" processed release "logging" processed -DELETED RELEASES: +========== Deleted Releases ========== NAME NAMESPACE DURATION frontend-v3 0s frontend-v2 0s diff --git a/pkg/config/apply.go b/pkg/config/apply.go index b8c4ccbb..a431d0dc 100644 --- a/pkg/config/apply.go +++ b/pkg/config/apply.go @@ -91,6 +91,11 @@ type ApplyOptions struct { TrackTimeout int // TrackLogs enables log streaming with kubedog TrackLogs bool + // TrackFailedLogs streams logs only for pods that enter a failed state. + TrackFailedLogs bool + // HelmStuckGrace, when > 0, enables the helm-killer safety valve. See + // ReleaseSpec.HelmStuckGrace for details. Value is in seconds. + HelmStuckGrace int // TrackFailOnError controls whether kubedog tracking failures cause a non-zero exit code TrackFailOnError bool // Description is the description that will be passed to helm upgrade --description @@ -326,6 +331,16 @@ func (a *ApplyImpl) TrackLogs() bool { return a.ApplyOptions.TrackLogs } +// TrackFailedLogs returns the track-failed-logs flag. +func (a *ApplyImpl) TrackFailedLogs() bool { + return a.ApplyOptions.TrackFailedLogs +} + +// HelmStuckGrace returns the helm-stuck-grace value (seconds, 0 = disabled). +func (a *ApplyImpl) HelmStuckGrace() int { + return a.ApplyOptions.HelmStuckGrace +} + // TrackFailOnError returns whether kubedog tracking failures should cause a non-zero exit code. func (a *ApplyImpl) TrackFailOnError() bool { return a.ApplyOptions.TrackFailOnError diff --git a/pkg/config/sync.go b/pkg/config/sync.go index bb9a5827..d6bbc97c 100644 --- a/pkg/config/sync.go +++ b/pkg/config/sync.go @@ -61,6 +61,11 @@ type SyncOptions struct { TrackTimeout int // TrackLogs enables log streaming with kubedog TrackLogs bool + // TrackFailedLogs streams logs only for pods that enter a failed state. + TrackFailedLogs bool + // HelmStuckGrace, when > 0, enables the helm-killer safety valve. See + // ReleaseSpec.HelmStuckGrace for details. Value is in seconds. + HelmStuckGrace int // TrackFailOnError controls whether kubedog tracking failures cause a non-zero exit code TrackFailOnError bool // Description is the description that will be passed to helm upgrade --description @@ -240,6 +245,16 @@ func (t *SyncImpl) TrackLogs() bool { return t.SyncOptions.TrackLogs } +// TrackFailedLogs returns the track-failed-logs flag. +func (t *SyncImpl) TrackFailedLogs() bool { + return t.SyncOptions.TrackFailedLogs +} + +// HelmStuckGrace returns the helm-stuck-grace value (seconds, 0 = disabled). +func (t *SyncImpl) HelmStuckGrace() int { + return t.SyncOptions.HelmStuckGrace +} + // TrackFailOnError returns whether kubedog tracking failures should cause a non-zero exit code. func (t *SyncImpl) TrackFailOnError() bool { return t.SyncOptions.TrackFailOnError diff --git a/pkg/helmexec/exec.go b/pkg/helmexec/exec.go index a2153576..59e18c6c 100644 --- a/pkg/helmexec/exec.go +++ b/pkg/helmexec/exec.go @@ -2,6 +2,7 @@ package helmexec import ( "bytes" + "context" "fmt" "io" "net/url" @@ -40,19 +41,25 @@ type HelmExecOptions struct { } type execer struct { - helmBinary string - options HelmExecOptions - version *semver.Version - runner Runner - logger *zap.SugaredLogger - kubeconfig string - kubeContext string - extra []string - decryptedSecretMutex sync.Mutex + helmBinary string + options HelmExecOptions + version *semver.Version + runner Runner + logger *zap.SugaredLogger + kubeconfig string + kubeContext string + extra []string + // decryptedSecretMutex guards decryptedSecrets. It's a pointer so that + // WithLogger() can shallow-copy the execer and share the same lock+map + // across clones (the underlying secret cache must remain a single source + // of truth across all clones). + decryptedSecretMutex *sync.Mutex decryptedSecrets map[string]*decryptedSecret writeTempFile func([]byte) (string, error) - unittestPluginOnce sync.Once - unittestPluginErr error + // unittestPluginOnce is a pointer so WithLogger() clones share the + // detection result with the original execer. + unittestPluginOnce *sync.Once + unittestPluginErr error } func NewLogger(writer io.Writer, logLevel string) *zap.SugaredLogger { @@ -182,17 +189,66 @@ func New(helmBinary string, options HelmExecOptions, logger *zap.SugaredLogger, } return &execer{ - helmBinary: helmBinary, - options: options, - version: version, - logger: logger, - kubeconfig: kubeconfig, - kubeContext: kubeContext, - runner: runner, - decryptedSecrets: make(map[string]*decryptedSecret), + helmBinary: helmBinary, + options: options, + version: version, + logger: logger, + kubeconfig: kubeconfig, + kubeContext: kubeContext, + runner: runner, + decryptedSecretMutex: &sync.Mutex{}, + decryptedSecrets: make(map[string]*decryptedSecret), + unittestPluginOnce: &sync.Once{}, }, nil } +// LoggerSwapper is the runtime contract that callers use to obtain a logger- +// scoped helm clone for capturing helm output into a buffer. It's a side +// interface (not part of helmexec.Interface) so mock implementations don't +// have to provide it. +type LoggerSwapper interface { + WithLogger(logger *zap.SugaredLogger) Interface +} + +// WithLogger returns a shallow copy of the execer that writes its log output +// to the given logger. The clone shares mutable state (decrypted-secret cache +// and its lock) with the original via the pointer mutex, so concurrent use +// of clone + original for decryption is safe. +func (helm *execer) WithLogger(logger *zap.SugaredLogger) Interface { + clone := *helm + clone.logger = logger + return &clone +} + +// ContextSwapper is the runtime contract for obtaining a context-scoped helm +// clone. Canceling the context cancels any helm subprocess started through +// the clone — the underlying ShellRunner already sends SIGINT to its +// subprocess on ctx.Done() (runner.go), so this gives callers a clean way to +// kill an in-flight helm without touching the global app context. Side +// interface so mocks don't have to implement it. +type ContextSwapper interface { + WithContext(ctx context.Context) Interface +} + +// WithContext returns a shallow copy of the execer whose runner uses the +// provided context. Canceling ctx triggers SIGINT to any helm subprocess +// started through the clone. Falls back gracefully (returns clone with the +// original runner) when the runner isn't a ShellRunner — only the kill path +// becomes a no-op in that case. +func (helm *execer) WithContext(ctx context.Context) Interface { + clone := *helm + switch r := helm.runner.(type) { + case *ShellRunner: + rClone := *r + rClone.Ctx = ctx + clone.runner = &rClone + case ShellRunner: + r.Ctx = ctx + clone.runner = r + } + return &clone +} + func (helm *execer) SetExtraArgs(args ...string) { helm.extra = args } @@ -726,7 +782,11 @@ func (helm *execer) TemplateRelease(name string, chart string, flags ...string) // but manifets. // // See https://github.com/roboll/helmfile/pull/1691#issuecomment-805636021 for more information. - helm.info(out) + // + // Helm emits one `wrote ` line per resource document in each file, so a + // multi-doc template produces N identical lines for the same path. Dedupe them + // to keep the output readable. + helm.info(dedupeWroteLines(out)) } else { // Always write to stdout for use with e.g. `helmfile template | kubectl apply -f -` helm.write(nil, out) @@ -1072,9 +1132,39 @@ func (helm *execer) azcli(name string) ([]byte, error) { return outBytes, err } +// dedupeWroteLines collapses repeated `wrote ` lines from helm-template +// stdout into a single occurrence per path (first-seen order). Helm v4 emits +// one line per resource document, so a multi-doc YAML produces N identical +// lines for the same file path; the duplicates are pure noise. +func dedupeWroteLines(out []byte) []byte { + if len(out) == 0 { + return out + } + lines := strings.Split(string(out), "\n") + seen := make(map[string]struct{}, len(lines)) + kept := make([]string, 0, len(lines)) + for _, line := range lines { + if strings.HasPrefix(line, "wrote ") { + if _, ok := seen[line]; ok { + continue + } + seen[line] = struct{}{} + } + kept = append(kept, line) + } + return []byte(strings.Join(kept, "\n")) +} + func (helm *execer) info(out []byte) { - if len(out) > 0 { - helm.logger.Infof("%s", out) + // Trim trailing newlines so the encoder's appended newline is the only + // one. Without this, helm stdout that ends in "\n" (e.g. a chart whose + // template renders only hooks) becomes a "\n" payload, and the encoder + // produces "\n\n" — extra blank lines that are very visible in + // timestamp-per-line CI logs. Skip the call entirely when the trim + // leaves nothing. + trimmed := bytes.TrimRight(out, "\n") + if len(trimmed) > 0 { + helm.logger.Infof("%s", trimmed) } } diff --git a/pkg/helmexec/exec_test.go b/pkg/helmexec/exec_test.go index 2d4dc577..2be59612 100644 --- a/pkg/helmexec/exec_test.go +++ b/pkg/helmexec/exec_test.go @@ -1708,3 +1708,47 @@ func Test_UpdatePlugin_Helm4SecretsUsesUninstallReinstall(t *testing.T) { checkInstall("secrets-getter-4.7.0.tgz") checkInstall("secrets-post-renderer-4.7.0.tgz") } + +func Test_dedupeWroteLines(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + { + name: "empty", + in: "", + want: "", + }, + { + name: "no wrote lines untouched", + in: "Found 25 resources in manifest for release system\n", + want: "Found 25 resources in manifest for release system\n", + }, + { + name: "collapses duplicate wrote lines preserving first-seen order", + in: "wrote /tmp/a/service_account.yaml\n" + + "wrote /tmp/a/service_account.yaml\n" + + "wrote /tmp/a/secrets.yaml\n" + + "wrote /tmp/a/service_account.yaml\n" + + "wrote /tmp/a/secrets.yaml\n" + + "Found 25 resources in manifest for release system", + want: "wrote /tmp/a/service_account.yaml\n" + + "wrote /tmp/a/secrets.yaml\n" + + "Found 25 resources in manifest for release system", + }, + { + name: "non-wrote duplicates kept", + in: "hello\nhello\n", + want: "hello\nhello\n", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := string(dedupeWroteLines([]byte(c.in))) + if got != c.want { + t.Errorf("dedupeWroteLines() = %q, want %q", got, c.want) + } + }) + } +} diff --git a/pkg/kubedog/display.go b/pkg/kubedog/display.go deleted file mode 100644 index 79a08436..00000000 --- a/pkg/kubedog/display.go +++ /dev/null @@ -1,312 +0,0 @@ -package kubedog - -import ( - "fmt" - "io" - "os" - "sort" - "strings" - - "github.com/werf/kubedog/pkg/tracker/daemonset" - "github.com/werf/kubedog/pkg/tracker/deployment" - "github.com/werf/kubedog/pkg/tracker/indicators" - "github.com/werf/kubedog/pkg/tracker/job" - "github.com/werf/kubedog/pkg/tracker/pod" - "github.com/werf/kubedog/pkg/tracker/statefulset" - "github.com/werf/kubedog/pkg/utils" - "golang.org/x/term" -) - -var statusProgressTableRatio = []float64{.58, .11, .12, .19} -var statusProgressSubTableRatio = []float64{.40, .15, .20, .25} - -func writeOut(out io.Writer, s string) { - _, _ = fmt.Fprint(out, s) -} - -func displayDeploymentStatusProgress(out io.Writer, resourceCaption string, status deployment.DeploymentStatus, prevStatus *deployment.DeploymentStatus) { - t := utils.NewTable(statusProgressTableRatio...) - t.SetWidth(termWidth()) - - showProgress := status.StatusGeneration > prevStatus.StatusGeneration - - replicas := "-" - if status.ReplicasIndicator != nil { - replicas = status.ReplicasIndicator.FormatTableElem(prevStatus.ReplicasIndicator, indicators.FormatTableElemOptions{ - ShowProgress: showProgress, - WithTargetValue: true, - }) - } - available := "-" - if status.AvailableIndicator != nil { - available = status.AvailableIndicator.FormatTableElem(prevStatus.AvailableIndicator, indicators.FormatTableElemOptions{ - ShowProgress: showProgress, - }) - } - uptodate := "-" - if status.UpToDateIndicator != nil { - uptodate = status.UpToDateIndicator.FormatTableElem(prevStatus.UpToDateIndicator, indicators.FormatTableElemOptions{ - ShowProgress: showProgress, - }) - } - - t.Header("DEPLOYMENT", "REPLICAS", "AVAILABLE", "UP-TO-DATE") - - args := []interface{}{resourceCaption, replicas, available, uptodate} - if status.IsFailed { - args = append(args, formatResourceError(status.FailedReason)) - } - t.Row(args...) - - displayChildPodsAndWaiting(&t, prevStatus.Pods, status.Pods, status.NewPodsNames, status.WaitingForMessages) - - writeOut(out, t.Render()) -} - -func displayStatefulSetStatusProgress(out io.Writer, resourceCaption string, status statefulset.StatefulSetStatus, prevStatus *statefulset.StatefulSetStatus) { - t := utils.NewTable(statusProgressTableRatio...) - t.SetWidth(termWidth()) - - showProgress := status.StatusGeneration > prevStatus.StatusGeneration - - replicas := "-" - if status.ReplicasIndicator != nil { - replicas = status.ReplicasIndicator.FormatTableElem(prevStatus.ReplicasIndicator, indicators.FormatTableElemOptions{ - ShowProgress: showProgress, - WithTargetValue: true, - }) - } - ready := "-" - if status.ReadyIndicator != nil { - ready = status.ReadyIndicator.FormatTableElem(prevStatus.ReadyIndicator, indicators.FormatTableElemOptions{ - ShowProgress: showProgress, - }) - } - uptodate := "-" - if status.UpToDateIndicator != nil { - uptodate = status.UpToDateIndicator.FormatTableElem(prevStatus.UpToDateIndicator, indicators.FormatTableElemOptions{ - ShowProgress: showProgress, - }) - } - - t.Header("STATEFULSET", "REPLICAS", "READY", "UP-TO-DATE") - - args := []interface{}{resourceCaption, replicas, ready, uptodate} - if status.IsFailed { - args = append(args, formatResourceError(status.FailedReason)) - } else { - for _, w := range status.WarningMessages { - args = append(args, formatResourceWarning(w)) - } - } - t.Row(args...) - - displayChildPodsAndWaiting(&t, prevStatus.Pods, status.Pods, status.NewPodsNames, status.WaitingForMessages) - - writeOut(out, t.Render()) -} - -func displayDaemonSetStatusProgress(out io.Writer, resourceCaption string, status daemonset.DaemonSetStatus, prevStatus *daemonset.DaemonSetStatus) { - t := utils.NewTable(statusProgressTableRatio...) - t.SetWidth(termWidth()) - - showProgress := status.StatusGeneration > prevStatus.StatusGeneration - - replicas := "-" - if status.ReplicasIndicator != nil { - replicas = status.ReplicasIndicator.FormatTableElem(prevStatus.ReplicasIndicator, indicators.FormatTableElemOptions{ - ShowProgress: showProgress, - WithTargetValue: true, - }) - } - available := "-" - if status.AvailableIndicator != nil { - available = status.AvailableIndicator.FormatTableElem(prevStatus.AvailableIndicator, indicators.FormatTableElemOptions{ - ShowProgress: showProgress, - }) - } - uptodate := "-" - if status.UpToDateIndicator != nil { - uptodate = status.UpToDateIndicator.FormatTableElem(prevStatus.UpToDateIndicator, indicators.FormatTableElemOptions{ - ShowProgress: showProgress, - }) - } - - t.Header("DAEMONSET", "REPLICAS", "AVAILABLE", "UP-TO-DATE") - - args := []interface{}{resourceCaption, replicas, available, uptodate} - if status.IsFailed { - args = append(args, formatResourceError(status.FailedReason)) - } - t.Row(args...) - - displayChildPodsAndWaiting(&t, prevStatus.Pods, status.Pods, status.NewPodsNames, status.WaitingForMessages) - - writeOut(out, t.Render()) -} - -func displayJobStatusProgress(out io.Writer, resourceCaption string, status job.JobStatus, prevStatus *job.JobStatus) { - t := utils.NewTable(statusProgressTableRatio...) - t.SetWidth(termWidth()) - - showProgress := status.StatusGeneration > prevStatus.StatusGeneration - - succeeded := "-" - if status.SucceededIndicator != nil { - succeeded = status.SucceededIndicator.FormatTableElem(prevStatus.SucceededIndicator, indicators.FormatTableElemOptions{ - ShowProgress: showProgress, - }) - } - - t.Header("JOB", "ACTIVE", "DURATION", "SUCCEEDED/FAILED") - - var active interface{} = "-" - if status.Active != 0 { - active = status.Active - } - failed := fmt.Sprintf("%d", status.Failed) - - args := []interface{}{resourceCaption, active, status.Age, strings.Join([]string{succeeded, failed}, "/")} - if status.IsFailed { - args = append(args, formatResourceError(status.FailedReason)) - } - t.Row(args...) - - if len(status.Pods) > 0 { - st := displayChildPodsStatusProgress(&t, prevStatus.Pods, status.Pods, nil, showProgress) - extraMsg := "" - if len(status.WaitingForMessages) > 0 { - extraMsg += "---\n" - extraMsg += utils.BlueF("Waiting for: %s", strings.Join(status.WaitingForMessages, ", ")) - } - st.Commit(extraMsg) - } - - writeOut(out, t.Render()) -} - -func displayChildPodsAndWaiting(t *utils.Table, prevPods, pods map[string]pod.PodStatus, newPodsNames []string, waitingForMessages []string) { - if len(pods) > 0 { - st := displayChildPodsStatusProgress(t, prevPods, pods, newPodsNames, true) - extraMsg := "" - if len(waitingForMessages) > 0 { - extraMsg += "---\n" - extraMsg += utils.BlueF("Waiting for: %s", strings.Join(waitingForMessages, ", ")) - } - st.Commit(extraMsg) - } -} - -func displayChildPodsStatusProgress(t *utils.Table, prevPods, pods map[string]pod.PodStatus, newPodsNames []string, showProgress bool) *utils.Table { - subT := t.SubTable(statusProgressSubTableRatio...) - st := &subT - - st.Header("POD", "READY", "RESTARTS", "STATUS") - - podsNames := make([]string, 0, len(pods)) - for podName := range pods { - podsNames = append(podsNames, podName) - } - sort.Strings(podsNames) - - var podRows [][]interface{} - - newPodSet := make(map[string]struct{}, len(newPodsNames)) - for _, name := range newPodsNames { - newPodSet[name] = struct{}{} - } - - for _, podName := range podsNames { - var podRow []interface{} - - _, isPodNew := newPodSet[podName] - - prevPodStatus := prevPods[podName] - podStatus := pods[podName] - - isReady := false - if podStatus.StatusIndicator != nil { - isReady = podStatus.StatusIndicator.IsReady() - } - - resource := formatPodResourceCaption(podName, isReady, podStatus.IsFailed, isPodNew) - ready := fmt.Sprintf("%d/%d", podStatus.ReadyContainers, podStatus.TotalContainers) - - status := "-" - if podStatus.StatusIndicator != nil { - status = podStatus.StatusIndicator.FormatTableElem(prevPodStatus.StatusIndicator, indicators.FormatTableElemOptions{ - ShowProgress: showProgress, - IsResourceNew: isPodNew, - }) - } - - podRow = append(podRow, resource, ready, podStatus.Restarts, status) - if podStatus.IsFailed { - podRow = append(podRow, formatResourceError(podStatus.FailedReason)) - } - - podRows = append(podRows, podRow) - } - - st.Rows(podRows...) - - return st -} - -func formatResourceCaption(caption string, isReady, isFailed bool) string { - switch { - case isReady: - return utils.GreenF("%s", caption) - case isFailed: - return utils.RedF("%s", caption) - default: - return utils.YellowF("%s", caption) - } -} - -func formatPodResourceCaption(podName string, isReady, isFailed, isNew bool) string { - if !isNew { - return podName - } - return formatResourceCaption(podName, isReady, isFailed) -} - -func formatResourceError(reason string) string { - return utils.RedF("error: %s", reason) -} - -func formatResourceWarning(reason string) string { - return utils.YellowF("warning: %s", reason) -} - -func termWidth() int { - if w, _, err := term.GetSize(int(os.Stderr.Fd())); err == nil && w > 0 { - return w - } - return 140 -} - -func displayCanaryStatus(out io.Writer, resourceCaption string, status CanaryStatusView) { - var parts []string - if status.Phase != "" { - parts = append(parts, fmt.Sprintf("phase %s", status.Phase)) - } - if status.Age != "" { - parts = append(parts, fmt.Sprintf("age %s", status.Age)) - } - msg := fmt.Sprintf("%s: %s", resourceCaption, strings.Join(parts, ", ")) - if status.IsFailed { - msg = utils.RedF("%s", msg) - } - _, _ = fmt.Fprintln(out, msg) -} - -type CanaryStatusView struct { - Phase string - Age string - IsFailed bool -} - -func statusOutput() io.Writer { - return os.Stderr -} diff --git a/pkg/kubedog/display_test.go b/pkg/kubedog/display_test.go deleted file mode 100644 index 254f3522..00000000 --- a/pkg/kubedog/display_test.go +++ /dev/null @@ -1,453 +0,0 @@ -package kubedog - -import ( - "bytes" - "os" - "strings" - "testing" - - "github.com/gookit/color" - "github.com/stretchr/testify/assert" - "github.com/werf/kubedog/pkg/tracker/daemonset" - "github.com/werf/kubedog/pkg/tracker/deployment" - "github.com/werf/kubedog/pkg/tracker/job" - "github.com/werf/kubedog/pkg/tracker/pod" - "github.com/werf/kubedog/pkg/tracker/statefulset" -) - -// TestMain forces ANSI color output so that tests asserting on escape codes -// pass in non-TTY environments such as CI runners. -func TestMain(m *testing.M) { - color.ForceColor() - os.Exit(m.Run()) -} - -// --- formatResourceCaption --- - -func TestFormatResourceCaption_Ready(t *testing.T) { - result := formatResourceCaption("deploy/myapp", true, false) - assert.Contains(t, result, "deploy/myapp") - // Green ANSI escape should be present - assert.Contains(t, result, "\033[") -} - -func TestFormatResourceCaption_Failed(t *testing.T) { - result := formatResourceCaption("deploy/myapp", false, true) - assert.Contains(t, result, "deploy/myapp") - assert.Contains(t, result, "\033[") -} - -func TestFormatResourceCaption_InProgress(t *testing.T) { - result := formatResourceCaption("deploy/myapp", false, false) - assert.Contains(t, result, "deploy/myapp") - // Yellow for in-progress - assert.Contains(t, result, "\033[") -} - -func TestFormatResourceCaption_ReadyTakesPrecedence(t *testing.T) { - // isReady=true should win over isFailed=true - resultReady := formatResourceCaption("x", true, false) - resultFailed := formatResourceCaption("x", false, true) - // Colors should differ - assert.NotEqual(t, resultReady, resultFailed) -} - -// --- formatPodResourceCaption --- - -func TestFormatPodResourceCaption_NotNew(t *testing.T) { - result := formatPodResourceCaption("my-pod-abc", true, false, false) - // Not a new pod: no coloring applied, just the plain name - assert.Equal(t, "my-pod-abc", result) -} - -func TestFormatPodResourceCaption_NewAndReady(t *testing.T) { - result := formatPodResourceCaption("my-pod-abc", true, false, true) - assert.Contains(t, result, "my-pod-abc") - assert.Contains(t, result, "\033[") -} - -func TestFormatPodResourceCaption_NewAndFailed(t *testing.T) { - result := formatPodResourceCaption("my-pod-abc", false, true, true) - assert.Contains(t, result, "my-pod-abc") - assert.Contains(t, result, "\033[") -} - -func TestFormatPodResourceCaption_NewInProgress(t *testing.T) { - result := formatPodResourceCaption("my-pod-abc", false, false, true) - assert.Contains(t, result, "my-pod-abc") - assert.Contains(t, result, "\033[") -} - -// --- formatResourceError / formatResourceWarning --- - -func TestFormatResourceError(t *testing.T) { - result := formatResourceError("CrashLoopBackOff") - assert.Contains(t, result, "error:") - assert.Contains(t, result, "CrashLoopBackOff") -} - -func TestFormatResourceWarning(t *testing.T) { - result := formatResourceWarning("PodNotScheduled") - assert.Contains(t, result, "warning:") - assert.Contains(t, result, "PodNotScheduled") -} - -// --- termWidth --- - -func TestTermWidth_ReturnsPositive(t *testing.T) { - w := termWidth() - assert.Greater(t, w, 0) -} - -// --- displayDeploymentStatusProgress --- - -func TestDisplayDeploymentStatusProgress_ZeroStatus(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("deploy/myapp", false, false) - var prev deployment.DeploymentStatus - status := deployment.DeploymentStatus{} - - // Must not panic and must produce some output - assert.NotPanics(t, func() { - displayDeploymentStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.NotEmpty(t, out) - assert.Contains(t, out, "DEPLOYMENT") -} - -func TestDisplayDeploymentStatusProgress_Failed(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("deploy/myapp", false, true) - var prev deployment.DeploymentStatus - status := deployment.DeploymentStatus{ - IsFailed: true, - FailedReason: "ImagePullBackOff", - } - - assert.NotPanics(t, func() { - displayDeploymentStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "error:") - assert.Contains(t, out, "ImagePullBackOff") -} - -func TestDisplayDeploymentStatusProgress_WithWaitingMessage(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("deploy/myapp", false, false) - var prev deployment.DeploymentStatus - // WaitingForMessages is only rendered when there are pods - status := deployment.DeploymentStatus{ - StatusGeneration: 1, - WaitingForMessages: []string{"up-to-date 1->3"}, - Pods: map[string]pod.PodStatus{ - "myapp-pod-abc": {ReadyContainers: 1, TotalContainers: 1}, - }, - } - - assert.NotPanics(t, func() { - displayDeploymentStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "Waiting for:") - assert.Contains(t, out, "up-to-date 1->3") -} - -func TestDisplayDeploymentStatusProgress_WithPods(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("deploy/myapp", false, false) - prev := deployment.DeploymentStatus{} - status := deployment.DeploymentStatus{ - StatusGeneration: 1, - Pods: map[string]pod.PodStatus{ - "myapp-abc-123": {ReadyContainers: 1, TotalContainers: 1}, - }, - NewPodsNames: []string{"myapp-abc-123"}, - } - - assert.NotPanics(t, func() { - displayDeploymentStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "POD") - assert.Contains(t, out, "myapp-abc-123") -} - -// --- displayStatefulSetStatusProgress --- - -func TestDisplayStatefulSetStatusProgress_ZeroStatus(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("sts/myapp", false, false) - var prev statefulset.StatefulSetStatus - status := statefulset.StatefulSetStatus{} - - assert.NotPanics(t, func() { - displayStatefulSetStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "STATEFULSET") -} - -func TestDisplayStatefulSetStatusProgress_WithWarnings(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("sts/myapp", false, false) - var prev statefulset.StatefulSetStatus - status := statefulset.StatefulSetStatus{ - WarningMessages: []string{"PodNotScheduled: insufficient resources"}, - } - - assert.NotPanics(t, func() { - displayStatefulSetStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "warning:") - assert.Contains(t, out, "PodNotScheduled") -} - -func TestDisplayStatefulSetStatusProgress_Failed(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("sts/myapp", false, true) - var prev statefulset.StatefulSetStatus - status := statefulset.StatefulSetStatus{ - IsFailed: true, - FailedReason: "timeout waiting for ready", - } - - assert.NotPanics(t, func() { - displayStatefulSetStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "error:") - assert.Contains(t, out, "timeout waiting for ready") -} - -// --- displayDaemonSetStatusProgress --- - -func TestDisplayDaemonSetStatusProgress_ZeroStatus(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("ds/myapp", false, false) - var prev daemonset.DaemonSetStatus - status := daemonset.DaemonSetStatus{} - - assert.NotPanics(t, func() { - displayDaemonSetStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "DAEMONSET") -} - -func TestDisplayDaemonSetStatusProgress_Failed(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("ds/myapp", false, true) - var prev daemonset.DaemonSetStatus - status := daemonset.DaemonSetStatus{ - IsFailed: true, - FailedReason: "node not ready", - } - - assert.NotPanics(t, func() { - displayDaemonSetStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "error:") - assert.Contains(t, out, "node not ready") -} - -// --- displayJobStatusProgress --- - -func TestDisplayJobStatusProgress_ZeroStatus(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("job/myjob", false, false) - var prev job.JobStatus - status := job.JobStatus{} - - assert.NotPanics(t, func() { - displayJobStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "JOB") -} - -func TestDisplayJobStatusProgress_Active(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("job/myjob", false, false) - var prev job.JobStatus - status := job.JobStatus{ - StatusGeneration: 1, - } - - assert.NotPanics(t, func() { - displayJobStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "ACTIVE") -} - -func TestDisplayJobStatusProgress_Failed(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("job/myjob", false, true) - var prev job.JobStatus - status := job.JobStatus{ - IsFailed: true, - FailedReason: "BackoffLimitExceeded", - } - - assert.NotPanics(t, func() { - displayJobStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "error:") - assert.Contains(t, out, "BackoffLimitExceeded") -} - -func TestDisplayJobStatusProgress_WithWaitingMessage(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("job/myjob", false, false) - var prev job.JobStatus - status := job.JobStatus{ - WaitingForMessages: []string{"succeeded 0->1"}, - Pods: map[string]pod.PodStatus{ - "myjob-abc": {ReadyContainers: 0, TotalContainers: 1}, - }, - } - - assert.NotPanics(t, func() { - displayJobStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "Waiting for:") - assert.Contains(t, out, "succeeded 0->1") -} - -// --- displayChildPodsStatusProgress --- - -func TestDisplayChildPodsStatusProgress_Empty(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("deploy/myapp", false, false) - // With no pods, only the header should be rendered - prev := deployment.DeploymentStatus{} - status := deployment.DeploymentStatus{ - Pods: map[string]pod.PodStatus{}, - } - assert.NotPanics(t, func() { - displayDeploymentStatusProgress(&buf, caption, status, &prev) - }) - // No POD sub-table header when pods is empty - out := buf.String() - assert.NotContains(t, out, "POD") -} - -func TestDisplayChildPodsStatusProgress_NewPodSet(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("deploy/myapp", false, false) - prev := deployment.DeploymentStatus{} - // Two pods: one new, one old - status := deployment.DeploymentStatus{ - StatusGeneration: 1, - Pods: map[string]pod.PodStatus{ - "pod-new-abc": {ReadyContainers: 0, TotalContainers: 1}, - "pod-old-xyz": {ReadyContainers: 1, TotalContainers: 1}, - }, - NewPodsNames: []string{"pod-new-abc"}, - } - - assert.NotPanics(t, func() { - displayDeploymentStatusProgress(&buf, caption, status, &prev) - }) - out := buf.String() - assert.Contains(t, out, "pod-new-abc") - assert.Contains(t, out, "pod-old-xyz") -} - -func TestDisplayChildPodsStatusProgress_ManyPodsO1Check(t *testing.T) { - // Verifies O(1) new-pod detection works correctly for many pods - var buf bytes.Buffer - caption := formatResourceCaption("deploy/myapp", false, false) - prev := deployment.DeploymentStatus{} - - pods := make(map[string]pod.PodStatus) - newNames := make([]string, 0, 10) - for i := 0; i < 20; i++ { - name := strings.Repeat("a", i+1) - pods[name] = pod.PodStatus{ReadyContainers: 1, TotalContainers: 1} - if i%2 == 0 { - newNames = append(newNames, name) - } - } - status := deployment.DeploymentStatus{ - StatusGeneration: 1, - Pods: pods, - NewPodsNames: newNames, - } - - assert.NotPanics(t, func() { - displayDeploymentStatusProgress(&buf, caption, status, &prev) - }) - assert.NotEmpty(t, buf.String()) -} - -// --- displayCanaryStatus --- - -func TestDisplayCanaryStatus_Normal(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("canary/myapp", false, false) - view := CanaryStatusView{Phase: "Progressing", Age: "1m"} - - assert.NotPanics(t, func() { - displayCanaryStatus(&buf, caption, view) - }) - out := buf.String() - assert.Contains(t, out, "Progressing") - assert.Contains(t, out, "1m") -} - -func TestDisplayCanaryStatus_Failed(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("canary/myapp", false, true) - view := CanaryStatusView{Phase: "Failed", IsFailed: true} - - assert.NotPanics(t, func() { - displayCanaryStatus(&buf, caption, view) - }) - out := buf.String() - assert.Contains(t, out, "Failed") -} - -func TestDisplayCanaryStatus_Succeeded(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("canary/myapp", true, false) - view := CanaryStatusView{Phase: "Succeeded"} - - assert.NotPanics(t, func() { - displayCanaryStatus(&buf, caption, view) - }) - out := buf.String() - assert.Contains(t, out, "Succeeded") -} - -func TestDisplayCanaryStatus_EmptyPhaseAndAge(t *testing.T) { - var buf bytes.Buffer - caption := formatResourceCaption("canary/myapp", false, false) - view := CanaryStatusView{} - - assert.NotPanics(t, func() { - displayCanaryStatus(&buf, caption, view) - }) - // Should still produce output (at least the caption + newline) - assert.NotEmpty(t, buf.String()) -} - -// --- writeOut --- - -func TestWriteOut(t *testing.T) { - var buf bytes.Buffer - writeOut(&buf, "hello world") - assert.Equal(t, "hello world", buf.String()) -} - -func TestWriteOut_Empty(t *testing.T) { - var buf bytes.Buffer - writeOut(&buf, "") - assert.Equal(t, "", buf.String()) -} diff --git a/pkg/kubedog/main_test.go b/pkg/kubedog/main_test.go new file mode 100644 index 00000000..49bbf99f --- /dev/null +++ b/pkg/kubedog/main_test.go @@ -0,0 +1,16 @@ +package kubedog + +import ( + "os" + "testing" + + "github.com/gookit/color" +) + +// TestMain forces color output on for the whole kubedog test binary so ANSI +// escapes are deterministic regardless of whether the test runner is attached +// to a TTY. +func TestMain(m *testing.M) { + color.ForceColor() + os.Exit(m.Run()) +} diff --git a/pkg/kubedog/options.go b/pkg/kubedog/options.go index 8e3293cc..0aece172 100644 --- a/pkg/kubedog/options.go +++ b/pkg/kubedog/options.go @@ -3,9 +3,21 @@ package kubedog import ( "time" + "k8s.io/apimachinery/pkg/types" + "github.com/helmfile/helmfile/pkg/resource" ) +// ResourceBaseline records a resource's identity and generation captured +// before an in-flight change (e.g. helm upgrade). The tracker uses it to +// distinguish "still observing the pre-change state" from "the change has +// landed in the cluster" and only then attaches its readiness logic. +type ResourceBaseline struct { + UID types.UID + Generation int64 + Exists bool +} + type TrackMode string const ( @@ -15,12 +27,26 @@ const ( ) type TrackOptions struct { - Timeout time.Duration - Logs bool - LogsSince time.Duration - Filter *resource.FilterConfig - QPS float32 - Burst int + Timeout time.Duration + // Logs enables emitting logs for every pod kubedog observes. + Logs bool + // FailedLogsOnly enables capturing logs in the background and emitting + // them only for pods that enter a failed state (CrashLoopBackOff, Error, + // ImagePullBackOff, etc.). Has no effect when Logs is true. + FailedLogsOnly bool + LogsSince time.Duration + Filter *resource.FilterConfig + QPS float32 + Burst int + // Baselines holds the pre-change state of each resource keyed by + // "Kind/Namespace/Name". When set, the tracker delays attaching kubedog + // to a resource until its UID changes or its generation increments past + // the recorded baseline — preventing false "ready" verdicts that would + // otherwise come from observing the old rolled-out state. + Baselines map[string]ResourceBaseline + // Color enables ANSI color escapes in the progress printer output. + // When false the printer emits plain text regardless of TTY detection. + Color bool } func NewTrackOptions() *TrackOptions { @@ -56,3 +82,18 @@ func (o *TrackOptions) WithBurst(burst int) *TrackOptions { o.Burst = burst return o } + +func (o *TrackOptions) WithBaselines(baselines map[string]ResourceBaseline) *TrackOptions { + o.Baselines = baselines + return o +} + +func (o *TrackOptions) WithColor(color bool) *TrackOptions { + o.Color = color + return o +} + +func (o *TrackOptions) WithFailedLogsOnly(v bool) *TrackOptions { + o.FailedLogsOnly = v + return o +} diff --git a/pkg/kubedog/printer.go b/pkg/kubedog/printer.go new file mode 100644 index 00000000..8e3746ef --- /dev/null +++ b/pkg/kubedog/printer.go @@ -0,0 +1,929 @@ +package kubedog + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/werf/kubedog/pkg/trackers/dyntracker/logstore" + "github.com/werf/kubedog/pkg/trackers/dyntracker/statestore" + kdutil "github.com/werf/kubedog/pkg/trackers/dyntracker/util" + "go.uber.org/zap" +) + +const ( + progressInterval = 10 * time.Second + logsInterval = 10 * time.Second + heartbeatInterval = 2 * time.Minute +) + +// ANSI escape codes. Hard-coded to keep the dependency list small — these are +// the standard 16-color SGR sequences and work on any reasonable terminal. +const ( + ansiReset = "\x1b[0m" + ansiGreen = "\x1b[32m" + ansiYellow = "\x1b[33m" + ansiRed = "\x1b[31m" + ansiCyan = "\x1b[36m" + ansiBlue = "\x1b[34m" + ansiGray = "\x1b[90m" + ansiBold = "\x1b[1m" +) + +// HeaderDivider returns a section header decorated so it stays visually +// prominent even in CI log viewers that collapse leading newlines (e.g. +// GitLab CI). The "==" borders survive newline-stripping because they live +// on the same line as the title. +func HeaderDivider(title string) string { + return "========== " + title + " ==========" +} + +// HeaderDividerStyled is HeaderDivider with the bold+blue style nelm uses for +// its section titles. When useColor is false it returns the plain divider so +// non-TTY/CI-without-color callers see identical bytes. +func HeaderDividerStyled(title string, useColor bool) string { + h := HeaderDivider(title) + if !useColor { + return h + } + return ansiBold + ansiBlue + h + ansiReset +} + +// StyleWarning wraps text in bold+yellow so it stands out among the regular +// info lines in CI logs without screaming "everything is on fire" — that's +// what red is for. Use this for messages an operator should notice and act +// on (e.g. orphan resource cleanup) but that don't represent a release-level +// failure. No-op when useColor is false so non-TTY consumers see clean text. +func StyleWarning(s string, useColor bool) string { + if !useColor { + return s + } + return ansiBold + ansiYellow + s + ansiReset +} + +// gateStatuses holds per-resource "waiting for freshness" messages keyed by +// BaselineKey. Producers (tracker goroutines) call set/clear; the printer +// reads via snapshot. +type gateStatuses struct { + mu sync.RWMutex + m map[string]string +} + +func newGateStatuses() *gateStatuses { + return &gateStatuses{m: make(map[string]string)} +} + +func (g *gateStatuses) set(key, status string) { + g.mu.Lock() + g.m[key] = status + g.mu.Unlock() +} + +func (g *gateStatuses) clear(key string) { + g.mu.Lock() + delete(g.m, key) + g.mu.Unlock() +} + +func (g *gateStatuses) snapshot() map[string]string { + g.mu.RLock() + defer g.mu.RUnlock() + out := make(map[string]string, len(g.m)) + for k, v := range g.m { + out[k] = v + } + return out +} + +// skippedKeys records the ResourceIDs of tasks the printer should hide +// (typically because the upstream operation finished without changing the +// resource and the task's "progressing" state would be misleading). +type skippedKeys struct { + mu sync.RWMutex + m map[string]struct{} +} + +func newSkippedKeys() *skippedKeys { + return &skippedKeys{m: make(map[string]struct{})} +} + +func (s *skippedKeys) add(id string) { + s.mu.Lock() + s.m[id] = struct{}{} + s.mu.Unlock() +} + +func (s *skippedKeys) has(id string) bool { + s.mu.RLock() + defer s.mu.RUnlock() + _, ok := s.m[id] + return ok +} + +// kubedog's util.ResourceID returns ":::". + +// formatResourceLabel renders a resource identifier, omitting the namespace +// when it matches the printer-wide common namespace. Cluster-scoped +// resources (empty namespace) always render as Kind/name. The goal is to +// stop repeating the same namespace on every row of a single-namespace +// release without losing the namespace when releases genuinely span +// multiple namespaces. +func formatResourceLabel(kind, ns, name, commonNS string) string { + if ns == "" || (commonNS != "" && ns == commonNS) { + return fmt.Sprintf("%s/%s", kind, name) + } + return fmt.Sprintf("%s/%s/%s", kind, ns, name) +} + +type progressPrinter struct { + logger *zap.SugaredLogger + releaseName string + taskStore *kdutil.Concurrent[*statestore.TaskStore] + logStore *kdutil.Concurrent[*logstore.LogStore] + skipLogs bool + failedLogsOnly bool + gates *gateStatuses + skipped *skippedKeys + useColor bool + startTime time.Time + // lastEmit tracks the most recent time we printed *anything* to the + // logger (progress block or log stream). The heartbeat ticker uses it to + // suppress its own output while the printer is actively chatty; it only + // emits when there's been a silent gap of at least heartbeatInterval. + lastEmit time.Time + lastStatus map[string]string + lastCounts map[string]int + // lastLogSource is the "|" emitted at the tail of + // the previous flushLogs call. Carrying it across flushes lets us drop + // the redundant "logs " header when consecutive flushes + // continue from the same source. + lastLogSource string +} + +func newProgressPrinter( + logger *zap.SugaredLogger, + releaseName string, + taskStore *kdutil.Concurrent[*statestore.TaskStore], + logStore *kdutil.Concurrent[*logstore.LogStore], + skipLogs bool, + failedLogsOnly bool, + gates *gateStatuses, + skipped *skippedKeys, + useColor bool, +) *progressPrinter { + return &progressPrinter{ + logger: logger, + releaseName: releaseName, + taskStore: taskStore, + logStore: logStore, + skipLogs: skipLogs, + failedLogsOnly: failedLogsOnly, + gates: gates, + skipped: skipped, + useColor: useColor, + startTime: time.Now(), + lastEmit: time.Now(), + lastStatus: make(map[string]string), + lastCounts: make(map[string]int), + } +} + +func (p *progressPrinter) run(ctx context.Context, done <-chan struct{}) { + progressTicker := time.NewTicker(progressInterval) + defer progressTicker.Stop() + logsTicker := time.NewTicker(logsInterval) + defer logsTicker.Stop() + heartbeatTicker := time.NewTicker(heartbeatInterval) + defer heartbeatTicker.Stop() + + p.flushProgress() + + for { + select { + case <-progressTicker.C: + p.flushProgress() + if !p.skipLogs { + p.flushLogs() + } + case <-logsTicker.C: + if !p.skipLogs { + p.flushLogs() + } + case <-heartbeatTicker.C: + p.flushHeartbeat() + case <-done: + p.flushProgress() + if !p.skipLogs { + p.flushLogs() + } + return + case <-ctx.Done(): + return + } + } +} + +// commonNamespace returns the namespace shared by every tracked task, or +// the empty string when tasks span multiple namespaces or there are no +// tasks. When non-empty, callers strip the namespace from each row's label +// and add it once to the block title via the formatResourceLabel helper. +func (p *progressPrinter) commonNamespace() string { + var ns string + multiple := false + p.taskStore.RTransaction(func(s *statestore.TaskStore) { + for _, taskC := range s.ReadinessTasksStates() { + if multiple { + return + } + taskC.RTransaction(func(ts *statestore.ReadinessTaskState) { + taskNS := ts.Namespace() + if taskNS == "" { + return + } + if ns == "" { + ns = taskNS + } else if ns != taskNS { + multiple = true + } + }) + } + }) + if multiple { + return "" + } + return ns +} + +func (p *progressPrinter) colorize(s, code string) string { + if !p.useColor || code == "" { + return s + } + return code + s + ansiReset +} + +// statusColor maps a status line to an ANSI color code. Failing pod-phase +// substrings ("Error", "CrashLoopBackOff", …) take priority — a Job whose +// pod failed but whose overall task is marked Ready (because a sibling pod +// completed) would otherwise render the failed pod's row in green. After +// that we check the leading kubedog state, then fall back to other known +// pod phases. +// +// parentKind is the kind of the workload owning this row (empty for the +// root row of a task). It only affects how "Running" is colored: under a +// Job parent, Running means "still executing toward Completed" so it's +// yellow; everywhere else (Deployment/StatefulSet/DaemonSet) Running is +// the steady state, so green. +func (p *progressPrinter) statusColor(status, parentKind string) string { + if containsAny(status, "Error", "CrashLoopBackOff", "ImagePullBackOff", "ErrImagePull", "Failed", "OOMKilled", + "CreateContainerConfigError", "CreateContainerError", "InvalidImageName") { + return ansiRed + } + + head := status + if idx := strings.Index(status, " "); idx >= 0 { + head = status[:idx] + } + switch head { + case "ready": + return ansiGreen + case "progressing": + return ansiYellow + case "failed": + return ansiRed + case "waiting": + return ansiCyan + case "created": + return ansiCyan + case "deleted": + return ansiGray + case "unknown": + return ansiGray + } + + // Job pods in Running state haven't completed yet — treat as in-progress. + if parentKind == "Job" && containsAny(status, "Running") && !containsAny(status, "Completed") { + return ansiYellow + } + + switch { + case containsAny(status, "Completed"): + return ansiGreen + case containsAny(status, "Running"): + return ansiGreen + case containsAny(status, "ContainerCreating", "PodInitializing", "Pending", "Init:", "ContainerStarting"): + return ansiYellow + case containsAny(status, "Terminating"): + return ansiGray + } + return "" +} + +func containsAny(s string, subs ...string) bool { + for _, sub := range subs { + if strings.Contains(s, sub) { + return true + } + } + return false +} + +// describeChildStatus collapses kubedog's bookkeeping status with the +// human-readable status attribute. When the bookkeeping value is "unknown" +// (which it usually is for transient pod phases) the attribute alone is +// shown; otherwise both are joined. +// +// If the attribute is a pre-ready pod phase (Pending, ContainerCreating, +// PodInitializing, Init:*) the attribute alone is shown even when kubedog +// reports the pod as ready — the two states are contradictory and the pod +// phase is what kubectl would show, so we treat it as authoritative for +// display purposes. +func describeChildStatus(kdStatus, statusAttr string) string { + if isPreReadyPodPhase(statusAttr) { + return statusAttr + } + switch { + case statusAttr == "" && kdStatus == "": + return "unknown" + case statusAttr == "": + return kdStatus + case kdStatus == "" || kdStatus == "unknown": + return statusAttr + default: + return fmt.Sprintf("%s (%s)", kdStatus, statusAttr) + } +} + +// isPreReadyPodPhase returns true for kubelet-reported phases that indicate +// the pod cannot yet be ready — startup transitions before the Ready +// condition has had a chance to flip. Used to override kubedog's +// ResourceStatus when the two disagree, so a pod reporting ContainerCreating +// in the attribute is never displayed or counted as ready. +func isPreReadyPodPhase(phase string) bool { + switch phase { + case "Pending", "ContainerCreating", "PodInitializing", "ContainerStarting": + return true + } + // Init:0/3, Init:1/3, etc. — pod is still running init containers. + if strings.HasPrefix(phase, "Init:") { + return true + } + return false +} + +type progressRow struct { + sortKey string + label string + status string + indent string + // parentKind is the Kind of the workload the row belongs to. Set + // only for child rows; empty for root rows. Used to bias the status + // color so a Running Job pod renders yellow (still in flight) while + // a Running Deployment pod renders green (steady state). + parentKind string +} + +// podChildRow is a child resource (typically a pod) summarized for display in +// the progress block. +type podChildRow struct { + label string + status string + ready bool + hasAttr bool // has a populated AttributeNameStatus +} + +// filterStaleReadyPods drops Ready pods that no longer carry a status +// attribute. They are leftovers from a previous ReplicaSet that kubedog never +// updated past Ready: they survive in the resource graph but don't reflect the +// current spec. Only applied when a required-replica count is known. +func filterStaleReadyPods(children []podChildRow, required int) []podChildRow { + if required <= 0 { + return children + } + filtered := children[:0] + for _, c := range children { + if c.ready && !c.hasAttr { + continue + } + filtered = append(filtered, c) + } + return filtered +} + +// countReady counts Ready children, capped at required so a stray stale pod +// can never produce a "ready (2/1)" display. +func countReady(children []podChildRow, required int) int { + readyChildren := 0 + for _, c := range children { + if c.ready { + readyChildren++ + } + } + if required > 0 && readyChildren > required { + readyChildren = required + } + return readyChildren +} + +// rowsForTask builds the progress rows for a single tracked resource and its +// children. Extracted from flushProgress to keep that method's cognitive +// complexity manageable. +func (p *progressPrinter) rowsForTask(ts *statestore.ReadinessTaskState, gateSnapshot map[string]string, commonNS string) []progressRow { + kind := ts.GroupVersionKind().Kind + name := ts.Name() + ns := ts.Namespace() + rootLabel := formatResourceLabel(kind, ns, name, commonNS) + // sortKey uses the full path so rows still sort deterministically + // regardless of how the displayed label was abbreviated. + rootSortKey := fmt.Sprintf("%s/%s/%s", kind, ns, name) + rootID := kdutil.ResourceID(name, ns, ts.GroupVersionKind()) + gateKey := BaselineKey(shortKind(kind), ns, name) + + if p.skipped != nil && p.skipped.has(rootID) { + return nil + } + + if gateMsg, gated := gateSnapshot[gateKey]; gated { + return []progressRow{{ + sortKey: rootSortKey, + label: rootLabel, + status: gateMsg, + }} + } + + var children []podChildRow + var required int + var rootStatusAttr string + var rootHasFailed bool + + for _, rsC := range ts.ResourceStates() { + rsC.RTransaction(func(rs *statestore.ResourceState) { + if rs.ID() == rootID { + for _, attr := range rs.Attributes() { + switch attr.Name() { + case statestore.AttributeNameRequiredReplicas: + if a, ok := attr.(*statestore.Attribute[int]); ok { + required = a.Value + } + case statestore.AttributeNameStatus: + if a, ok := attr.(*statestore.Attribute[string]); ok { + rootStatusAttr = a.Value + } + } + } + if rs.Status() == statestore.ResourceStatusFailed { + rootHasFailed = true + } + return + } + // Skip placeholder children that dyntracker inserts when + // it has observed a reference to a resource but hasn't + // yet ingested the full object — they have no name and + // no status, so they can't tell the operator anything. + if rs.Name() == "" { + return + } + label := formatResourceLabel(rs.GroupVersionKind().Kind, rs.Namespace(), rs.Name(), commonNS) + var podStatusAttr string + for _, attr := range rs.Attributes() { + if attr.Name() == statestore.AttributeNameStatus { + if a, ok := attr.(*statestore.Attribute[string]); ok { + podStatusAttr = a.Value + } + } + } + // A pre-ready phase in the attribute overrides kubedog's + // ResourceStatus for counting purposes — without this, a + // pod whose attribute is ContainerCreating but whose + // ResourceStatus has been flipped to Ready by some other + // code path would inflate the (N/M) count even though + // the display correctly shows it as ContainerCreating. + ready := rs.Status() == statestore.ResourceStatusReady && !isPreReadyPodPhase(podStatusAttr) + children = append(children, podChildRow{ + label: label, + status: describeChildStatus(string(rs.Status()), podStatusAttr), + ready: ready, + hasAttr: podStatusAttr != "", + }) + }) + } + + children = filterStaleReadyPods(children, required) + readyChildren := countReady(children, required) + + var rootStatus string + switch { + case rootHasFailed: + rootStatus = "failed" + case required > 0: + if readyChildren >= required { + rootStatus = fmt.Sprintf("ready (%d/%d)", readyChildren, required) + } else { + rootStatus = fmt.Sprintf("progressing (%d/%d)", readyChildren, required) + } + default: + rootStatus = string(ts.Status()) + } + if rootStatusAttr != "" { + rootStatus = fmt.Sprintf("%s %s", rootStatus, rootStatusAttr) + } + + out := make([]progressRow, 0, 1+len(children)) + out = append(out, progressRow{ + sortKey: rootSortKey, + label: rootLabel, + status: rootStatus, + }) + sort.Slice(children, func(i, j int) bool { return children[i].label < children[j].label }) + for _, c := range children { + out = append(out, progressRow{ + sortKey: rootSortKey + "/" + c.label, + label: c.label, + status: c.status, + indent: " • ", + parentKind: kind, + }) + } + return out +} + +func (p *progressPrinter) flushProgress() { + gateSnapshot := map[string]string{} + if p.gates != nil { + gateSnapshot = p.gates.snapshot() + } + commonNS := p.commonNamespace() + + var rows []progressRow + p.taskStore.RTransaction(func(s *statestore.TaskStore) { + for _, taskC := range s.ReadinessTasksStates() { + taskC.RTransaction(func(ts *statestore.ReadinessTaskState) { + rows = append(rows, p.rowsForTask(ts, gateSnapshot, commonNS)...) + }) + } + }) + + sort.Slice(rows, func(i, j int) bool { return rows[i].sortKey < rows[j].sortKey }) + + current := make(map[string]string, len(rows)) + for _, r := range rows { + current[r.indent+r.label] = r.status + } + changed := len(current) != len(p.lastStatus) + if !changed { + for k, v := range current { + if p.lastStatus[k] != v { + changed = true + break + } + } + } + if !changed { + return + } + p.lastStatus = current + + if len(rows) == 0 { + return + } + + // Auto-size label column to the widest row so long pod names don't + // run into the status text. We use rune count (not byte length) because + // the child indent contains "•" (3 bytes, 1 visual column) — counting + // bytes would over-estimate child row widths and under-pad parent rows + // by the difference. + visibleWidth := func(s string) int { return utf8.RuneCountInString(s) } + maxLabelWidth := 0 + for _, r := range rows { + w := visibleWidth(r.indent) + visibleWidth(r.label) + if w > maxLabelWidth { + maxLabelWidth = w + } + } + const statusGap = 2 + + elapsed := time.Since(p.startTime).Round(time.Second) + title := fmt.Sprintf("kubedog progress (%s)", elapsed) + if p.releaseName != "" { + title = fmt.Sprintf("Release '%s' progress (%s)", p.releaseName, elapsed) + } + if commonNS != "" { + title = fmt.Sprintf("%s in '%s'", title, commonNS) + } + + var sb strings.Builder + // Leading newline visually separates each progress block from helm output + // and from any log lines that came right before. It may be collapsed in + // some CI log viewers — the "==" decoration in the header survives. + sb.WriteString("\n") + sb.WriteString(HeaderDividerStyled(title, p.useColor)) + for _, r := range rows { + labelVisual := visibleWidth(r.label) + // Target column for the status = maxLabelWidth + statusGap + + // indentVisual. After subtracting the row's actual prefix width + // (indentVisual + labelVisual), indentVisual cancels and we're left + // with padding = maxLabelWidth - labelVisual + statusGap. This nests + // child statuses further right than their parent's by exactly the + // child indent width. + pad := maxLabelWidth - labelVisual + statusGap + padding := strings.Repeat(" ", pad) + fmt.Fprintf(&sb, "\n %s%s%s%s", + r.indent, + r.label, + padding, + p.colorize(r.status, p.statusColor(r.status, r.parentKind)), + ) + } + p.logger.Info(sb.String()) + p.lastEmit = time.Now() + // A progress block visually breaks any in-flight log stream, so the next + // flushLogs call should re-emit the "logs :" header even + // if it continues from the same source. + p.lastLogSource = "" +} + +func (p *progressPrinter) flushLogs() { + // failedPodIDs holds the kdutil.ResourceID of every pod currently in a + // failed state. Populated only in failed-only mode; otherwise we don't + // need it. + var failedPodIDs map[string]struct{} + if p.failedLogsOnly { + failedPodIDs = p.collectFailedPodIDs() + } + + commonNS := p.commonNamespace() + + type entry struct { + // resourceKey is the full Kind/ns/name path used for cursor + // uniqueness across flushes. displayLabel is the ns-stripped form + // rendered in the per-source header. + resourceKey string + displayLabel string + source string + line string + ts time.Time + } + var pending []entry + + p.logStore.RTransaction(func(s *logstore.LogStore) { + for _, rlC := range s.ResourcesLogs() { + rlC.RTransaction(func(rl *logstore.ResourceLogs) { + resourceKey := fmt.Sprintf("%s/%s/%s", rl.GroupVersionKind().Kind, rl.Namespace(), rl.Name()) + displayLabel := formatResourceLabel(rl.GroupVersionKind().Kind, rl.Namespace(), rl.Name(), commonNS) + // In failed-only mode, gate emission on the pod's current + // status. We keep the cursor frozen for non-failed pods so + // their accumulated lines stay queued and get printed in full + // if/when the pod transitions to failed later. + if p.failedLogsOnly { + podID := kdutil.ResourceID(rl.Name(), rl.Namespace(), rl.GroupVersionKind()) + if _, failed := failedPodIDs[podID]; !failed { + return + } + } + for source, lines := range rl.LogLines() { + cursorKey := resourceKey + "|" + source + start := p.lastCounts[cursorKey] + if start >= len(lines) { + continue + } + for _, ll := range lines[start:] { + pending = append(pending, entry{ + resourceKey: resourceKey, + displayLabel: displayLabel, + source: source, + line: ll.Line, + ts: ll.Time, + }) + } + p.lastCounts[cursorKey] = len(lines) + } + }) + } + }) + + sort.Slice(pending, func(i, j int) bool { return pending[i].ts.Before(pending[j].ts) }) + + if len(pending) == 0 { + return + } + + // Group consecutive lines from the same pod/container into a single + // emission so the per-source header is only printed once and the lines + // look like an excerpt rather than scattered noise. + var sb strings.Builder + prevKey := p.lastLogSource + startedWithHeader := false + lastEmitted := "" + for _, e := range pending { + line := strings.TrimRight(e.line, "\n") + if line == "" { + continue + } + key := e.resourceKey + "|" + e.source + if key != prevKey { + // On a source change we emit a "Logs :" header + // (with "==" decoration for CI log readability) preceded by a + // blank line so it visually detaches from whatever came before. + // Continuation flushes (same source) skip both the blank line AND + // the header to avoid stacking newlines from zap. + if sb.Len() > 0 { + sb.WriteString("\n\n") + } else { + sb.WriteString("\n") + startedWithHeader = true + } + sb.WriteString(HeaderDividerStyled(fmt.Sprintf("Logs %s %s", e.displayLabel, e.source), p.useColor)) + prevKey = key + } + // Indent each line; the first line of a continuation flush has no + // leading newline so zap's own message-boundary newline alone + // separates it from the previous flush. + if startedWithHeader || sb.Len() > 0 { + sb.WriteString("\n ") + } else { + sb.WriteString(" ") + } + sb.WriteString(line) + lastEmitted = key + } + if sb.Len() > 0 { + p.logger.Info(sb.String()) + p.lastEmit = time.Now() + } + if lastEmitted != "" { + p.lastLogSource = lastEmitted + } +} + +// flushHeartbeat emits a single-line "still tracking" digest when nothing +// else has been printed for at least heartbeatInterval. It's the only signal +// CI operators get during long silent stretches (e.g. an hour-long Job that +// never changes state and never logs) that helmfile is alive rather than +// hung. Suppressed when (a) the printer recently produced output already, or +// (b) every tracked task is Ready — in which case there's no useful content. +func (p *progressPrinter) flushHeartbeat() { + // Suppress only if some *change-driven* output (progress block or log + // stream) happened recently. We deliberately do NOT update p.lastEmit + // from this function: two consecutive heartbeats during a silent stretch + // are exactly what the operator needs. Updating lastEmit on every + // heartbeat caused the next tick (which fires on a fixed wall-clock + // schedule) to land microseconds short of heartbeatInterval after our + // processing delay, so it got suppressed — observed effect was a tick + // emitted every 4 minutes instead of 2. + if time.Since(p.lastEmit) < heartbeatInterval { + return + } + + gateSnapshot := map[string]string{} + if p.gates != nil { + gateSnapshot = p.gates.snapshot() + } + commonNS := p.commonNamespace() + + var progressing, waiting []string + p.taskStore.RTransaction(func(s *statestore.TaskStore) { + for _, taskC := range s.ReadinessTasksStates() { + taskC.RTransaction(func(ts *statestore.ReadinessTaskState) { + if ts.Status() == statestore.ReadinessTaskStatusReady { + return + } + if p.skipped != nil && p.skipped.has(kdutil.ResourceID(ts.Name(), ts.Namespace(), ts.GroupVersionKind())) { + return + } + kind := ts.GroupVersionKind().Kind + name := ts.Name() + ns := ts.Namespace() + label := formatResourceLabel(kind, ns, name, commonNS) + if _, gated := gateSnapshot[BaselineKey(shortKind(kind), ns, name)]; gated { + waiting = append(waiting, label) + } else { + progressing = append(progressing, label) + } + }) + } + }) + + if len(progressing) == 0 && len(waiting) == 0 { + return + } + + sort.Strings(progressing) + sort.Strings(waiting) + + elapsed := time.Since(p.startTime).Round(time.Second) + timestamp := time.Now().Format("15:04:05") + label := "kubedog" + if p.releaseName != "" { + label = fmt.Sprintf("Release '%s'", p.releaseName) + } + if commonNS != "" { + label = fmt.Sprintf("%s in '%s'", label, commonNS) + } + + // Render the progressing group with its names (capped); the waiting group + // is summarized as a count only because listing every queued resource + // makes the line too long for one CI log row. Both groups carry their own + // count so the split is obvious at a glance. + var parts []string + if len(progressing) > 0 { + const maxShown = 3 + names := progressing + overflow := "" + if len(names) > maxShown { + overflow = fmt.Sprintf(", +%d", len(names)-maxShown) + names = names[:maxShown] + } + parts = append(parts, fmt.Sprintf("%d progressing (%s%s)", + len(progressing), strings.Join(names, ", "), overflow)) + } + if len(waiting) > 0 { + parts = append(parts, fmt.Sprintf("%d waiting", len(waiting))) + } + + line := fmt.Sprintf("[%s] %s still tracking (%s) — %s", + timestamp, label, elapsed, strings.Join(parts, ", ")) + p.logger.Info(p.colorize(line, ansiCyan)) +} + +// collectFailedPodIDs walks the task store and returns the set of pod +// resource IDs that are in a failed-ish state — either marked as +// ResourceStatusFailed, carrying recorded errors, or whose status attribute +// matches a known pod-phase failure (CrashLoopBackOff, ImagePullBackOff, +// Error, etc.). Used by failed-only log mode to gate emission. +func (p *progressPrinter) collectFailedPodIDs() map[string]struct{} { + out := map[string]struct{}{} + p.taskStore.RTransaction(func(s *statestore.TaskStore) { + for _, taskC := range s.ReadinessTasksStates() { + taskC.RTransaction(func(ts *statestore.ReadinessTaskState) { + for _, rsC := range ts.ResourceStates() { + rsC.RTransaction(func(rs *statestore.ResourceState) { + // We only filter pod logs, so only collect Pod IDs. + if rs.GroupVersionKind().Kind != "Pod" { + return + } + if rs.Status() == statestore.ResourceStatusFailed { + out[rs.ID()] = struct{}{} + return + } + if len(rs.Errors()) > 0 { + out[rs.ID()] = struct{}{} + return + } + for _, attr := range rs.Attributes() { + if attr.Name() != statestore.AttributeNameStatus { + continue + } + a, ok := attr.(*statestore.Attribute[string]) + if !ok { + continue + } + if isFailingPodPhase(a.Value) { + out[rs.ID()] = struct{}{} + } + } + }) + } + }) + } + }) + return out +} + +// isFailingPodPhase returns true for kubelet-reported pod phases that +// indicate the pod is in a broken state, even if kubedog hasn't yet flipped +// the underlying ResourceStatus to Failed. +func isFailingPodPhase(phase string) bool { + switch phase { + case "Error", "Failed", + "CrashLoopBackOff", + "ImagePullBackOff", "ErrImagePull", + "OOMKilled", + "CreateContainerConfigError", "CreateContainerError", + "InvalidImageName": + return true + } + return false +} + +// shortKind maps full Kind names (as reported by the task store) back to the +// short identifiers used as the BaselineKey prefix. +func shortKind(kind string) string { + switch kind { + case "Deployment": + return "deploy" + case "StatefulSet": + return "sts" + case "DaemonSet": + return "ds" + case "Job": + return "job" + case "Canary": + return "canary" + case "PersistentVolumeClaim": + return "pvc" + } + return strings.ToLower(kind) +} diff --git a/pkg/kubedog/printer_test.go b/pkg/kubedog/printer_test.go new file mode 100644 index 00000000..30dad0c5 --- /dev/null +++ b/pkg/kubedog/printer_test.go @@ -0,0 +1,863 @@ +package kubedog + +import ( + "bytes" + "context" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/werf/kubedog/pkg/trackers/dyntracker/logstore" + "github.com/werf/kubedog/pkg/trackers/dyntracker/statestore" + kdutil "github.com/werf/kubedog/pkg/trackers/dyntracker/util" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// newBufferedLogger returns a zap.SugaredLogger whose Info-level writes are +// captured into a buffer for inspection in tests. +func newBufferedLogger(t *testing.T) (*zap.SugaredLogger, *bytes.Buffer, *sync.Mutex) { + t.Helper() + buf := &bytes.Buffer{} + mu := &sync.Mutex{} + w := &syncedBuffer{buf: buf, mu: mu} + + var cfg zapcore.EncoderConfig + cfg.MessageKey = "message" + core := zapcore.NewCore(zapcore.NewConsoleEncoder(cfg), zapcore.AddSync(w), zapcore.InfoLevel) + return zap.New(core).Sugar(), buf, mu +} + +type syncedBuffer struct { + buf *bytes.Buffer + mu *sync.Mutex +} + +func (s *syncedBuffer) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.buf.Write(p) +} + +func capturedOutput(buf *bytes.Buffer, mu *sync.Mutex) string { + mu.Lock() + defer mu.Unlock() + return buf.String() +} + +var ( + deploymentGVK = schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"} + podGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + jobGVK = schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"} +) + +// addPodChild attaches a pod resource state under a task and sets its status +// + optional pod-phase attribute. Returns the wrapped ResourceState. +func addPodChild(t *testing.T, ts *statestore.ReadinessTaskState, podName, namespace string, + status statestore.ResourceStatus, statusAttr string) { + t.Helper() + ts.AddResourceState(podName, namespace, podGVK) + ts.AddDependency(ts.Name(), ts.Namespace(), ts.GroupVersionKind(), podName, namespace, podGVK) + rs := ts.ResourceState(podName, namespace, podGVK) + rs.RWTransaction(func(rs *statestore.ResourceState) { + rs.SetStatus(status) + if statusAttr != "" { + rs.AddAttribute(statestore.NewAttribute(statestore.AttributeNameStatus, statusAttr)) + } + }) +} + +// setRequiredReplicas writes the AttributeNameRequiredReplicas on the root +// resource state of a task — what kubedog's deployment handler populates from +// Status.Replicas in real runs. +func setRequiredReplicas(t *testing.T, ts *statestore.ReadinessTaskState, count int) { + t.Helper() + root := ts.ResourceState(ts.Name(), ts.Namespace(), ts.GroupVersionKind()) + root.RWTransaction(func(rs *statestore.ResourceState) { + rs.AddAttribute(statestore.NewAttribute(statestore.AttributeNameRequiredReplicas, count)) + }) +} + +func TestStatusColor_FailingPhaseOverridesReady(t *testing.T) { + p := &progressPrinter{useColor: true} + + // kubedog reports the pod's ResourceStatus as "ready" because the parent + // task is ready, but the pod's actual phase is "Error" — must render red. + assert.Equal(t, ansiRed, p.statusColor("ready (Error)", "")) + assert.Equal(t, ansiRed, p.statusColor("progressing (CrashLoopBackOff)", "")) + assert.Equal(t, ansiRed, p.statusColor("ready (ImagePullBackOff)", "")) + assert.Equal(t, ansiRed, p.statusColor("ready (OOMKilled)", "")) +} + +func TestStatusColor_LeadingState(t *testing.T) { + p := &progressPrinter{useColor: true} + assert.Equal(t, ansiGreen, p.statusColor("ready (1/1)", "")) + assert.Equal(t, ansiGreen, p.statusColor("ready (Running)", "")) + assert.Equal(t, ansiYellow, p.statusColor("progressing (0/1)", "")) + assert.Equal(t, ansiRed, p.statusColor("failed", "")) + assert.Equal(t, ansiCyan, p.statusColor("waiting for update (uid=abc gen=1)", "")) + assert.Equal(t, ansiGray, p.statusColor("unknown", "")) +} + +func TestStatusColor_PodPhaseFallback(t *testing.T) { + p := &progressPrinter{useColor: true} + // Bare pod phases (no leading kubedog state word), no parent context — + // defaults match the long-running workload assumptions (Running = green). + assert.Equal(t, ansiGreen, p.statusColor("Running", "")) + assert.Equal(t, ansiGreen, p.statusColor("Completed", "")) + assert.Equal(t, ansiYellow, p.statusColor("ContainerCreating", "")) + assert.Equal(t, ansiYellow, p.statusColor("Pending", "")) + assert.Equal(t, ansiYellow, p.statusColor("Init:0/2", "")) + assert.Equal(t, ansiGray, p.statusColor("Terminating", "")) +} + +func TestStatusColor_JobChildRunningIsYellow(t *testing.T) { + p := &progressPrinter{useColor: true} + // A Job pod in Running phase is still working toward Completed, so the + // row should render in-progress yellow instead of steady-state green. + assert.Equal(t, ansiYellow, p.statusColor("Running", "Job")) + // Once the Job pod hits Completed it's done — green. + assert.Equal(t, ansiGreen, p.statusColor("ready (Completed)", "Job")) + assert.Equal(t, ansiGreen, p.statusColor("Completed", "Job")) + // Other workload kinds keep the steady-state semantics for Running. + assert.Equal(t, ansiGreen, p.statusColor("Running", "Deployment")) + assert.Equal(t, ansiGreen, p.statusColor("Running", "StatefulSet")) +} + +func TestColorize_RespectsUseColor(t *testing.T) { + with := &progressPrinter{useColor: true} + assert.Equal(t, ansiGreen+"ready"+ansiReset, with.colorize("ready", ansiGreen)) + + without := &progressPrinter{useColor: false} + assert.Equal(t, "ready", without.colorize("ready", ansiGreen), + "colorize must be a no-op when useColor is false") +} + +func TestDescribeChildStatus(t *testing.T) { + tests := []struct { + name string + kd, attr string + want string + }{ + {"both empty", "", "", "unknown"}, + {"attr only", "", "Running", "Running"}, + {"kd only", "ready", "", "ready"}, + {"unknown plus attr drops unknown", "unknown", "ContainerCreating", "ContainerCreating"}, + {"both meaningful", "ready", "Running", "ready (Running)"}, + {"failed plus error", "failed", "CrashLoopBackOff", "failed (CrashLoopBackOff)"}, + // Pre-ready phases override kubedog's ResourceStatus — kubedog's + // internal flag can briefly disagree with the kubelet-visible phase + // and we surface the phase as the source of truth for display. + {"ready vs ContainerCreating shows phase", "ready", "ContainerCreating", "ContainerCreating"}, + {"ready vs Pending shows phase", "ready", "Pending", "Pending"}, + {"ready vs PodInitializing shows phase", "ready", "PodInitializing", "PodInitializing"}, + {"ready vs Init:1/3 shows phase", "ready", "Init:1/3", "Init:1/3"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, describeChildStatus(tc.kd, tc.attr)) + }) + } +} + +func TestIsPreReadyPodPhase(t *testing.T) { + for _, phase := range []string{"Pending", "ContainerCreating", "PodInitializing", "ContainerStarting", "Init:0/3", "Init:2/5"} { + assert.True(t, isPreReadyPodPhase(phase), "%q should be a pre-ready phase", phase) + } + for _, phase := range []string{"Running", "Completed", "Terminating", "Error", "", "CrashLoopBackOff"} { + assert.False(t, isPreReadyPodPhase(phase), "%q should not be a pre-ready phase", phase) + } +} + +func TestIsFailingPodPhase(t *testing.T) { + for _, p := range []string{"Error", "Failed", "CrashLoopBackOff", "ImagePullBackOff", + "ErrImagePull", "OOMKilled", "CreateContainerConfigError", "CreateContainerError", + "InvalidImageName"} { + assert.True(t, isFailingPodPhase(p), "%q should be classified as failing", p) + } + for _, p := range []string{"Running", "Completed", "ContainerCreating", "Pending", + "PodInitializing", "Terminating", ""} { + assert.False(t, isFailingPodPhase(p), "%q should not be classified as failing", p) + } +} + +func TestShortKind(t *testing.T) { + assert.Equal(t, "deploy", shortKind("Deployment")) + assert.Equal(t, "sts", shortKind("StatefulSet")) + assert.Equal(t, "ds", shortKind("DaemonSet")) + assert.Equal(t, "job", shortKind("Job")) + assert.Equal(t, "canary", shortKind("Canary")) + assert.Equal(t, "pvc", shortKind("PersistentVolumeClaim")) + // Anything else lowercases for safety. + assert.Equal(t, "pod", shortKind("Pod")) +} + +func TestFlushProgress_HidesStaleReadyPodWithoutAttr(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + ts := statestore.NewReadinessTaskState("app", "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, ts, 1) + // New pod that kubedog has observed (status attr set). + addPodChild(t, ts, "app-new-xyz", "ns", statestore.ResourceStatusReady, "Running") + // Stale leftover from previous ReplicaSet: ready but no Status attribute. + addPodChild(t, ts, "app-old-abc", "ns", statestore.ResourceStatusReady, "") + + taskC := kdutil.NewConcurrent(ts) + taskStore.RWTransaction(func(s *statestore.TaskStore) { s.AddReadinessTaskState(taskC) }) + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "", taskStore, logStore, true /*skipLogs*/, false, newGateStatuses(), newSkippedKeys(), false) + p.flushProgress() + + out := capturedOutput(buf, mu) + assert.Contains(t, out, "Deployment/app") + assert.Contains(t, out, "ready (1/1)", "ready count must be capped to required replicas") + assert.Contains(t, out, "Pod/app-new-xyz") + assert.NotContains(t, out, "app-old-abc", "stale ready pod without status attribute must be hidden") +} + +func TestFlushProgress_PreReadyPhaseAttributeOverridesReady(t *testing.T) { + // Recreate the dyntracker discrepancy the operator hit: kubedog's + // ResourceStatus is Ready, but the status attribute still says + // ContainerCreating from an earlier event. We surface the phase as the + // truth — the row reads "ContainerCreating" cleanly, and the parent + // task's count doesn't inflate to (1/1) for a pod that hasn't actually + // reached ready yet. + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + ts := statestore.NewReadinessTaskState("system-manager", "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, ts, 1) + addPodChild(t, ts, "system-manager-pod", "ns", statestore.ResourceStatusReady, "ContainerCreating") + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "", taskStore, logStore, true, false, newGateStatuses(), newSkippedKeys(), false) + p.flushProgress() + + out := capturedOutput(buf, mu) + assert.Contains(t, out, "Pod/system-manager-pod") + assert.Contains(t, out, "ContainerCreating", + "pod row must show the kubelet-visible phase") + assert.NotContains(t, out, "ready (ContainerCreating)", + "row must not show the contradictory ready+ContainerCreating combination") + assert.Contains(t, out, "progressing (0/1)", + "parent count must reflect that the pod isn't yet ready despite kubedog's ResourceStatus") + assert.NotContains(t, out, "ready (1/1)", + "parent must not claim ready while the pod is still ContainerCreating") +} + +func TestFlushProgress_KeepsNamespacesWhenReleaseSpansMultiple(t *testing.T) { + // A release that creates workloads in two namespaces must keep the full + // Kind/ns/name on every row — there's no single namespace to elide. + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + a := statestore.NewReadinessTaskState("app-a", "ns-one", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + b := statestore.NewReadinessTaskState("app-b", "ns-two", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(a)) + s.AddReadinessTaskState(kdutil.NewConcurrent(b)) + }) + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "vray", taskStore, logStore, true, false, newGateStatuses(), newSkippedKeys(), false) + p.flushProgress() + out := capturedOutput(buf, mu) + + assert.Contains(t, out, "Deployment/ns-one/app-a") + assert.Contains(t, out, "Deployment/ns-two/app-b") + assert.NotContains(t, out, "in 'ns-one'", + "multi-namespace release must not pick one namespace for the header") + assert.NotContains(t, out, "in 'ns-two'", "header must not pick a single namespace") +} + +func TestFlushProgress_HidesEmptyNameChildPlaceholder(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + ts := statestore.NewReadinessTaskState("init", "ns", jobGVK, statestore.ReadinessTaskStateOptions{}) + addPodChild(t, ts, "real-pod", "ns", statestore.ResourceStatusReady, "Running") + // Placeholder child that dyntracker can insert when it has observed a + // reference to a pod but hasn't ingested the object yet — empty name, + // no status, no attributes. It must not be rendered. + addPodChild(t, ts, "", "ns", statestore.ResourceStatusUnknown, "") + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "", taskStore, logStore, true, false, newGateStatuses(), newSkippedKeys(), false) + p.flushProgress() + + out := capturedOutput(buf, mu) + assert.Contains(t, out, "Pod/real-pod") + // "Pod/ns/" with no name suffix is the visible footprint of the + // placeholder. It must not appear in the rendered output. + assert.NotContains(t, out, "Pod/ ", + "empty-name placeholder child must not be rendered") + assert.NotContains(t, out, "Pod/\n", + "empty-name placeholder child must not be rendered as a trailing row") +} + +func TestFlushProgress_HeaderUsesReleaseName(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + ts := statestore.NewReadinessTaskState("app", "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "vray", taskStore, logStore, true, false, newGateStatuses(), newSkippedKeys(), false) + p.flushProgress() + out := capturedOutput(buf, mu) + assert.Contains(t, out, "Release 'vray' progress (", + "header must use release name when set and include elapsed time") + assert.Contains(t, out, "==========", + "header must include divider that survives CI newline stripping") + assert.NotContains(t, out, "Release 'vray' progress:", + "header must not carry a trailing colon") +} + +func TestFlushProgress_HeaderFallsBackWithoutReleaseName(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + ts := statestore.NewReadinessTaskState("app", "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "", taskStore, logStore, true, false, newGateStatuses(), newSkippedKeys(), false) + p.flushProgress() + out := capturedOutput(buf, mu) + assert.Contains(t, out, "kubedog progress (") + assert.NotContains(t, out, "Release ''", "empty release name must not produce 'Release '' progress'") + assert.NotContains(t, out, "kubedog progress:", "header must not carry a trailing colon") +} + +func TestFlushProgress_NestingAndAlignment(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + ts := statestore.NewReadinessTaskState("svc", "default", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, ts, 2) + addPodChild(t, ts, "svc-pod-aaa", "default", statestore.ResourceStatusReady, "Running") + addPodChild(t, ts, "svc-pod-bbb", "default", statestore.ResourceStatusUnknown, "ContainerCreating") + + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "", taskStore, logStore, true, false, newGateStatuses(), newSkippedKeys(), false) + p.flushProgress() + + out := capturedOutput(buf, mu) + require.Contains(t, out, "Deployment/svc") + require.Contains(t, out, "in 'default'", + "single-namespace release must surface the namespace in the header") + require.Contains(t, out, " • Pod/svc-pod-aaa", "pod rows must be indented with a bullet") + require.Contains(t, out, "Running") + require.Contains(t, out, "ContainerCreating") + // readyChildren counts only Ready pods; one of two is ready. + require.Contains(t, out, "progressing (1/2)") +} + +func TestFlushProgress_RespectsGateAndSkip(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + gated := statestore.NewReadinessTaskState("gated", "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + skipped := statestore.NewReadinessTaskState("skipped", "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + visible := statestore.NewReadinessTaskState("visible", "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, visible, 1) + addPodChild(t, visible, "visible-aaa", "ns", statestore.ResourceStatusReady, "Running") + + for _, ts := range []*statestore.ReadinessTaskState{gated, skipped, visible} { + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + } + + gates := newGateStatuses() + gates.set(BaselineKey("deploy", "ns", "gated"), "waiting for update (uid=abc gen=2)") + skips := newSkippedKeys() + skips.add(kdutil.ResourceID("skipped", "ns", deploymentGVK)) + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "", taskStore, logStore, true, false, gates, skips, false) + p.flushProgress() + + out := capturedOutput(buf, mu) + assert.Contains(t, out, "Deployment/gated") + assert.Contains(t, out, "waiting for update (uid=abc gen=2)") + assert.NotContains(t, out, "Deployment/skipped", "skipped tasks must be omitted") + assert.Contains(t, out, "Deployment/visible") + assert.Contains(t, out, "ready (1/1)") +} + +func TestFlushLogs_FailedOnlyMode_GatesUnchanged(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + // Job with two pods: one Completed (success), one Error (failed). + ts := statestore.NewReadinessTaskState("init", "ns", jobGVK, statestore.ReadinessTaskStateOptions{}) + addPodChild(t, ts, "init-good", "ns", statestore.ResourceStatusReady, "Completed") + addPodChild(t, ts, "init-bad", "ns", statestore.ResourceStatusReady, "Error") + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + // Both pods have log lines accumulated in the log store. + addPodLogs(t, logStore, "init-good", "good line 1", "good line 2") + addPodLogs(t, logStore, "init-bad", "bad line 1", "panic: boom") + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "", taskStore, logStore, false /*skipLogs*/, true /*failedLogsOnly*/, newGateStatuses(), newSkippedKeys(), false) + p.flushLogs() + + out := capturedOutput(buf, mu) + assert.NotContains(t, out, "good line", "successful-pod logs must be hidden in failed-only mode") + assert.Contains(t, out, "bad line 1") + assert.Contains(t, out, "panic: boom") + assert.Contains(t, out, "Pod/init-bad") +} + +func TestFlushLogs_FailedOnlyMode_DoesNotAdvanceCursorForSuccess(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + // Pod is currently classified as successful (Completed) — its lines must + // stay queued. When the pod transitions to a failing phase, all lines + // since the start should be emitted. + ts := statestore.NewReadinessTaskState("flaky", "ns", jobGVK, statestore.ReadinessTaskStateOptions{}) + addPodChild(t, ts, "flaky-pod", "ns", statestore.ResourceStatusReady, "Completed") + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + addPodLogs(t, logStore, "flaky-pod", "early line A", "early line B") + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "", taskStore, logStore, false, true, newGateStatuses(), newSkippedKeys(), false) + + p.flushLogs() + assert.NotContains(t, capturedOutput(buf, mu), "early line", + "flushLogs must skip successful pods in failed-only mode") + + // Pod transitions to failing — re-flush should now drain the buffered lines. + flipPodPhase(t, ts, "flaky-pod", "ns", "CrashLoopBackOff") + p.flushLogs() + + out := capturedOutput(buf, mu) + assert.Contains(t, out, "early line A", "lines accumulated before failure must be emitted on flip") + assert.Contains(t, out, "early line B") +} + +func TestFlushLogs_AllPodsMode_EmitsEverything(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + ts := statestore.NewReadinessTaskState("init", "ns", jobGVK, statestore.ReadinessTaskStateOptions{}) + addPodChild(t, ts, "init-good", "ns", statestore.ResourceStatusReady, "Completed") + addPodChild(t, ts, "init-bad", "ns", statestore.ResourceStatusReady, "Error") + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + addPodLogs(t, logStore, "init-good", "good line 1") + addPodLogs(t, logStore, "init-bad", "bad line 1") + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "", taskStore, logStore, false, false /*failedLogsOnly=off*/, newGateStatuses(), newSkippedKeys(), false) + p.flushLogs() + + out := capturedOutput(buf, mu) + assert.Contains(t, out, "good line 1") + assert.Contains(t, out, "bad line 1") +} + +func TestFlushLogs_HeaderDedupedAcrossFlushes(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + ts := statestore.NewReadinessTaskState("app", "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + addPodChild(t, ts, "app-pod", "ns", statestore.ResourceStatusReady, "Running") + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "", taskStore, logStore, false, false, newGateStatuses(), newSkippedKeys(), false) + + addPodLogs(t, logStore, "app-pod", "line 1", "line 2") + p.flushLogs() + addPodLogs(t, logStore, "app-pod", "line 3", "line 4") + p.flushLogs() + + out := capturedOutput(buf, mu) + // The header should appear exactly once — second flush is a continuation + // of the same source. + assert.Equal(t, 1, strings.Count(out, "Logs Pod/app-pod container/main"), + "continuation flushes must not re-emit the per-source header") + assert.NotContains(t, out, "Logs Pod/app-pod container/main:", + "log header must not carry a trailing colon") + assert.Contains(t, out, "line 1") + assert.Contains(t, out, "line 4") +} + +func TestHeaderDivider_FormatsBothBorders(t *testing.T) { + assert.Equal(t, "========== title: ==========", HeaderDivider("title:")) +} + +func TestStyleWarning(t *testing.T) { + // No-op when useColor is false so non-TTY consumers see clean bytes. + assert.Equal(t, "watch out", StyleWarning("watch out", false)) + + // With color, wrap in bold+yellow and reset. Match prefix/suffix exactly + // so accidental style drift surfaces as a test failure. + styled := StyleWarning("watch out", true) + assert.True(t, strings.HasPrefix(styled, ansiBold+ansiYellow), + "styled warning must start with bold+yellow, got %q", styled) + assert.True(t, strings.HasSuffix(styled, ansiReset), + "styled warning must end with reset, got %q", styled) + assert.Contains(t, styled, "watch out") +} + +func TestHeaderDividerStyled(t *testing.T) { + // Without color the styled helper must be byte-identical to HeaderDivider — + // CI logs and non-TTY consumers should see no ANSI noise. + assert.Equal(t, HeaderDivider("title:"), HeaderDividerStyled("title:", false)) + + // With color enabled, the divider is wrapped in bold + blue and ends with + // a single reset. We match the exact prefix/suffix so future drift in the + // style is a deliberate change. + styled := HeaderDividerStyled("title:", true) + assert.True(t, strings.HasPrefix(styled, ansiBold+ansiBlue), + "styled header must start with bold+blue, got %q", styled) + assert.True(t, strings.HasSuffix(styled, ansiReset), + "styled header must end with reset, got %q", styled) + assert.Contains(t, styled, HeaderDivider("title:"), + "styled header must contain the plain divider unchanged") +} + +func TestFlushHeartbeat_EmitsWhenIdleWithInFlight(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + // One progressing Deployment with a not-yet-ready pod — the heartbeat + // should pick this up as a single in-flight task. + ts := statestore.NewReadinessTaskState("app", "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, ts, 1) + addPodChild(t, ts, "app-pending", "ns", statestore.ResourceStatusUnknown, "ContainerCreating") + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "vray", taskStore, logStore, true, false, newGateStatuses(), newSkippedKeys(), false) + // Simulate the silent gap: pretend we last printed long enough ago that + // the heartbeat ticker would now fire. + p.lastEmit = time.Now().Add(-3 * heartbeatInterval) + + p.flushHeartbeat() + + out := capturedOutput(buf, mu) + assert.Contains(t, out, "Release 'vray' in 'ns' still tracking", + "heartbeat header must include release name + namespace when all tasks share one") + assert.Contains(t, out, "1 progressing (Deployment/app)", + "heartbeat must report the count and name of progressing resources") + assert.Regexp(t, `\[\d\d:\d\d:\d\d\]`, out, + "heartbeat must be prefixed with a wall-clock timestamp") +} + +func TestFlushHeartbeat_LabelsGatedTasksAsWaiting(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + // One task that is mid-rollout, and one that's still gated by the + // freshness gate (helm hasn't gotten to it yet) — the heartbeat must + // distinguish them so operators know what's running vs what's queued. + progressing := statestore.NewReadinessTaskState("running-job", "ns", jobGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, progressing, 1) + addPodChild(t, progressing, "running-job-pod", "ns", statestore.ResourceStatusUnknown, "Running") + + queued := statestore.NewReadinessTaskState("queued-job", "ns", jobGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, queued, 1) + + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(progressing)) + s.AddReadinessTaskState(kdutil.NewConcurrent(queued)) + }) + + gates := newGateStatuses() + gates.set(BaselineKey("job", "ns", "queued-job"), "waiting for update (uid=abc gen=1)") + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "vray", taskStore, logStore, true, false, gates, newSkippedKeys(), false) + p.lastEmit = time.Now().Add(-3 * heartbeatInterval) + + p.flushHeartbeat() + + out := capturedOutput(buf, mu) + assert.Contains(t, out, "1 progressing (Job/running-job)", + "progressing task must be listed by name with the progressing count") + assert.Contains(t, out, "1 waiting", + "freshness-gated task must appear as a waiting count") + assert.NotContains(t, out, "queued-job", + "waiting tasks must be summarized by count, not listed by name (keeps the line one-row in CI)") +} + +func TestFlushHeartbeat_ProgressingItemsListedBeforeWaiting(t *testing.T) { + // Two waiting items with names that sort before the progressing one — + // without the gated-aware sort, the alphabetical order would push the + // progressing item past the cap or just bury it. We care most about + // what's actually running, so progressing must always render first. + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + for _, name := range []string{"aaa-queued", "bbb-queued"} { + ts := statestore.NewReadinessTaskState(name, "ns", jobGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, ts, 1) + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + } + running := statestore.NewReadinessTaskState("zzz-running", "ns", jobGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, running, 1) + addPodChild(t, running, "zzz-running-pod", "ns", statestore.ResourceStatusUnknown, "Running") + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(running)) + }) + + gates := newGateStatuses() + gates.set(BaselineKey("job", "ns", "aaa-queued"), "waiting for update (uid=a gen=1)") + gates.set(BaselineKey("job", "ns", "bbb-queued"), "waiting for update (uid=b gen=1)") + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "vray", taskStore, logStore, true, false, gates, newSkippedKeys(), false) + p.lastEmit = time.Now().Add(-3 * heartbeatInterval) + + p.flushHeartbeat() + + out := capturedOutput(buf, mu) + runningIdx := strings.Index(out, "1 progressing (Job/zzz-running)") + waitingIdx := strings.Index(out, "2 waiting") + require.Positive(t, runningIdx, "progressing group must appear") + require.Positive(t, waitingIdx, "waiting group must appear") + assert.Less(t, runningIdx, waitingIdx, + "progressing group must precede the waiting count so the head of the line surfaces what's actually running") + assert.NotContains(t, out, "aaa-queued", "waiting items must not be listed by name") + assert.NotContains(t, out, "bbb-queued", "waiting items must not be listed by name") +} + +func TestFlushHeartbeat_AllWaitingShowsCountOnly(t *testing.T) { + // Early in a rollout every task can be gated (helm hasn't gotten there + // yet). The heartbeat must still emit so operators see "yes, we're + // alive, just queued up" — but with only the waiting count, no names. + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + gates := newGateStatuses() + for _, name := range []string{"q1", "q2", "q3"} { + ts := statestore.NewReadinessTaskState(name, "ns", jobGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, ts, 1) + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + gates.set(BaselineKey("job", "ns", name), "waiting for update") + } + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "vray", taskStore, logStore, true, false, gates, newSkippedKeys(), false) + p.lastEmit = time.Now().Add(-3 * heartbeatInterval) + + p.flushHeartbeat() + out := capturedOutput(buf, mu) + + assert.Contains(t, out, "3 waiting") + assert.NotContains(t, out, "progressing", + "if nothing is progressing the line must not show a zero-count progressing group") +} + +func TestFlushHeartbeat_DoesNotUpdateLastEmit(t *testing.T) { + // Regression: when flushHeartbeat updated p.lastEmit on emit, the next + // scheduled ticker tick (which fires on a fixed wall-clock interval from + // ticker creation) landed microseconds short of heartbeatInterval after + // our processing delay — the time.Since check then suppressed every + // other tick, producing an effective 4-minute cadence instead of 2. + // Subsequent heartbeats during a silent stretch must rely on p.lastEmit + // staying pinned to the previous change-driven flush. + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + ts := statestore.NewReadinessTaskState("app", "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, ts, 1) + addPodChild(t, ts, "app-pending", "ns", statestore.ResourceStatusUnknown, "ContainerCreating") + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + logger, _, _ := newBufferedLogger(t) + p := newProgressPrinter(logger, "vray", taskStore, logStore, true, false, newGateStatuses(), newSkippedKeys(), false) + pinned := time.Now().Add(-3 * heartbeatInterval) + p.lastEmit = pinned + + p.flushHeartbeat() + + assert.Equal(t, pinned, p.lastEmit, + "flushHeartbeat must not touch lastEmit — that's what caused the every-other-tick suppression") +} + +func TestFlushHeartbeat_SuppressedWhenRecentEmit(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + ts := statestore.NewReadinessTaskState("app", "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, ts, 1) + addPodChild(t, ts, "app-pending", "ns", statestore.ResourceStatusUnknown, "ContainerCreating") + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "vray", taskStore, logStore, true, false, newGateStatuses(), newSkippedKeys(), false) + // lastEmit is current — we just printed something, so the heartbeat + // should stay quiet. + p.lastEmit = time.Now() + + p.flushHeartbeat() + + assert.Empty(t, capturedOutput(buf, mu), + "heartbeat must not emit while printer is actively chatty") +} + +func TestFlushHeartbeat_SuppressedWhenAllReady(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + // Set the task status explicitly to Ready. kubedog's default condition + // function requires 1+replicas resources to be Ready (root + children), + // so it's simpler and more direct to assert the post-rollout state by + // pinning the task status — that's how kubedog itself flips a task to + // Ready at the end of a successful track. + ts := statestore.NewReadinessTaskState("app", "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + ts.SetStatus(statestore.ReadinessTaskStatusReady) + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "vray", taskStore, logStore, true, false, newGateStatuses(), newSkippedKeys(), false) + p.lastEmit = time.Now().Add(-3 * heartbeatInterval) + + p.flushHeartbeat() + + assert.Empty(t, capturedOutput(buf, mu), + "heartbeat must not emit when every tracked task is Ready") +} + +func TestFlushHeartbeat_CapsResourceListAndShowsOverflow(t *testing.T) { + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + // Five progressing tasks — heartbeat caps the display at 3 and reports + // the remaining count. + for _, name := range []string{"a", "b", "c", "d", "e"} { + ts := statestore.NewReadinessTaskState(name, "ns", deploymentGVK, statestore.ReadinessTaskStateOptions{}) + setRequiredReplicas(t, ts, 1) + addPodChild(t, ts, name+"-pod", "ns", statestore.ResourceStatusUnknown, "ContainerCreating") + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + } + + logger, buf, mu := newBufferedLogger(t) + p := newProgressPrinter(logger, "vray", taskStore, logStore, true, false, newGateStatuses(), newSkippedKeys(), false) + p.lastEmit = time.Now().Add(-3 * heartbeatInterval) + + p.flushHeartbeat() + out := capturedOutput(buf, mu) + + assert.Contains(t, out, "5 progressing") + assert.Contains(t, out, "Deployment/a") + assert.Contains(t, out, "Deployment/c") + assert.Contains(t, out, "+2", "overflow count must indicate how many progressing items were hidden") + assert.NotContains(t, out, "Deployment/e", + "progressing items beyond the cap must be summarized by overflow count, not listed") +} + +func TestProgressPrinter_FullRun_CancelsAndDrainsOnContextDone(t *testing.T) { + // Smoke test that the run loop terminates cleanly on context cancel. + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + logger, _, _ := newBufferedLogger(t) + p := newProgressPrinter(logger, "", taskStore, logStore, true, false, newGateStatuses(), newSkippedKeys(), false) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + returned := make(chan struct{}) + go func() { + p.run(ctx, done) + close(returned) + }() + cancel() + select { + case <-returned: + case <-time.After(2 * time.Second): + t.Fatal("printer.run did not return after ctx cancel") + } +} + +// addPodLogs appends log lines to a pod's ResourceLogs entry in the log +// store, creating the entry if needed. Uses an incrementing timestamp so +// the printer's chronological sort is deterministic. +func addPodLogs(t *testing.T, logStore *kdutil.Concurrent[*logstore.LogStore], podName string, lines ...string) { + t.Helper() + logStore.RWTransaction(func(s *logstore.LogStore) { + var existing *kdutil.Concurrent[*logstore.ResourceLogs] + for _, rlC := range s.ResourcesLogs() { + rlC.RTransaction(func(rl *logstore.ResourceLogs) { + if rl.Name() == podName && rl.Namespace() == "ns" { + existing = rlC + } + }) + if existing != nil { + break + } + } + if existing == nil { + existing = kdutil.NewConcurrent(logstore.NewResourceLogs(podName, "ns", podGVK)) + s.AddResourceLogs(existing) + } + base := time.Unix(0, 0) + existing.RWTransaction(func(rl *logstore.ResourceLogs) { + for i, line := range lines { + rl.AddLogLine(line, "container/main", base.Add(time.Duration(i)*time.Millisecond)) + } + }) + }) +} + +// flipPodPhase mutates an existing pod child's Status attribute in place. +// Used to simulate transitions like "Completed" -> "CrashLoopBackOff". +func flipPodPhase(t *testing.T, ts *statestore.ReadinessTaskState, podName, namespace, newPhase string) { + t.Helper() + rs := ts.ResourceState(podName, namespace, podGVK) + rs.RWTransaction(func(rs *statestore.ResourceState) { + // Find existing Status attribute and rewrite its Value, if it exists. + for _, attr := range rs.Attributes() { + if attr.Name() == statestore.AttributeNameStatus { + if a, ok := attr.(*statestore.Attribute[string]); ok { + a.Value = newPhase + return + } + } + } + // Otherwise add one. + rs.AddAttribute(statestore.NewAttribute(statestore.AttributeNameStatus, newPhase)) + }) +} diff --git a/pkg/kubedog/tracker.go b/pkg/kubedog/tracker.go index e31d4b76..7f51d66a 100644 --- a/pkg/kubedog/tracker.go +++ b/pkg/kubedog/tracker.go @@ -2,24 +2,26 @@ package kubedog import ( "context" + "errors" "fmt" "math" "os" + "sort" "strings" "sync" "time" - "github.com/werf/kubedog/pkg/display" "github.com/werf/kubedog/pkg/informer" - "github.com/werf/kubedog/pkg/tracker" - "github.com/werf/kubedog/pkg/tracker/canary" - "github.com/werf/kubedog/pkg/tracker/daemonset" - "github.com/werf/kubedog/pkg/tracker/deployment" - "github.com/werf/kubedog/pkg/tracker/job" - "github.com/werf/kubedog/pkg/tracker/statefulset" - "github.com/werf/kubedog/pkg/trackers/dyntracker/util" + "github.com/werf/kubedog/pkg/trackers/dyntracker" + "github.com/werf/kubedog/pkg/trackers/dyntracker/logstore" + "github.com/werf/kubedog/pkg/trackers/dyntracker/statestore" + kdutil "github.com/werf/kubedog/pkg/trackers/dyntracker/util" "go.uber.org/zap" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery" "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" @@ -27,7 +29,6 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/clientcmd" - watchtools "k8s.io/client-go/tools/watch" "github.com/helmfile/helmfile/pkg/resource" ) @@ -44,7 +45,7 @@ type clientCacheEntry struct { dynamicClient dynamic.Interface restConfig *rest.Config discovery discovery.CachedDiscoveryInterface - mapper meta.RESTMapper + mapper meta.ResettableRESTMapper } var ( @@ -57,17 +58,35 @@ type Tracker struct { clientSet kubernetes.Interface dynamicClient dynamic.Interface discovery discovery.CachedDiscoveryInterface - mapper meta.RESTMapper + mapper meta.ResettableRESTMapper trackOptions *TrackOptions filter *resource.ResourceFilter namespace string + releaseName string + + // upstreamDoneCh is closed when the calling code (e.g. helm.SyncRelease) + // finishes. Per-resource freshness gates use it to give up waiting for a + // generation bump that will never come (e.g. when helm produced no diff + // for that resource). + upstreamDoneCh chan struct{} + upstreamOnce sync.Once + + // skipped records resources whose freshness gate exited without ever + // observing a change (helm decided not to create/update them, e.g. a + // post-upgrade hook on an install). Shared between TrackResources and + // VerifyAllConverged so the safety valve doesn't keep polling for + // resources that will never exist. + skipped *skippedKeys } type TrackerConfig struct { - Logger *zap.SugaredLogger - Namespace string - KubeContext string - Kubeconfig string + Logger *zap.SugaredLogger + Namespace string + KubeContext string + Kubeconfig string + // ReleaseName is shown in the progress header (e.g. "Release 'foo' + // progress:"). When empty the printer falls back to a generic header. + ReleaseName string TrackOptions *TrackOptions KubedogQPS *float32 KubedogBurst *int @@ -117,17 +136,30 @@ func NewTracker(config *TrackerConfig) (*Tracker, error) { } return &Tracker{ - logger: logger, - clientSet: cacheEntry.clientSet, - dynamicClient: cacheEntry.dynamicClient, - discovery: cacheEntry.discovery, - mapper: cacheEntry.mapper, - trackOptions: options, - filter: filter, - namespace: config.Namespace, + logger: logger, + clientSet: cacheEntry.clientSet, + dynamicClient: cacheEntry.dynamicClient, + discovery: cacheEntry.discovery, + mapper: cacheEntry.mapper, + trackOptions: options, + filter: filter, + namespace: config.Namespace, + releaseName: config.ReleaseName, + upstreamDoneCh: make(chan struct{}), + skipped: newSkippedKeys(), }, nil } +// MarkUpstreamCompleted signals to any in-flight freshness gates that the +// upstream operation (helm upgrade) has finished. Resources whose baseline +// generation never bumped will then exit their gate without attaching kubedog +// rather than wait indefinitely. Safe to call multiple times. +func (t *Tracker) MarkUpstreamCompleted() { + t.upstreamOnce.Do(func() { + close(t.upstreamDoneCh) + }) +} + func getOrCreateClients(kubeContext, kubeconfig string, qps float32, burst int) (clientCacheEntry, error) { key := cacheKey{ kubeContext: kubeContext, @@ -199,32 +231,145 @@ type trackTarget struct { kind string name string namespace string + gvk schema.GroupVersionKind } -// defaultSaveLogsReplicas is the number of pods per resource for which kubedog -// will stream logs when log tracking is enabled. kubedog ignores logs unless -// SaveLogsOnlyForNumberOfReplicas is positive (see ignoreLogs in -// pkg/tracker/deployment/tracker.go), so leaving it at 0 silently suppresses -// all log output even when trackLogs is true. -const defaultSaveLogsReplicas = 10 +// BaselineKey returns the map key used to associate a resource with its +// pre-change ResourceBaseline. +func BaselineKey(kind, namespace, name string) string { + return kind + "/" + namespace + "/" + name +} -// buildTrackerOptions constructs the kubedog tracker options from the tracker's -// configured TrackOptions. It is factored out of TrackResources so the option -// mapping (in particular enabling log streaming) can be unit tested without a -// live cluster. -func (t *Tracker) buildTrackerOptions(ctx context.Context) tracker.Options { - opts := tracker.Options{ - ParentContext: ctx, - Timeout: t.trackOptions.Timeout, - LogsFromTime: time.Now().Add(-t.trackOptions.LogsSince), - IgnoreLogs: !t.trackOptions.Logs, +// CaptureBaselines fetches the current UID and metadata.generation for each +// resource via the dynamic client. Resources that don't exist yet are +// recorded with Exists=false so the freshness gate can detect first creation. +// Errors other than NotFound are logged and the baseline is omitted (the +// gate will then attach immediately rather than block on a degraded probe). +func (t *Tracker) CaptureBaselines(ctx context.Context, resources []*resource.Resource) map[string]ResourceBaseline { + baselines := make(map[string]ResourceBaseline, len(resources)) + for _, res := range resources { + ns := res.Namespace + if ns == "" { + ns = t.namespace + } + kind, gvk, ok := classifyResource(res.Kind) + if !ok { + continue + } + gvr, err := t.gvrFor(gvk) + if err != nil { + t.logger.Debugf("kubedog: cannot resolve GVR for %s/%s/%s baseline: %v", kind, ns, res.Name, err) + continue + } + key := BaselineKey(kind, ns, res.Name) + obj, err := t.dynamicClient.Resource(gvr).Namespace(ns).Get(ctx, res.Name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + baselines[key] = ResourceBaseline{Exists: false} + continue + } + t.logger.Debugf("kubedog: baseline fetch for %s/%s/%s failed: %v", kind, ns, res.Name, err) + continue + } + baselines[key] = ResourceBaseline{ + UID: obj.GetUID(), + Generation: obj.GetGeneration(), + Exists: true, + } + } + return baselines +} + +func (t *Tracker) gvrFor(gvk schema.GroupVersionKind) (schema.GroupVersionResource, error) { + mapping, err := t.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return schema.GroupVersionResource{}, err + } + return mapping.Resource, nil +} + +// errUpstreamDoneNoChange signals that the upstream operation (helm) finished +// without the resource ever changing — the goroutine should skip the dyntracker +// attach and exit cleanly. +var errUpstreamDoneNoChange = errors.New("upstream completed without resource change") + +// waitForFreshness blocks until the resource has either appeared (when it +// didn't exist at baseline), been recreated (UID changed), or had its spec +// updated (generation > baseline). Returns nil once the resource is fresh, +// errUpstreamDoneNoChange when the upstream operation completed and no change +// was ever observed, or ctx.Err() on cancellation. +func (t *Tracker) waitForFreshness(ctx context.Context, tgt trackTarget, baseline ResourceBaseline, statusCb func(string)) error { + gvr, err := t.gvrFor(tgt.gvk) + if err != nil { + t.logger.Debugf("kubedog: cannot resolve GVR for %s/%s/%s, skipping freshness gate: %v", tgt.kind, tgt.namespace, tgt.name, err) + return nil } - if t.trackOptions.Logs { - opts.SaveLogsOnlyForNumberOfReplicas = defaultSaveLogsReplicas + if statusCb != nil { + if baseline.Exists { + statusCb(fmt.Sprintf("waiting for update (uid=%s gen=%d)", baseline.UID, baseline.Generation)) + } else { + statusCb("waiting for creation") + } } - return opts + const ( + pollInterval = 500 * time.Millisecond + // Grace window after helm finishes: keep checking briefly in case the + // API server lags. If nothing observed in this window, give up. + postUpstreamGrace = 3 * time.Second + ) + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + + probe := func() (fresh bool) { + obj, err := t.dynamicClient.Resource(gvr).Namespace(tgt.namespace).Get(ctx, tgt.name, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + t.logger.Debugf("kubedog: freshness probe for %s/%s/%s failed: %v", tgt.kind, tgt.namespace, tgt.name, err) + } + return false + } + if !baseline.Exists { + return true + } + if obj.GetUID() != baseline.UID { + return true + } + if obj.GetGeneration() > baseline.Generation { + return true + } + return false + } + + var upstreamDoneAt time.Time + // upstreamCh is a local view of t.upstreamDoneCh. We nil it out the first + // time it fires so the select below can't busy-loop on the closed channel + // (a closed channel is always ready, so without this each iteration would + // re-probe the API as fast as the round-trip allows for the whole grace + // window). After it fires once, polling continues ticker-bound. + upstreamCh := t.upstreamDoneCh + for { + if probe() { + return nil + } + if !upstreamDoneAt.IsZero() && time.Since(upstreamDoneAt) >= postUpstreamGrace { + return errUpstreamDoneNoChange + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-upstreamCh: + if upstreamDoneAt.IsZero() { + upstreamDoneAt = time.Now() + } + // Record the timestamp once, then stop selecting on the (now + // closed) channel so subsequent polls are ticker-throttled. + upstreamCh = nil + case <-ticker.C: + } + } } func (t *Tracker) TrackResources(ctx context.Context, resources []*resource.Resource) error { @@ -239,343 +384,203 @@ func (t *Tracker) TrackResources(ctx context.Context, resources []*resource.Reso return nil } - t.logger.Infof("Tracking %d resources with kubedog (filtered from %d total)", len(filtered), len(resources)) + // The "Tracking N resources from release X with kubedog" line and the + // "Tracking breakdown" summary are emitted by the caller (helmx) as part + // of the per-release preamble block, so we don't repeat them here. Only + // mention the filter when it actually removed something — that's + // behavioral info, not preamble. + if len(filtered) < len(resources) { + t.logger.Infof("kubedog filter kept %d of %d resources", len(filtered), len(resources)) + } targets := t.buildTargets(filtered) if len(targets) == 0 { - t.logger.Info("No trackable resources found (only Deployment, StatefulSet, DaemonSet, Job, and Canary are supported)") + t.logger.Info("No trackable resources found (only Deployment, StatefulSet, DaemonSet, Job, Canary, and PersistentVolumeClaim are supported)") return nil } - t.logger.Infof("Tracking breakdown: %s", t.targetSummary(targets)) + trackCtx, cancel := context.WithCancel(ctx) + defer cancel() - watchErrCh := make(chan error, len(targets)) + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + logStore := kdutil.NewConcurrent(logstore.NewLogStore()) + + watchErrCh := make(chan error, 16) informerFactory := informer.NewConcurrentInformerFactory( - ctx.Done(), - watchErrCh, - t.dynamicClient, + trackCtx.Done(), watchErrCh, t.dynamicClient, informer.ConcurrentInformerFactoryOptions{}, ) - opts := t.buildTrackerOptions(ctx) - - var wg sync.WaitGroup - errCh := make(chan error, len(targets)) - - for _, target := range targets { - wg.Add(1) - go func(tgt trackTarget) { - defer wg.Done() - if err := t.trackSingleResource(tgt, informerFactory, opts); err != nil { - errCh <- fmt.Errorf("%s/%s tracking failed: %w", tgt.kind, tgt.name, err) - } - }(target) - } - - done := make(chan struct{}) + // Drain informer watch errors so the factory doesn't block. go func() { - wg.Wait() - close(done) + for { + select { + case err, ok := <-watchErrCh: + if !ok { + return + } + if err != nil { + t.logger.Warnf("kubedog informer watch error: %v", err) + } + case <-trackCtx.Done(): + return + } + } }() + captureLogsFromTime := time.Now().Add(-t.trackOptions.LogsSince) + // Capture logs whenever either flag wants them. The kubedog tracker + // records logs into the logStore unconditionally; the failed-only mode + // is enforced at print time by the printer. + wantLogsCapture := t.trackOptions.Logs || t.trackOptions.FailedLogsOnly + ignoreLogs := !wantLogsCapture + // failedLogsOnly takes effect only when full streaming isn't already on. + failedLogsOnly := t.trackOptions.FailedLogsOnly && !t.trackOptions.Logs + // skipLogsInPrinter mutes the printer entirely if neither mode is on. + skipLogsInPrinter := !wantLogsCapture + + type trackerEntry struct { + target trackTarget + taskState *kdutil.Concurrent[*statestore.ReadinessTaskState] + } + entries := make([]trackerEntry, 0, len(targets)) + + for _, tgt := range targets { + taskState := kdutil.NewConcurrent( + statestore.NewReadinessTaskState(tgt.name, tgt.namespace, tgt.gvk, statestore.ReadinessTaskStateOptions{}), + ) + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(taskState) + }) + entries = append(entries, trackerEntry{target: tgt, taskState: taskState}) + } + + gateStatuses := newGateStatuses() + printer := newProgressPrinter(t.logger, t.releaseName, taskStore, logStore, skipLogsInPrinter, failedLogsOnly, gateStatuses, t.skipped, t.trackOptions.Color) + printerDone := make(chan struct{}) + + // Spawn a parallel failure watchdog. It catches pods that genuinely + // failed in the cluster but never made it into dyntracker's graph due + // to the Pod-before-ReplicaSet linkage race, so they're at least + // visible to the operator even when kubedog's tree is missing them. + go t.runFailureWatchdog(trackCtx, taskStore, resources) + + var trackerWg sync.WaitGroup + errCh := make(chan error, len(entries)) + + for _, entry := range entries { + trackerWg.Add(1) + tgt := entry.target + ts := entry.taskState + baselineKey := BaselineKey(tgt.kind, tgt.namespace, tgt.name) + baseline, hasBaseline := t.trackOptions.Baselines[baselineKey] + go func() { + defer trackerWg.Done() + + if hasBaseline { + err := t.waitForFreshness(trackCtx, tgt, baseline, func(msg string) { + gateStatuses.set(baselineKey, msg) + }) + gateStatuses.clear(baselineKey) + switch { + case err == nil: + // resource changed; proceed to attach the tracker + case errors.Is(err, errUpstreamDoneNoChange): + t.logger.Debugf("kubedog: %s/%s/%s unchanged by upstream; skipping tracker", tgt.kind, tgt.namespace, tgt.name) + // Hide this task from the printer output: it never changed, + // so the persistent "progressing" we'd otherwise show is + // misleading. + t.skipped.add(kdutil.ResourceID(tgt.name, tgt.namespace, tgt.gvk)) + return + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + return + default: + errCh <- fmt.Errorf("%s/%s freshness gate failed: %w", tgt.kind, tgt.name, err) + return + } + } + + dt, err := dyntracker.NewDynamicReadinessTracker( + trackCtx, ts, logStore, informerFactory, + t.clientSet, t.dynamicClient, t.discovery, t.mapper, + dyntracker.DynamicReadinessTrackerOptions{ + Timeout: t.trackOptions.Timeout, + CaptureLogsFromTime: captureLogsFromTime, + IgnoreLogs: ignoreLogs, + // Kubedog uses this as a "stream logs for at most N replicas" + // cap. The zero value means "0 replicas", which silently disables + // log streaming even when IgnoreLogs is false. Use a high value + // so all replicas in any realistic deployment stream their logs. + SaveLogsOnlyForNumberOfReplicas: math.MaxInt32, + }, + ) + if err != nil { + errCh <- fmt.Errorf("create dynamic tracker for %s/%s: %w", tgt.kind, tgt.name, err) + return + } + if err := dt.Track(trackCtx); err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return + } + errCh <- fmt.Errorf("%s/%s tracking failed: %w", tgt.kind, tgt.name, err) + } + }() + } + + trackersDone := make(chan struct{}) + go func() { + trackerWg.Wait() + close(trackersDone) + }() + + go func() { + printer.run(trackCtx, trackersDone) + close(printerDone) + }() + + var firstErr error select { case err := <-errCh: - return fmt.Errorf("tracking failed: %w", err) - case <-done: - t.logger.Info("All resources tracked successfully") - return nil - case <-ctx.Done(): - return fmt.Errorf("tracking canceled: %w", ctx.Err()) + firstErr = err + // Surface the error immediately, while it's happening, instead of + // silently canceling everything and only logging at wait() time. + // Without this, the operator sees the progress block stop dead and + // then nothing for minutes (until helm.SyncRelease returns and + // helmfile drains the result channel). A one-line warning here + // tells them right away that kubedog tracking failed and that + // helmfile is now just waiting for helm to finish on its own. + releaseLabel := "kubedog" + if t.releaseName != "" { + releaseLabel = fmt.Sprintf("release '%s'", t.releaseName) + } + t.logger.Warnf("kubedog tracker stopped for %s: %v — helmfile will continue waiting for helm to finish; no more progress/log output until then", + releaseLabel, err) + cancel() + case <-trackersDone: + case <-trackCtx.Done(): + firstErr = trackCtx.Err() } -} -func (t *Tracker) trackSingleResource(target trackTarget, informerFactory *util.Concurrent[*informer.InformerFactory], opts tracker.Options) error { - parentContext := opts.ParentContext - if parentContext == nil { - parentContext = context.Background() - } - ctx, cancel := watchtools.ContextWithOptionalTimeout(parentContext, opts.Timeout) - defer cancel() + cancel() + <-trackersDone + <-printerDone - trackErrCh := make(chan error, 1) - doneCh := make(chan struct{}) - - switch target.kind { - case "deploy": - tr := deployment.NewTracker(target.name, target.namespace, t.clientSet, informerFactory, opts) - go t.runDeploymentTracker(ctx, tr, trackErrCh, doneCh) - return t.waitDeploymentTracker(ctx, tr, trackErrCh, doneCh) - case "sts": - tr := statefulset.NewTracker(target.name, target.namespace, t.clientSet, informerFactory, opts) - go t.runStatefulSetTracker(ctx, tr, trackErrCh, doneCh) - return t.waitStatefulSetTracker(ctx, tr, trackErrCh, doneCh) - case "ds": - tr := daemonset.NewTracker(target.name, target.namespace, t.clientSet, informerFactory, opts) - go t.runDaemonSetTracker(ctx, tr, trackErrCh, doneCh) - return t.waitDaemonSetTracker(ctx, tr, trackErrCh, doneCh) - case "job": - tr := job.NewTracker(target.name, target.namespace, t.clientSet, informerFactory, opts) - go t.runJobTracker(ctx, tr, trackErrCh, doneCh) - return t.waitJobTracker(ctx, tr, trackErrCh, doneCh) - case "canary": - tr := canary.NewTracker(target.name, target.namespace, t.clientSet, t.dynamicClient, informerFactory, opts) - go t.runCanaryTracker(ctx, tr, trackErrCh, doneCh) - return t.waitCanaryTracker(ctx, tr, trackErrCh, doneCh) - default: - return fmt.Errorf("unsupported resource kind: %s", target.kind) - } -} - -func (t *Tracker) runDeploymentTracker(ctx context.Context, tr *deployment.Tracker, errCh chan<- error, doneCh chan<- struct{}) { - if err := tr.Track(ctx); err != nil { - errCh <- err - } else { - close(doneCh) - } -} - -func (t *Tracker) waitDeploymentTracker(ctx context.Context, tr *deployment.Tracker, trackErrCh <-chan error, doneCh <-chan struct{}) error { - var prevStatus deployment.DeploymentStatus - out := statusOutput() - resourceName := fmt.Sprintf("deploy/%s", tr.ResourceName) - - for { - select { - case status := <-tr.Added: - displayDeploymentStatusProgress(out, formatResourceCaption(resourceName, false, false), status, &prevStatus) - prevStatus = status - case <-tr.Ready: - displayDeploymentStatusProgress(out, formatResourceCaption(resourceName, true, false), prevStatus, &prevStatus) - t.logger.Infof("Deployment %s/%s is ready", tr.Namespace, tr.ResourceName) - return nil - case status := <-tr.Failed: - displayDeploymentStatusProgress(out, formatResourceCaption(resourceName, false, true), status, &prevStatus) - return fmt.Errorf("deployment %s/%s failed: %s", tr.Namespace, tr.ResourceName, status.FailedReason) - case status := <-tr.Status: - if status.StatusGeneration > prevStatus.StatusGeneration { - displayDeploymentStatusProgress(out, formatResourceCaption(resourceName, false, false), status, &prevStatus) - prevStatus = status - } - case msg := <-tr.EventMsg: - t.logger.Infof("deploy/%s: %s", tr.ResourceName, msg) - case chunk := <-tr.PodLogChunk: - t.logPodLogChunk(chunk.PodName, chunk.LogLines) - case report := <-tr.PodError: - t.logger.Warnf("deploy/%s pod %s: %s: %s", tr.ResourceName, report.ReplicaSetPodError.PodName, report.ReplicaSetPodError.ContainerName, report.ReplicaSetPodError.Message) - case <-tr.AddedReplicaSet: - case <-tr.AddedPod: - case err := <-trackErrCh: - return err - case <-doneCh: - return nil - case <-ctx.Done(): - return fmt.Errorf("tracking canceled for deployment %s/%s: %w", tr.Namespace, tr.ResourceName, ctx.Err()) + // Drain remaining tracker errors so they're surfaced in logs. + close(errCh) + for err := range errCh { + if firstErr == nil { + firstErr = err + } else { + t.logger.Warnf("additional tracking error: %v", err) } } -} -func (t *Tracker) runStatefulSetTracker(ctx context.Context, tr *statefulset.Tracker, errCh chan<- error, doneCh chan<- struct{}) { - if err := tr.Track(ctx); err != nil { - errCh <- err - } else { - close(doneCh) + if firstErr != nil { + return firstErr } -} -func (t *Tracker) waitStatefulSetTracker(ctx context.Context, tr *statefulset.Tracker, trackErrCh <-chan error, doneCh <-chan struct{}) error { - var prevStatus statefulset.StatefulSetStatus - out := statusOutput() - resourceName := fmt.Sprintf("sts/%s", tr.ResourceName) - - for { - select { - case status := <-tr.Added: - displayStatefulSetStatusProgress(out, formatResourceCaption(resourceName, false, false), status, &prevStatus) - prevStatus = status - case <-tr.Ready: - displayStatefulSetStatusProgress(out, formatResourceCaption(resourceName, true, false), prevStatus, &prevStatus) - t.logger.Infof("StatefulSet %s/%s is ready", tr.Namespace, tr.ResourceName) - return nil - case status := <-tr.Failed: - displayStatefulSetStatusProgress(out, formatResourceCaption(resourceName, false, true), status, &prevStatus) - return fmt.Errorf("statefulset %s/%s failed: %s", tr.Namespace, tr.ResourceName, status.FailedReason) - case status := <-tr.Status: - if status.StatusGeneration > prevStatus.StatusGeneration { - displayStatefulSetStatusProgress(out, formatResourceCaption(resourceName, false, false), status, &prevStatus) - prevStatus = status - } - case msg := <-tr.EventMsg: - t.logger.Infof("sts/%s: %s", tr.ResourceName, msg) - case chunk := <-tr.PodLogChunk: - t.logPodLogChunk(chunk.PodName, chunk.LogLines) - case report := <-tr.PodError: - t.logger.Warnf("sts/%s pod %s: %s: %s", tr.ResourceName, report.ReplicaSetPodError.PodName, report.ReplicaSetPodError.ContainerName, report.ReplicaSetPodError.Message) - case <-tr.AddedPod: - case err := <-trackErrCh: - return err - case <-doneCh: - return nil - case <-ctx.Done(): - return fmt.Errorf("tracking canceled for statefulset %s/%s: %w", tr.Namespace, tr.ResourceName, ctx.Err()) - } - } -} - -func (t *Tracker) runDaemonSetTracker(ctx context.Context, tr *daemonset.Tracker, errCh chan<- error, doneCh chan<- struct{}) { - if err := tr.Track(ctx); err != nil { - errCh <- err - } else { - close(doneCh) - } -} - -func (t *Tracker) waitDaemonSetTracker(ctx context.Context, tr *daemonset.Tracker, trackErrCh <-chan error, doneCh <-chan struct{}) error { - var prevStatus daemonset.DaemonSetStatus - out := statusOutput() - resourceName := fmt.Sprintf("ds/%s", tr.ResourceName) - - for { - select { - case status := <-tr.Added: - displayDaemonSetStatusProgress(out, formatResourceCaption(resourceName, false, false), status, &prevStatus) - prevStatus = status - case <-tr.Ready: - displayDaemonSetStatusProgress(out, formatResourceCaption(resourceName, true, false), prevStatus, &prevStatus) - t.logger.Infof("DaemonSet %s/%s is ready", tr.Namespace, tr.ResourceName) - return nil - case status := <-tr.Failed: - displayDaemonSetStatusProgress(out, formatResourceCaption(resourceName, false, true), status, &prevStatus) - return fmt.Errorf("daemonset %s/%s failed: %s", tr.Namespace, tr.ResourceName, status.FailedReason) - case status := <-tr.Status: - if status.StatusGeneration > prevStatus.StatusGeneration { - displayDaemonSetStatusProgress(out, formatResourceCaption(resourceName, false, false), status, &prevStatus) - prevStatus = status - } - case msg := <-tr.EventMsg: - t.logger.Infof("ds/%s: %s", tr.ResourceName, msg) - case chunk := <-tr.PodLogChunk: - t.logPodLogChunk(chunk.PodName, chunk.LogLines) - case report := <-tr.PodError: - t.logger.Warnf("ds/%s pod %s: %s: %s", tr.ResourceName, report.PodError.PodName, report.PodError.ContainerName, report.PodError.Message) - case <-tr.AddedPod: - case err := <-trackErrCh: - return err - case <-doneCh: - return nil - case <-ctx.Done(): - return fmt.Errorf("tracking canceled for daemonset %s/%s: %w", tr.Namespace, tr.ResourceName, ctx.Err()) - } - } -} - -func (t *Tracker) runJobTracker(ctx context.Context, tr *job.Tracker, errCh chan<- error, doneCh chan<- struct{}) { - if err := tr.Track(ctx); err != nil { - errCh <- err - } else { - close(doneCh) - } -} - -func (t *Tracker) waitJobTracker(ctx context.Context, tr *job.Tracker, trackErrCh <-chan error, doneCh <-chan struct{}) error { - var prevStatus job.JobStatus - out := statusOutput() - resourceName := fmt.Sprintf("job/%s", tr.ResourceName) - - for { - select { - case status := <-tr.Added: - displayJobStatusProgress(out, formatResourceCaption(resourceName, false, false), status, &prevStatus) - prevStatus = status - case <-tr.Succeeded: - displayJobStatusProgress(out, formatResourceCaption(resourceName, true, false), prevStatus, &prevStatus) - t.logger.Infof("Job %s/%s succeeded", tr.Namespace, tr.ResourceName) - return nil - case status := <-tr.Failed: - displayJobStatusProgress(out, formatResourceCaption(resourceName, false, true), status, &prevStatus) - return fmt.Errorf("job %s/%s failed: %s", tr.Namespace, tr.ResourceName, status.FailedReason) - case status := <-tr.Status: - if status.StatusGeneration > prevStatus.StatusGeneration { - displayJobStatusProgress(out, formatResourceCaption(resourceName, false, false), status, &prevStatus) - prevStatus = status - } - case msg := <-tr.EventMsg: - t.logger.Infof("job/%s: %s", tr.ResourceName, msg) - case chunk := <-tr.PodLogChunk: - t.logPodLogChunk(chunk.PodName, chunk.LogLines) - case report := <-tr.PodError: - t.logger.Warnf("job/%s pod %s: %s: %s", tr.ResourceName, report.PodError.PodName, report.PodError.ContainerName, report.PodError.Message) - case <-tr.AddedPod: - case err := <-trackErrCh: - return err - case <-doneCh: - return nil - case <-ctx.Done(): - return fmt.Errorf("tracking canceled for job %s/%s: %w", tr.Namespace, tr.ResourceName, ctx.Err()) - } - } -} - -func (t *Tracker) runCanaryTracker(ctx context.Context, tr *canary.Tracker, errCh chan<- error, doneCh chan<- struct{}) { - if err := tr.Track(ctx); err != nil { - errCh <- err - } else { - close(doneCh) - } -} - -func (t *Tracker) waitCanaryTracker(ctx context.Context, tr *canary.Tracker, trackErrCh <-chan error, doneCh <-chan struct{}) error { - out := statusOutput() - resourceName := fmt.Sprintf("canary/%s", tr.ResourceName) - var lastView CanaryStatusView - - for { - select { - case status := <-tr.Added: - view := CanaryStatusView{ - Phase: string(status.CanaryStatus.Phase), - IsFailed: status.IsFailed, - } - displayCanaryStatus(out, formatResourceCaption(resourceName, false, false), view) - lastView = view - case <-tr.Succeeded: - displayCanaryStatus(out, formatResourceCaption(resourceName, true, false), CanaryStatusView{Phase: lastView.Phase}) - t.logger.Infof("Canary %s/%s succeeded", tr.Namespace, tr.ResourceName) - return nil - case status := <-tr.Failed: - displayCanaryStatus(out, formatResourceCaption(resourceName, false, true), CanaryStatusView{ - Phase: status.FailedReason, - IsFailed: true, - }) - return fmt.Errorf("canary %s/%s failed: %s", tr.Namespace, tr.ResourceName, status.FailedReason) - case status := <-tr.Status: - view := CanaryStatusView{ - Phase: func() string { - if status.StatusIndicator != nil { - return status.StatusIndicator.Value - } - return "" - }(), - Age: status.Age, - IsFailed: status.IsFailed, - } - displayCanaryStatus(out, formatResourceCaption(resourceName, false, false), view) - lastView = view - case msg := <-tr.EventMsg: - t.logger.Infof("canary/%s: %s", tr.ResourceName, msg) - case err := <-trackErrCh: - return err - case <-doneCh: - return nil - case <-ctx.Done(): - return fmt.Errorf("tracking canceled for canary %s/%s: %w", tr.Namespace, tr.ResourceName, ctx.Err()) - } - } -} - -func (t *Tracker) logPodLogChunk(podName string, logLines []display.LogLine) { - for _, line := range logLines { - t.logger.Infof("po/%s [%s] %s", podName, line.Timestamp, line.Message) - } + t.logger.Info("All resources tracked successfully") + return nil } func (t *Tracker) buildTargets(resources []*resource.Resource) []trackTarget { @@ -586,19 +591,8 @@ func (t *Tracker) buildTargets(resources []*resource.Resource) []trackTarget { namespace = t.namespace } - kind := "" - switch strings.ToLower(res.Kind) { - case "deployment", "deploy": - kind = "deploy" - case "statefulset", "sts": - kind = "sts" - case "daemonset", "ds": - kind = "ds" - case "job": - kind = "job" - case "canary": - kind = "canary" - default: + kind, gvk, ok := classifyResource(res.Kind) + if !ok { t.logger.Debugf("Skipping unsupported kind %s for resource %s/%s", res.Kind, namespace, res.Name) continue } @@ -607,11 +601,196 @@ func (t *Tracker) buildTargets(resources []*resource.Resource) []trackTarget { kind: kind, name: res.Name, namespace: namespace, + gvk: gvk, }) } return targets } +// VerifyAllConverged returns true if every tracked resource in the list is in +// a terminal "done" state per its kind's readiness convention, as reported by +// the live API. It's used as a safety valve in the parallel-tracking flow: +// once helm has signaled success and a grace period has elapsed, if the +// cluster confirms everything healthy but the dyntracker is still wedged +// (a known kubedog race where a Job pod completes faster than the resource +// graph updates ResourceStatus to Ready), the caller can confidently cancel +// the tracker instead of waiting out --track-timeout. +// +// Conservative on errors: any failure to fetch or any unrecognized kind +// returns false so the caller keeps polling rather than declaring premature +// success. +func (t *Tracker) VerifyAllConverged(ctx context.Context, resources []*resource.Resource) bool { + for _, res := range resources { + kind, gvk, ok := classifyResource(res.Kind) + if !ok { + // Resource is not a kind we track — skip it. Same semantics as + // buildTargets, so the verification scope matches what kubedog + // is actually waiting on. + continue + } + // Skip resources whose freshness gate exited via + // errUpstreamDoneNoChange: helm decided not to create or update + // them (e.g. a post-upgrade-only hook on an install). They'll + // never exist in the cluster, so polling Get on them would + // permanently return NotFound and block the safety valve. + if t.skipped != nil && t.skipped.has(kdutil.ResourceID(res.Name, res.Namespace, gvk)) { + continue + } + gvr, err := t.gvrFor(gvk) + if err != nil { + t.logger.Debugf("kubedog safety valve: cannot resolve GVR for %s/%s/%s: %v", kind, res.Namespace, res.Name, err) + return false + } + obj, err := t.dynamicClient.Resource(gvr).Namespace(res.Namespace).Get(ctx, res.Name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + // Helm reported success but the resource is missing — that's + // not "converged", it's a different problem. Let kubedog + // continue waiting and surface whatever error it picks up. + return false + } + t.logger.Debugf("kubedog safety valve: GET %s/%s/%s failed: %v", kind, res.Namespace, res.Name, err) + return false + } + if !isResourceConverged(kind, obj) { + return false + } + } + return true +} + +// isResourceConverged dispatches to the kind-specific terminal-done check. +// Returns false for kinds whose readiness can't be summarized as a single +// API field (e.g. Canary, which has its own multi-phase state machine) — in +// those cases we defer to dyntracker rather than risk false positives. +func isResourceConverged(kind string, obj *unstructured.Unstructured) bool { + switch kind { + case "deploy": + return isDeploymentConverged(obj) + case "sts": + return isStatefulSetConverged(obj) + case "ds": + return isDaemonSetConverged(obj) + case "job": + return isJobConverged(obj) + case "pvc": + return isPVCConverged(obj) + } + return false +} + +// isDeploymentConverged: observedGeneration must have caught up with the +// current spec, and availableReplicas must satisfy the desired replica count. +// availableReplicas is the right field (not readyReplicas) because it accounts +// for minReadySeconds — matching kstatus's Current definition. +func isDeploymentConverged(obj *unstructured.Unstructured) bool { + gen, _, _ := unstructured.NestedInt64(obj.Object, "metadata", "generation") + observed, _, _ := unstructured.NestedInt64(obj.Object, "status", "observedGeneration") + if observed < gen { + return false + } + replicas, found, _ := unstructured.NestedInt64(obj.Object, "spec", "replicas") + if !found { + replicas = 1 + } + if replicas == 0 { + return true + } + available, _, _ := unstructured.NestedInt64(obj.Object, "status", "availableReplicas") + return available >= replicas +} + +// isStatefulSetConverged: same generation gate as Deployment, plus the rolling +// update must be complete (currentRevision == updateRevision) and readyReplicas +// must meet the desired count. +func isStatefulSetConverged(obj *unstructured.Unstructured) bool { + gen, _, _ := unstructured.NestedInt64(obj.Object, "metadata", "generation") + observed, _, _ := unstructured.NestedInt64(obj.Object, "status", "observedGeneration") + if observed < gen { + return false + } + currentRev, _, _ := unstructured.NestedString(obj.Object, "status", "currentRevision") + updateRev, _, _ := unstructured.NestedString(obj.Object, "status", "updateRevision") + if updateRev != "" && currentRev != updateRev { + return false + } + replicas, found, _ := unstructured.NestedInt64(obj.Object, "spec", "replicas") + if !found { + replicas = 1 + } + if replicas == 0 { + return true + } + ready, _, _ := unstructured.NestedInt64(obj.Object, "status", "readyReplicas") + return ready >= replicas +} + +// isDaemonSetConverged: numberReady must meet desiredNumberScheduled and the +// rolling update must have observed the current spec. +func isDaemonSetConverged(obj *unstructured.Unstructured) bool { + gen, _, _ := unstructured.NestedInt64(obj.Object, "metadata", "generation") + observed, _, _ := unstructured.NestedInt64(obj.Object, "status", "observedGeneration") + if observed < gen { + return false + } + desired, _, _ := unstructured.NestedInt64(obj.Object, "status", "desiredNumberScheduled") + ready, _, _ := unstructured.NestedInt64(obj.Object, "status", "numberReady") + return desired > 0 && ready >= desired +} + +// isJobConverged: succeeded count meets the completions target. This is the +// exact check whose miss in dyntracker motivated the safety valve. +func isJobConverged(obj *unstructured.Unstructured) bool { + completions, found, _ := unstructured.NestedInt64(obj.Object, "spec", "completions") + if !found { + completions = 1 + } + succeeded, _, _ := unstructured.NestedInt64(obj.Object, "status", "succeeded") + return succeeded >= completions +} + +// isPVCConverged: Bound is the only phase that means "usable by workloads." +func isPVCConverged(obj *unstructured.Unstructured) bool { + phase, _, _ := unstructured.NestedString(obj.Object, "status", "phase") + return phase == "Bound" +} + +func classifyResource(rawKind string) (string, schema.GroupVersionKind, bool) { + switch strings.ToLower(rawKind) { + case "deployment", "deploy": + return "deploy", schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, true + case "statefulset", "sts": + return "sts", schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"}, true + case "daemonset", "ds": + return "ds", schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "DaemonSet"}, true + case "job": + return "job", schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"}, true + case "canary": + return "canary", schema.GroupVersionKind{Group: "flagger.app", Version: "v1beta1", Kind: "Canary"}, true + case "persistentvolumeclaim", "pvc": + return "pvc", schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PersistentVolumeClaim"}, true + } + return "", schema.GroupVersionKind{}, false +} + +// PreviewBreakdown applies the same filtering and classification that +// TrackResources will, then returns the kept resources and a human-readable +// breakdown string (e.g. "deploys=3, jobs=2, stss=1") suitable for the +// per-release preamble. Returns an empty summary if no resources survive +// filtering or no resources are of a trackable kind. Pure: no logging, no +// state mutation, no API calls. +func (t *Tracker) PreviewBreakdown(resources []*resource.Resource) (kept []*resource.Resource, summary string) { + filtered := t.filterResources(resources) + if len(filtered) == 0 { + return nil, "" + } + targets := t.buildTargets(filtered) + if len(targets) == 0 { + return filtered, "" + } + return filtered, t.targetSummary(targets) +} + func (t *Tracker) targetSummary(targets []trackTarget) string { counts := make(map[string]int) for _, tgt := range targets { @@ -621,6 +800,7 @@ func (t *Tracker) targetSummary(targets []trackTarget) string { for kind, count := range counts { parts = append(parts, fmt.Sprintf("%ss=%d", kind, count)) } + sort.Strings(parts) return strings.Join(parts, ", ") } diff --git a/pkg/kubedog/tracker_test.go b/pkg/kubedog/tracker_test.go index 96e23f80..abfb4b02 100644 --- a/pkg/kubedog/tracker_test.go +++ b/pkg/kubedog/tracker_test.go @@ -4,11 +4,20 @@ import ( "context" "math" "os" + "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + kdutil "github.com/werf/kubedog/pkg/trackers/dyntracker/util" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic/fake" "github.com/helmfile/helmfile/pkg/resource" ) @@ -277,30 +286,349 @@ users: } } -func TestBuildTrackerOptions_LogsEnabled(t *testing.T) { - tr := &Tracker{ - trackOptions: NewTrackOptions().WithLogs(true), +func TestClassifyResource(t *testing.T) { + tests := []struct { + rawKind string + wantKind string + wantGVK schema.GroupVersionKind + wantOK bool + }{ + {"Deployment", "deploy", schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, true}, + {"deployment", "deploy", schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, true}, + {"deploy", "deploy", schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, true}, + {"StatefulSet", "sts", schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"}, true}, + {"sts", "sts", schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"}, true}, + {"DaemonSet", "ds", schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "DaemonSet"}, true}, + {"Job", "job", schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"}, true}, + {"Canary", "canary", schema.GroupVersionKind{Group: "flagger.app", Version: "v1beta1", Kind: "Canary"}, true}, + {"PersistentVolumeClaim", "pvc", schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PersistentVolumeClaim"}, true}, + {"pvc", "pvc", schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PersistentVolumeClaim"}, true}, + {"ConfigMap", "", schema.GroupVersionKind{}, false}, + {"Service", "", schema.GroupVersionKind{}, false}, + {"", "", schema.GroupVersionKind{}, false}, + } + for _, tc := range tests { + t.Run(tc.rawKind, func(t *testing.T) { + kind, gvk, ok := classifyResource(tc.rawKind) + assert.Equal(t, tc.wantOK, ok) + assert.Equal(t, tc.wantKind, kind) + assert.Equal(t, tc.wantGVK, gvk) + }) } - - opts := tr.buildTrackerOptions(context.Background()) - - // With logs enabled, kubedog must be told to stream logs and to save logs - // for a positive number of replicas, otherwise log output is silently - // suppressed. - assert.False(t, opts.IgnoreLogs) - assert.Equal(t, defaultSaveLogsReplicas, opts.SaveLogsOnlyForNumberOfReplicas) - assert.Equal(t, 5*time.Minute, opts.Timeout) } -func TestBuildTrackerOptions_LogsDisabled(t *testing.T) { +func TestBaselineKey(t *testing.T) { + assert.Equal(t, "deploy/ns/name", BaselineKey("deploy", "ns", "name")) + assert.Equal(t, "job//standalone", BaselineKey("job", "", "standalone")) +} + +func TestResourceBaseline_DefaultsAreZeroValue(t *testing.T) { + var b ResourceBaseline + assert.Equal(t, types.UID(""), b.UID) + assert.Equal(t, int64(0), b.Generation) + assert.False(t, b.Exists) +} + +func TestTrackOptions_WithColor(t *testing.T) { + opts := NewTrackOptions() + assert.False(t, opts.Color) + + opts = opts.WithColor(true) + assert.True(t, opts.Color) +} + +func TestTrackOptions_WithFailedLogsOnly(t *testing.T) { + opts := NewTrackOptions() + assert.False(t, opts.FailedLogsOnly) + + opts = opts.WithFailedLogsOnly(true) + assert.True(t, opts.FailedLogsOnly) +} + +func TestTrackOptions_WithBaselines(t *testing.T) { + opts := NewTrackOptions() + assert.Nil(t, opts.Baselines) + + baselines := map[string]ResourceBaseline{ + BaselineKey("deploy", "ns", "foo"): {UID: "uid-a", Generation: 3, Exists: true}, + } + opts = opts.WithBaselines(baselines) + assert.Equal(t, baselines, opts.Baselines) + + // Mutating the original map after the call must be visible through the + // stored field — confirms the setter retains the map by reference rather + // than copying. + baselines[BaselineKey("deploy", "ns", "bar")] = ResourceBaseline{Generation: 5} + assert.Contains(t, opts.Baselines, BaselineKey("deploy", "ns", "bar")) +} + +func TestTrackOptions_Chaining_AllSetters(t *testing.T) { + filter := &resource.FilterConfig{TrackKinds: []string{"Deployment"}} + baselines := map[string]ResourceBaseline{ + BaselineKey("deploy", "ns", "foo"): {UID: "uid-a", Generation: 1, Exists: true}, + } + opts := NewTrackOptions(). + WithTimeout(2 * time.Minute). + WithLogs(true). + WithFailedLogsOnly(true). + WithFilterConfig(filter). + WithQPS(50). + WithBurst(80). + WithColor(true). + WithBaselines(baselines) + + assert.Equal(t, 2*time.Minute, opts.Timeout) + assert.True(t, opts.Logs) + assert.True(t, opts.FailedLogsOnly) + assert.Equal(t, filter, opts.Filter) + assert.Equal(t, float32(50), opts.QPS) + assert.Equal(t, 80, opts.Burst) + assert.True(t, opts.Color) + assert.Equal(t, baselines, opts.Baselines) +} + +// makeObj builds an *unstructured.Unstructured with the given path/value +// pairs. Each key is a dotted path (e.g. "status.succeeded"); the value is +// stored verbatim, with int values written as int64 to match +// unstructured.NestedInt64 expectations. +func makeObj(t *testing.T, paths map[string]any) *unstructured.Unstructured { + t.Helper() + obj := &unstructured.Unstructured{Object: map[string]any{}} + for path, val := range paths { + fields := strings.Split(path, ".") + // Normalize ints to int64 because that's what the API returns and + // what NestedInt64 will type-assert against. + if iv, ok := val.(int); ok { + val = int64(iv) + } + require.NoError(t, unstructured.SetNestedField(obj.Object, val, fields...)) + } + return obj +} + +func TestIsJobConverged(t *testing.T) { + t.Run("default completions (1) and succeeded 1 — done", func(t *testing.T) { + obj := makeObj(t, map[string]any{ + "status.succeeded": 1, + }) + assert.True(t, isJobConverged(obj)) + }) + t.Run("explicit completions 3 and succeeded 3 — done", func(t *testing.T) { + obj := makeObj(t, map[string]any{ + "spec.completions": 3, + "status.succeeded": 3, + }) + assert.True(t, isJobConverged(obj)) + }) + t.Run("succeeded 0 — not done", func(t *testing.T) { + obj := makeObj(t, map[string]any{ + "status.succeeded": 0, + }) + assert.False(t, isJobConverged(obj)) + }) + t.Run("succeeded less than completions — not done", func(t *testing.T) { + obj := makeObj(t, map[string]any{ + "spec.completions": 5, + "status.succeeded": 4, + }) + assert.False(t, isJobConverged(obj)) + }) +} + +func TestIsDeploymentConverged(t *testing.T) { + t.Run("observedGeneration caught up and availableReplicas meets replicas — done", func(t *testing.T) { + obj := makeObj(t, map[string]any{ + "metadata.generation": 3, + "status.observedGeneration": 3, + "spec.replicas": 2, + "status.availableReplicas": 2, + }) + assert.True(t, isDeploymentConverged(obj)) + }) + t.Run("observedGeneration lags — not done", func(t *testing.T) { + obj := makeObj(t, map[string]any{ + "metadata.generation": 5, + "status.observedGeneration": 4, + "spec.replicas": 1, + "status.availableReplicas": 1, + }) + assert.False(t, isDeploymentConverged(obj)) + }) + t.Run("available less than replicas — not done", func(t *testing.T) { + obj := makeObj(t, map[string]any{ + "metadata.generation": 1, + "status.observedGeneration": 1, + "spec.replicas": 3, + "status.availableReplicas": 2, + }) + assert.False(t, isDeploymentConverged(obj)) + }) + t.Run("replicas explicitly 0 — done regardless of availability", func(t *testing.T) { + obj := makeObj(t, map[string]any{ + "metadata.generation": 1, + "status.observedGeneration": 1, + "spec.replicas": 0, + }) + assert.True(t, isDeploymentConverged(obj)) + }) +} + +func TestIsStatefulSetConverged(t *testing.T) { + t.Run("rolling update finished, ready meets replicas — done", func(t *testing.T) { + obj := makeObj(t, map[string]any{ + "metadata.generation": 2, + "status.observedGeneration": 2, + "status.currentRevision": "rev-a", + "status.updateRevision": "rev-a", + "spec.replicas": 3, + "status.readyReplicas": 3, + }) + assert.True(t, isStatefulSetConverged(obj)) + }) + t.Run("rolling update in progress (revisions differ) — not done", func(t *testing.T) { + obj := makeObj(t, map[string]any{ + "metadata.generation": 2, + "status.observedGeneration": 2, + "status.currentRevision": "rev-a", + "status.updateRevision": "rev-b", + "spec.replicas": 3, + "status.readyReplicas": 3, + }) + assert.False(t, isStatefulSetConverged(obj)) + }) +} + +func TestIsDaemonSetConverged(t *testing.T) { + t.Run("ready meets desired — done", func(t *testing.T) { + obj := makeObj(t, map[string]any{ + "metadata.generation": 1, + "status.observedGeneration": 1, + "status.desiredNumberScheduled": 4, + "status.numberReady": 4, + }) + assert.True(t, isDaemonSetConverged(obj)) + }) + t.Run("ready short of desired — not done", func(t *testing.T) { + obj := makeObj(t, map[string]any{ + "metadata.generation": 1, + "status.observedGeneration": 1, + "status.desiredNumberScheduled": 4, + "status.numberReady": 3, + }) + assert.False(t, isDaemonSetConverged(obj)) + }) + t.Run("zero desired (no matching nodes) — not yet converged", func(t *testing.T) { + // Zero desired typically means the selector matched no nodes yet; + // safety-valve must not treat that as success because it could just + // be that controllers haven't seen the DaemonSet. + obj := makeObj(t, map[string]any{ + "metadata.generation": 1, + "status.observedGeneration": 1, + "status.desiredNumberScheduled": 0, + "status.numberReady": 0, + }) + assert.False(t, isDaemonSetConverged(obj)) + }) +} + +func TestIsPVCConverged(t *testing.T) { + assert.True(t, isPVCConverged(makeObj(t, map[string]any{"status.phase": "Bound"}))) + assert.False(t, isPVCConverged(makeObj(t, map[string]any{"status.phase": "Pending"}))) + assert.False(t, isPVCConverged(makeObj(t, map[string]any{}))) +} + +func TestIsResourceConverged_DispatchesByKind(t *testing.T) { + job := makeObj(t, map[string]any{"status.succeeded": 1}) + assert.True(t, isResourceConverged("job", job)) + + // Canary intentionally has no live-API converged check — we always + // return false and defer to dyntracker for its multi-phase progression. + assert.False(t, isResourceConverged("canary", job)) + + // Unknown kind also returns false to avoid false-positive cancellations. + assert.False(t, isResourceConverged("totally-made-up", job)) +} + +// staticRESTMapper resolves a fixed set of GVK -> GVR mappings for the test; +// avoids spinning up a discovery client. +type staticRESTMapper struct { + meta.RESTMapper + mappings map[schema.GroupVersionKind]schema.GroupVersionResource +} + +func (m *staticRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { + for gvk, gvr := range m.mappings { + if gvk.GroupKind() == gk { + return &meta.RESTMapping{Resource: gvr, GroupVersionKind: gvk}, nil + } + } + return nil, &meta.NoKindMatchError{GroupKind: gk} +} + +func (m *staticRESTMapper) Reset() {} + +func TestVerifyAllConverged_SkipsResourcesThatHelmNeverCreated(t *testing.T) { + // The bug: on an install, post-upgrade-only hooks appear in the + // templated resource list but helm correctly skips them. Their + // freshness gate exits via errUpstreamDoneNoChange and they land in + // the skipped set. VerifyAllConverged used to GET them anyway, get + // NotFound, and return false — keeping the safety valve permanently + // suppressed for any install with post-upgrade hooks present. + jobGVK := schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"} + jobGVR := schema.GroupVersionResource{Group: "batch", Version: "v1", Resource: "jobs"} + + // Only the "real" Job exists in the cluster; the skipped one does not. + realJob := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": map[string]any{"name": "feeds-db-insert-init", "namespace": "ns"}, + "spec": map[string]any{"completions": int64(1)}, + "status": map[string]any{"succeeded": int64(1)}, + }} + scheme := runtime.NewScheme() + fakeClient := fake.NewSimpleDynamicClient(scheme, realJob) + tr := &Tracker{ - trackOptions: NewTrackOptions().WithLogs(false), + logger: zap.NewNop().Sugar(), + dynamicClient: fakeClient, + mapper: &staticRESTMapper{mappings: map[schema.GroupVersionKind]schema.GroupVersionResource{jobGVK: jobGVR}}, + skipped: newSkippedKeys(), + } + // Pretend the freshness gate flagged update-vray-schema as skipped + // (post-upgrade hook helm didn't create on this install). + tr.skipped.add(kdutil.ResourceID("update-vray-schema", "ns", jobGVK)) + + resources := []*resource.Resource{ + {Kind: "Job", Name: "feeds-db-insert-init", Namespace: "ns"}, + {Kind: "Job", Name: "update-vray-schema", Namespace: "ns"}, } - opts := tr.buildTrackerOptions(context.Background()) + assert.True(t, tr.VerifyAllConverged(context.Background(), resources), + "safety valve must consider the release converged when the only non-existent resource is one helm deliberately skipped") +} - // With logs disabled, kubedog should ignore logs and not save any replicas' - // logs (leaving the default zero value). - assert.True(t, opts.IgnoreLogs) - assert.Zero(t, opts.SaveLogsOnlyForNumberOfReplicas) +func TestVerifyAllConverged_NotFoundForNonSkippedReturnsFalse(t *testing.T) { + // Counterexample: if a resource that helm was supposed to create is + // genuinely missing from the cluster, VerifyAllConverged must NOT + // claim convergence — that's a different problem and we want kubedog + // to keep waiting / surface the real error. + jobGVK := schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"} + jobGVR := schema.GroupVersionResource{Group: "batch", Version: "v1", Resource: "jobs"} + + scheme := runtime.NewScheme() + fakeClient := fake.NewSimpleDynamicClient(scheme) // cluster is empty + + tr := &Tracker{ + logger: zap.NewNop().Sugar(), + dynamicClient: fakeClient, + mapper: &staticRESTMapper{mappings: map[schema.GroupVersionKind]schema.GroupVersionResource{jobGVK: jobGVR}}, + skipped: newSkippedKeys(), + } + + resources := []*resource.Resource{ + {Kind: "Job", Name: "expected-but-missing", Namespace: "ns"}, + } + + assert.False(t, tr.VerifyAllConverged(context.Background(), resources), + "missing resource that wasn't deliberately skipped must keep the safety valve suppressed") } diff --git a/pkg/kubedog/watchdog.go b/pkg/kubedog/watchdog.go new file mode 100644 index 00000000..1d7e0602 --- /dev/null +++ b/pkg/kubedog/watchdog.go @@ -0,0 +1,308 @@ +package kubedog + +import ( + "context" + "fmt" + "time" + + "github.com/werf/kubedog/pkg/trackers/dyntracker/statestore" + kdutil "github.com/werf/kubedog/pkg/trackers/dyntracker/util" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + + "github.com/helmfile/helmfile/pkg/resource" +) + +// failureWatchdogInterval is how often we re-scan the cluster for failing +// pods that dyntracker hasn't surfaced. 60s keeps API server pressure +// negligible (one LIST per tracked workload per minute) while still +// catching CrashLoopBackOff loops well before --track-timeout fires. +const failureWatchdogInterval = 60 * time.Second + +var ( + watchdogPodGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + watchdogPodGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} + watchdogReplicaSetGVR = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "replicasets"} +) + +// watchdogWorkload identifies a workload whose pods we'll independently +// LIST against the live API. +type watchdogWorkload struct { + kind string + gvk schema.GroupVersionKind + name string + namespace string +} + +// runFailureWatchdog watches for pods that genuinely failed in the cluster +// but never made it into dyntracker's resource graph. This case happens +// when dyntracker loses the Pod-to-Deployment linkage race (Pod CREATE +// event arriving before its ReplicaSet CREATE event) — the Pod is +// completely invisible to kubedog's tracking pipeline: no progress row, +// no log stream, no failed-only-mode entry. Without this watchdog, a Pod +// in CrashLoopBackOff under a missed link would be silent until +// --track-timeout, which is a real safety gap for CI pipelines that rely +// on helmfile to surface failures. +// +// The watchdog runs alongside dyntracker, does not gate any decisions, and +// only emits warnings. It cannot fix the linkage — but it can make sure +// failures are at least visible to the operator. +func (t *Tracker) runFailureWatchdog(ctx context.Context, taskStore *kdutil.Concurrent[*statestore.TaskStore], resources []*resource.Resource) { + workloads := watchdogWorkloads(resources) + if len(workloads) == 0 { + return + } + + // warned tracks pod IDs we've already surfaced so we don't repeat the + // same warning every minute for the same failing pod. + warned := map[string]struct{}{} + + ticker := time.NewTicker(failureWatchdogInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + t.scanForMissedFailures(ctx, taskStore, workloads, warned) + } + } +} + +// watchdogWorkloads filters tracked resources down to the kinds whose +// pods we can reasonably LIST by selector. PVCs have no pods; Canaries +// have complex multi-revision pod sets we shouldn't second-guess. +func watchdogWorkloads(resources []*resource.Resource) []watchdogWorkload { + var out []watchdogWorkload + for _, res := range resources { + kind, gvk, ok := classifyResource(res.Kind) + if !ok { + continue + } + if kind == "pvc" || kind == "canary" { + continue + } + out = append(out, watchdogWorkload{kind: kind, gvk: gvk, name: res.Name, namespace: res.Namespace}) + } + return out +} + +func (t *Tracker) scanForMissedFailures(ctx context.Context, taskStore *kdutil.Concurrent[*statestore.TaskStore], workloads []watchdogWorkload, warned map[string]struct{}) { + for _, wl := range workloads { + gvr, err := t.gvrFor(wl.gvk) + if err != nil { + continue + } + obj, err := t.dynamicClient.Resource(gvr).Namespace(wl.namespace).Get(ctx, wl.name, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + t.logger.Debugf("kubedog watchdog: GET %s/%s/%s failed: %v", wl.kind, wl.namespace, wl.name, err) + } + continue + } + workloadUID := obj.GetUID() + + selector := extractPodSelector(obj) + if selector == "" { + continue + } + + pods, err := t.dynamicClient.Resource(watchdogPodGVR).Namespace(wl.namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + t.logger.Debugf("kubedog watchdog: LIST pods for %s/%s/%s failed: %v", wl.kind, wl.namespace, wl.name, err) + continue + } + + // For Deployments, pods are not directly owned by the workload — + // they're owned by a ReplicaSet that's in turn owned by the + // Deployment. We lazy-load the RS UIDs only if we actually have a + // failing pod that doesn't match the workload UID directly. + var deploymentRSUIDs map[string]struct{} + + for i := range pods.Items { + pod := &pods.Items[i] + podName := pod.GetName() + podID := kdutil.ResourceID(podName, wl.namespace, watchdogPodGVK) + if _, seen := warned[podID]; seen { + continue + } + reason := detectPodFailureReason(pod) + if reason == "" { + continue + } + if podIsInTaskStore(taskStore, podName, wl.namespace) { + continue + } + // Ownership check: the pod must belong to the current workload, + // not to a previous instance whose pods still share the same + // labels (e.g., a failed pre-install hook whose Job hasn't been + // cleaned up yet). Without this check, the watchdog cries wolf + // every time a stale failing pod is still around at the start + // of a new install. + if !podOwnedBy(pod, workloadUID, wl.kind, func() map[string]struct{} { + if deploymentRSUIDs == nil { + deploymentRSUIDs = t.rsUIDsOwnedBy(ctx, wl.namespace, workloadUID) + } + return deploymentRSUIDs + }) { + continue + } + msg := fmt.Sprintf("[watchdog] Pod %s/%s is failing (%s) under %s/%s/%s but kubedog tracker is not tracking it. Inspect with: kubectl -n %s logs %s", + wl.namespace, podName, reason, wl.kind, wl.namespace, wl.name, wl.namespace, podName) + t.logger.Warnf("%s", StyleWarning(msg, t.trackOptions.Color)) + warned[podID] = struct{}{} + } + } +} + +// podOwnedBy reports whether the pod's ownerReferences chain leads to the +// given workload UID. For Job/StatefulSet/DaemonSet the workload directly +// owns the pod, so a direct UID match suffices. For Deployment the pod is +// owned by an intermediate ReplicaSet — the caller passes a lazy lookup of +// "RS UIDs owned by this Deployment" so we only LIST RSes when we actually +// need to disambiguate. +func podOwnedBy(pod *unstructured.Unstructured, workloadUID types.UID, workloadKind string, deploymentRSUIDs func() map[string]struct{}) bool { + for _, uid := range podOwnerUIDs(pod) { + if uid == workloadUID { + return true + } + } + if workloadKind != "deploy" { + return false + } + rsUIDs := deploymentRSUIDs() + for _, uid := range podOwnerUIDs(pod) { + if _, ok := rsUIDs[string(uid)]; ok { + return true + } + } + return false +} + +func podOwnerUIDs(obj *unstructured.Unstructured) []types.UID { + refs := obj.GetOwnerReferences() + out := make([]types.UID, 0, len(refs)) + for _, r := range refs { + out = append(out, r.UID) + } + return out +} + +// rsUIDsOwnedBy returns the set of ReplicaSet UIDs in the namespace whose +// metadata.ownerReferences contain the given workload UID. Used to validate +// pod ownership for Deployments, whose pods are one indirection away from +// the workload (Pod → RS → Deployment). +func (t *Tracker) rsUIDsOwnedBy(ctx context.Context, namespace string, workloadUID types.UID) map[string]struct{} { + rsList, err := t.dynamicClient.Resource(watchdogReplicaSetGVR).Namespace(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + t.logger.Debugf("kubedog watchdog: LIST replicasets in %s failed: %v", namespace, err) + return map[string]struct{}{} + } + out := map[string]struct{}{} + for i := range rsList.Items { + rs := &rsList.Items[i] + for _, uid := range podOwnerUIDs(rs) { + if uid == workloadUID { + out[string(rs.GetUID())] = struct{}{} + break + } + } + } + return out +} + +// extractPodSelector pulls the workload's pod label selector. Deployments, +// StatefulSets, DaemonSets, and Jobs all use spec.selector.matchLabels by +// convention; we ignore matchExpressions for simplicity (rare in practice +// and the watchdog is best-effort). Returns "" when nothing usable is +// present. +func extractPodSelector(obj *unstructured.Unstructured) string { + matchLabels, found, _ := unstructured.NestedStringMap(obj.Object, "spec", "selector", "matchLabels") + if !found || len(matchLabels) == 0 { + return "" + } + return labels.SelectorFromSet(matchLabels).String() +} + +// detectPodFailureReason returns a short reason string for the pod's +// failing state, or "" if the pod looks healthy. Mirrors the phases +// isFailingPodPhase classifies as failing, plus the additional +// terminated-with-error and OOMKilled signals available from container +// statuses. +func detectPodFailureReason(pod *unstructured.Unstructured) string { + if phase, _, _ := unstructured.NestedString(pod.Object, "status", "phase"); phase == "Failed" { + return "Failed" + } + + containers, found, _ := unstructured.NestedSlice(pod.Object, "status", "containerStatuses") + if found { + if r := scanContainerStatusesForFailure(containers); r != "" { + return r + } + } + initContainers, found, _ := unstructured.NestedSlice(pod.Object, "status", "initContainerStatuses") + if found { + if r := scanContainerStatusesForFailure(initContainers); r != "" { + return fmt.Sprintf("init: %s", r) + } + } + return "" +} + +func scanContainerStatusesForFailure(containers []any) string { + for _, c := range containers { + cMap, ok := c.(map[string]any) + if !ok { + continue + } + state, hasState, _ := unstructured.NestedMap(cMap, "state") + if !hasState { + continue + } + if waiting, ok := state["waiting"].(map[string]any); ok { + if reason, _ := waiting["reason"].(string); isFailingPodPhase(reason) { + return reason + } + } + if terminated, ok := state["terminated"].(map[string]any); ok { + if reason, _ := terminated["reason"].(string); reason == "OOMKilled" || reason == "Error" { + return reason + } + } + } + return "" +} + +// podIsInTaskStore returns true if dyntracker has any ResourceState for a +// Pod with the given name/namespace anywhere in the task store. A "true" +// result means kubedog already sees this pod and will surface its state +// through the normal progress/log pipeline; the watchdog should stay +// quiet. +func podIsInTaskStore(taskStore *kdutil.Concurrent[*statestore.TaskStore], podName, namespace string) bool { + var present bool + taskStore.RTransaction(func(s *statestore.TaskStore) { + for _, taskC := range s.ReadinessTasksStates() { + if present { + return + } + taskC.RTransaction(func(ts *statestore.ReadinessTaskState) { + for _, rsC := range ts.ResourceStates() { + if present { + return + } + rsC.RTransaction(func(rs *statestore.ResourceState) { + if rs.GroupVersionKind().Kind == "Pod" && rs.Name() == podName && rs.Namespace() == namespace { + present = true + } + }) + } + }) + } + }) + return present +} diff --git a/pkg/kubedog/watchdog_test.go b/pkg/kubedog/watchdog_test.go new file mode 100644 index 00000000..206ca035 --- /dev/null +++ b/pkg/kubedog/watchdog_test.go @@ -0,0 +1,500 @@ +package kubedog + +import ( + "bytes" + "context" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/werf/kubedog/pkg/trackers/dyntracker/statestore" + kdutil "github.com/werf/kubedog/pkg/trackers/dyntracker/util" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic/fake" + + "github.com/helmfile/helmfile/pkg/resource" +) + +func TestDetectPodFailureReason(t *testing.T) { + tests := []struct { + name string + obj map[string]any + want string + }{ + { + name: "phase Failed", + obj: map[string]any{"status": map[string]any{"phase": "Failed"}}, + want: "Failed", + }, + { + name: "container waiting CrashLoopBackOff", + obj: map[string]any{"status": map[string]any{ + "phase": "Running", + "containerStatuses": []any{ + map[string]any{"state": map[string]any{ + "waiting": map[string]any{"reason": "CrashLoopBackOff"}, + }}, + }, + }}, + want: "CrashLoopBackOff", + }, + { + name: "container waiting ImagePullBackOff", + obj: map[string]any{"status": map[string]any{ + "containerStatuses": []any{ + map[string]any{"state": map[string]any{ + "waiting": map[string]any{"reason": "ImagePullBackOff"}, + }}, + }, + }}, + want: "ImagePullBackOff", + }, + { + name: "container terminated OOMKilled", + obj: map[string]any{"status": map[string]any{ + "containerStatuses": []any{ + map[string]any{"state": map[string]any{ + "terminated": map[string]any{"reason": "OOMKilled"}, + }}, + }, + }}, + want: "OOMKilled", + }, + { + name: "init container failing surfaces with init: prefix", + obj: map[string]any{"status": map[string]any{ + "initContainerStatuses": []any{ + map[string]any{"state": map[string]any{ + "waiting": map[string]any{"reason": "CrashLoopBackOff"}, + }}, + }, + }}, + want: "init: CrashLoopBackOff", + }, + { + name: "running pod with no waiting/terminated reason — healthy", + obj: map[string]any{"status": map[string]any{ + "phase": "Running", + "containerStatuses": []any{ + map[string]any{"state": map[string]any{ + "running": map[string]any{}, + }}, + }, + }}, + want: "", + }, + { + name: "empty status block", + obj: map[string]any{}, + want: "", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := detectPodFailureReason(&unstructured.Unstructured{Object: tc.obj}) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestExtractPodSelector(t *testing.T) { + t.Run("matchLabels round-trips through SelectorFromSet", func(t *testing.T) { + obj := &unstructured.Unstructured{Object: map[string]any{ + "spec": map[string]any{ + "selector": map[string]any{ + "matchLabels": map[string]any{ + "app": "malware", "component": "worker", + }, + }, + }, + }} + got := extractPodSelector(obj) + // SelectorFromSet sorts keys, so the output is deterministic. + assert.Equal(t, "app=malware,component=worker", got) + }) + t.Run("missing selector returns empty string", func(t *testing.T) { + assert.Equal(t, "", extractPodSelector(&unstructured.Unstructured{Object: map[string]any{}})) + }) + t.Run("empty matchLabels returns empty string", func(t *testing.T) { + obj := &unstructured.Unstructured{Object: map[string]any{ + "spec": map[string]any{ + "selector": map[string]any{ + "matchLabels": map[string]any{}, + }, + }, + }} + assert.Equal(t, "", extractPodSelector(obj)) + }) +} + +func TestPodIsInTaskStore(t *testing.T) { + // Build a task store with one Deployment that has one tracked Pod child. + deployGVK := schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"} + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + ts := statestore.NewReadinessTaskState("app", "ns", deployGVK, statestore.ReadinessTaskStateOptions{}) + ts.AddResourceState("tracked-pod", "ns", watchdogPodGVK) + ts.AddDependency(ts.Name(), ts.Namespace(), ts.GroupVersionKind(), "tracked-pod", "ns", watchdogPodGVK) + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + assert.True(t, podIsInTaskStore(taskStore, "tracked-pod", "ns"), + "tracked pod must be found in the task store") + assert.False(t, podIsInTaskStore(taskStore, "untracked-pod", "ns"), + "pod the watchdog will need to surface must NOT be found in the task store") + // Different namespace must not match a same-named pod. + assert.False(t, podIsInTaskStore(taskStore, "tracked-pod", "other-ns"), + "pod lookup must be namespace-scoped") +} + +// newCaptureLogger returns a SugaredLogger whose Warn-level writes are +// captured into a buffer — sufficient to verify the watchdog actually emits +// the expected warning text. +func newCaptureLogger(t *testing.T) (*zap.SugaredLogger, *bytes.Buffer) { + t.Helper() + buf := &bytes.Buffer{} + var cfg zapcore.EncoderConfig + cfg.MessageKey = "message" + core := zapcore.NewCore(zapcore.NewConsoleEncoder(cfg), zapcore.AddSync(&captureWriter{buf: buf, mu: &sync.Mutex{}}), zapcore.WarnLevel) + return zap.New(core).Sugar(), buf +} + +type captureWriter struct { + buf *bytes.Buffer + mu *sync.Mutex +} + +func (w *captureWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.buf.Write(p) +} + +// newWatchdogFakeClient registers the list kinds the watchdog actually +// needs (Pods, Jobs, Deployments, ReplicaSets). The fake dynamic client +// panics on List for unregistered list kinds, so this helper is shared by +// the integration-style tests below. +func newWatchdogFakeClient(t *testing.T, objs ...runtime.Object) *fake.FakeDynamicClient { + t.Helper() + scheme := runtime.NewScheme() + for _, kind := range []schema.GroupVersionKind{ + {Group: "", Version: "v1", Kind: "PodList"}, + {Group: "batch", Version: "v1", Kind: "JobList"}, + {Group: "apps", Version: "v1", Kind: "DeploymentList"}, + {Group: "apps", Version: "v1", Kind: "ReplicaSetList"}, + } { + scheme.AddKnownTypeWithName(kind, &unstructured.UnstructuredList{}) + } + return fake.NewSimpleDynamicClient(scheme, objs...) +} + +func TestScanForMissedFailures_WarnsForUntrackedFailingPod(t *testing.T) { + // Cluster contains a Job and one failing pod owned by it (the one + // dyntracker missed linking). Task store has the Job but no Pod + // children — simulating the linkage race the watchdog is built to + // mitigate. Using a Job keeps the ownership chain direct (Pod → Job). + jobGVK := schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"} + jobGVR := schema.GroupVersionResource{Group: "batch", Version: "v1", Resource: "jobs"} + const jobUID = "11111111-1111-1111-1111-111111111111" + + jobObj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": map[string]any{"name": "malware", "namespace": "ns", "uid": jobUID}, + "spec": map[string]any{ + "selector": map[string]any{"matchLabels": map[string]any{"app": "malware"}}, + }, + }} + failingPod := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": map[string]any{ + "name": "malware-pod-untracked", + "namespace": "ns", + "labels": map[string]any{"app": "malware"}, + "ownerReferences": []any{ + map[string]any{"apiVersion": "batch/v1", "kind": "Job", "name": "malware", "uid": jobUID, "controller": true}, + }, + }, + "status": map[string]any{ + "phase": "Running", + "containerStatuses": []any{ + map[string]any{"state": map[string]any{ + "waiting": map[string]any{"reason": "CrashLoopBackOff"}, + }}, + }, + }, + }} + + logger, buf := newCaptureLogger(t) + tr := &Tracker{ + logger: logger, + dynamicClient: newWatchdogFakeClient(t, jobObj, failingPod), + mapper: &staticRESTMapper{mappings: map[schema.GroupVersionKind]schema.GroupVersionResource{ + jobGVK: jobGVR, + watchdogPodGVK: watchdogPodGVR, + }}, + trackOptions: &TrackOptions{}, + } + + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + warned := map[string]struct{}{} + tr.scanForMissedFailures(context.Background(), taskStore, []watchdogWorkload{ + {kind: "job", gvk: jobGVK, name: "malware", namespace: "ns"}, + }, warned) + + out := buf.String() + require.Contains(t, out, "malware-pod-untracked", "watchdog must name the missing failing pod") + require.Contains(t, out, "CrashLoopBackOff", "watchdog must name the failure reason") + require.Contains(t, out, "kubectl", "watchdog must give the operator an actionable inspection command") +} + +func TestScanForMissedFailures_IgnoresStalePodFromPreviousInstall(t *testing.T) { + // Reproduces the "stale failing pod" false positive: the current Job + // has its own UID, but a leftover failing pod from the previous install + // (with a different owner UID) still matches the label selector. The + // watchdog must skip it — that's not "our" pod and helm will clean it + // up shortly anyway. + jobGVK := schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"} + jobGVR := schema.GroupVersionResource{Group: "batch", Version: "v1", Resource: "jobs"} + const currentJobUID = "22222222-2222-2222-2222-222222222222" + const previousJobUID = "33333333-3333-3333-3333-333333333333" + + currentJob := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": map[string]any{"name": "feeds-db-insert-init", "namespace": "ns", "uid": currentJobUID}, + "spec": map[string]any{ + "selector": map[string]any{"matchLabels": map[string]any{"app": "feeds-db-insert-init"}}, + }, + }} + stalePod := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": map[string]any{ + "name": "feeds-db-insert-init-cbxvc", + "namespace": "ns", + "labels": map[string]any{"app": "feeds-db-insert-init"}, + "ownerReferences": []any{ + map[string]any{"apiVersion": "batch/v1", "kind": "Job", "name": "feeds-db-insert-init", "uid": previousJobUID, "controller": true}, + }, + }, + "status": map[string]any{"phase": "Failed"}, + }} + + logger, buf := newCaptureLogger(t) + tr := &Tracker{ + logger: logger, + dynamicClient: newWatchdogFakeClient(t, currentJob, stalePod), + mapper: &staticRESTMapper{mappings: map[schema.GroupVersionKind]schema.GroupVersionResource{ + jobGVK: jobGVR, + watchdogPodGVK: watchdogPodGVR, + }}, + trackOptions: &TrackOptions{}, + } + + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + warned := map[string]struct{}{} + tr.scanForMissedFailures(context.Background(), taskStore, []watchdogWorkload{ + {kind: "job", gvk: jobGVK, name: "feeds-db-insert-init", namespace: "ns"}, + }, warned) + + assert.NotContains(t, buf.String(), "feeds-db-insert-init-cbxvc", + "watchdog must NOT warn about a failing pod left over from a previous install (different owner UID)") +} + +func TestScanForMissedFailures_StaysQuietWhenDyntrackerAlreadyHasPod(t *testing.T) { + // Task store DOES contain the failing pod, so dyntracker is already + // surfacing it via the normal pipeline. Watchdog stays silent. + jobGVK := schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"} + jobGVR := schema.GroupVersionResource{Group: "batch", Version: "v1", Resource: "jobs"} + const jobUID = "44444444-4444-4444-4444-444444444444" + + jobObj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": map[string]any{"name": "app", "namespace": "ns", "uid": jobUID}, + "spec": map[string]any{"selector": map[string]any{"matchLabels": map[string]any{"app": "app"}}}, + }} + failingPod := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": map[string]any{ + "name": "app-pod", "namespace": "ns", + "labels": map[string]any{"app": "app"}, + "ownerReferences": []any{ + map[string]any{"apiVersion": "batch/v1", "kind": "Job", "name": "app", "uid": jobUID, "controller": true}, + }, + }, + "status": map[string]any{ + "containerStatuses": []any{ + map[string]any{"state": map[string]any{ + "waiting": map[string]any{"reason": "CrashLoopBackOff"}, + }}, + }, + }, + }} + + logger, buf := newCaptureLogger(t) + tr := &Tracker{ + logger: logger, + dynamicClient: newWatchdogFakeClient(t, jobObj, failingPod), + mapper: &staticRESTMapper{mappings: map[schema.GroupVersionKind]schema.GroupVersionResource{ + jobGVK: jobGVR, + watchdogPodGVK: watchdogPodGVR, + }}, + trackOptions: &TrackOptions{}, + } + + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + ts := statestore.NewReadinessTaskState("app", "ns", jobGVK, statestore.ReadinessTaskStateOptions{}) + ts.AddResourceState("app-pod", "ns", watchdogPodGVK) + ts.AddDependency(ts.Name(), ts.Namespace(), ts.GroupVersionKind(), "app-pod", "ns", watchdogPodGVK) + taskStore.RWTransaction(func(s *statestore.TaskStore) { + s.AddReadinessTaskState(kdutil.NewConcurrent(ts)) + }) + + warned := map[string]struct{}{} + tr.scanForMissedFailures(context.Background(), taskStore, []watchdogWorkload{ + {kind: "job", gvk: jobGVK, name: "app", namespace: "ns"}, + }, warned) + + assert.NotContains(t, buf.String(), "watchdog", "watchdog must not warn when dyntracker is already tracking the failing pod") +} + +func TestScanForMissedFailures_DoesNotRepeatWarningsAcrossScans(t *testing.T) { + jobGVK := schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"} + jobGVR := schema.GroupVersionResource{Group: "batch", Version: "v1", Resource: "jobs"} + const jobUID = "55555555-5555-5555-5555-555555555555" + + jobObj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": map[string]any{"name": "malware", "namespace": "ns", "uid": jobUID}, + "spec": map[string]any{"selector": map[string]any{"matchLabels": map[string]any{"app": "malware"}}}, + }} + failingPod := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": map[string]any{ + "name": "malware-pod-untracked", "namespace": "ns", + "labels": map[string]any{"app": "malware"}, + "ownerReferences": []any{ + map[string]any{"apiVersion": "batch/v1", "kind": "Job", "name": "malware", "uid": jobUID, "controller": true}, + }, + }, + "status": map[string]any{ + "containerStatuses": []any{ + map[string]any{"state": map[string]any{ + "waiting": map[string]any{"reason": "CrashLoopBackOff"}, + }}, + }, + }, + }} + + logger, buf := newCaptureLogger(t) + tr := &Tracker{ + logger: logger, + dynamicClient: newWatchdogFakeClient(t, jobObj, failingPod), + mapper: &staticRESTMapper{mappings: map[schema.GroupVersionKind]schema.GroupVersionResource{ + jobGVK: jobGVR, + watchdogPodGVK: watchdogPodGVR, + }}, + trackOptions: &TrackOptions{}, + } + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + warned := map[string]struct{}{} + workloads := []watchdogWorkload{{kind: "job", gvk: jobGVK, name: "malware", namespace: "ns"}} + tr.scanForMissedFailures(context.Background(), taskStore, workloads, warned) + tr.scanForMissedFailures(context.Background(), taskStore, workloads, warned) + + count := strings.Count(buf.String(), "[watchdog]") + assert.Equal(t, 1, count, "watchdog must warn at most once per pod across consecutive scans") +} + +func TestScanForMissedFailures_DeploymentPodMatchesViaReplicaSet(t *testing.T) { + // Deployment pods are not directly owned by the workload; the watchdog + // must walk Pod → ReplicaSet → Deployment to recognize a failing pod + // as "ours". + deployGVK := schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"} + deployGVR := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} + const deployUID = "66666666-6666-6666-6666-666666666666" + const rsUID = "77777777-7777-7777-7777-777777777777" + + deploy := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]any{"name": "app", "namespace": "ns", "uid": deployUID}, + "spec": map[string]any{ + "selector": map[string]any{"matchLabels": map[string]any{"app": "app"}}, + "replicas": int64(1), + }, + }} + rs := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "metadata": map[string]any{ + "name": "app-abc", "namespace": "ns", "uid": rsUID, + "ownerReferences": []any{ + map[string]any{"apiVersion": "apps/v1", "kind": "Deployment", "name": "app", "uid": deployUID, "controller": true}, + }, + }, + }} + failingPod := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": map[string]any{ + "name": "app-abc-xyz", "namespace": "ns", + "labels": map[string]any{"app": "app"}, + "ownerReferences": []any{ + map[string]any{"apiVersion": "apps/v1", "kind": "ReplicaSet", "name": "app-abc", "uid": rsUID, "controller": true}, + }, + }, + "status": map[string]any{ + "containerStatuses": []any{ + map[string]any{"state": map[string]any{ + "waiting": map[string]any{"reason": "CrashLoopBackOff"}, + }}, + }, + }, + }} + + logger, buf := newCaptureLogger(t) + tr := &Tracker{ + logger: logger, + dynamicClient: newWatchdogFakeClient(t, deploy, rs, failingPod), + mapper: &staticRESTMapper{mappings: map[schema.GroupVersionKind]schema.GroupVersionResource{ + deployGVK: deployGVR, + watchdogPodGVK: watchdogPodGVR, + }}, + trackOptions: &TrackOptions{}, + } + taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) + warned := map[string]struct{}{} + tr.scanForMissedFailures(context.Background(), taskStore, []watchdogWorkload{ + {kind: "deploy", gvk: deployGVK, name: "app", namespace: "ns"}, + }, warned) + + assert.Contains(t, buf.String(), "app-abc-xyz", + "watchdog must recognize a failing Deployment pod via the Pod → RS → Deployment chain") +} + +func TestWatchdogWorkloads_FiltersUntrackableKinds(t *testing.T) { + in := []*resource.Resource{ + {Kind: "Deployment", Name: "d", Namespace: "ns"}, + {Kind: "ConfigMap", Name: "cm", Namespace: "ns"}, // not tracked at all + {Kind: "PersistentVolumeClaim", Name: "p", Namespace: "ns"}, // tracked but no pods + {Kind: "Job", Name: "j", Namespace: "ns"}, + {Kind: "Canary", Name: "c", Namespace: "ns"}, // tracked but complex pod set; skip + } + got := watchdogWorkloads(in) + require.Len(t, got, 2) + assert.Equal(t, "deploy", got[0].kind) + assert.Equal(t, "job", got[1].kind) +} diff --git a/pkg/state/helmx.go b/pkg/state/helmx.go index 900e8213..62eb7bea 100644 --- a/pkg/state/helmx.go +++ b/pkg/state/helmx.go @@ -1,6 +1,7 @@ package state import ( + "bytes" "context" "fmt" "os" @@ -8,9 +9,12 @@ import ( "slices" "sort" "strings" + "sync" + "sync/atomic" "time" "github.com/helmfile/chartify" + "go.uber.org/zap" "helm.sh/helm/v4/pkg/storage/driver" "github.com/helmfile/helmfile/pkg/filesystem" @@ -255,6 +259,482 @@ func (st *HelmState) trackReleaseIfEnabled(ctx context.Context, release *Release return nil } +// kubedogTrackingHandle bundles the closures the caller needs to coordinate +// parallel kubedog tracking with helm execution. See +// startBackgroundKubedogTracking for the lifecycle. +type kubedogTrackingHandle struct { + // Helm is the helm.Interface the caller MUST use for SyncRelease and any + // follow-up commands (e.g. listReleases) while tracking is running. It's + // a logger-scoped clone of the original helm that captures all its output + // into the per-release buffer so it doesn't interleave with kubedog + // progress on stdout. + Helm helmexec.Interface + // Wait blocks until the tracker exits and returns the resulting error + // (already shaped by trackFailOnError policy). + Wait func() *ReleaseError + // Cancel cancels the tracker; call when helm itself fails. + Cancel func() + // NotifyHelmDone signals to in-flight freshness gates that helm finished + // so unchanged resources can stop waiting for a generation bump. + NotifyHelmDone func() + // FlushBufferedHelmOutput emits the captured helm output (upgrades, list, + // etc.) through the real logger in a single block. Safe to call multiple + // times — second call is a no-op. + FlushBufferedHelmOutput func() + // WasHelmKilled reports whether the safety-valve helm-killer fired during + // tracking. When true, the caller MUST treat any error returned by helm + // SyncRelease as success — helm was deliberately interrupted because the + // cluster confirmed convergence and helm wedged on its hook waiter. + WasHelmKilled func() bool +} + +// startBackgroundKubedogTracking templates the release upfront and starts a +// kubedog tracker in a goroutine so it runs in parallel with helm. It returns +// a handle whose Helm field MUST be used for the helm calls during the +// tracking window — that helm clone buffers output for clean ordering. +// +// When started is false the caller must fall back to the sequential +// trackReleaseIfEnabled path (e.g. templating failed before helm ran, so we +// retry after helm finishes to preserve the previous behavior). +// buildReleaseTracker resolves the per-release tracking options (timeout, +// log capture, filter config, color) and constructs the kubedog tracker. +// Extracted from startBackgroundKubedogTracking to keep that function within +// the statement-count limit. +func (st *HelmState) buildReleaseTracker(release *ReleaseSpec, opts *SyncOpts, useColor bool) (*kubedog.Tracker, *kubedog.TrackOptions, error) { + timeout := 5 * time.Minute + if release.TrackTimeout != nil && *release.TrackTimeout > 0 { + timeout = time.Duration(*release.TrackTimeout) * time.Second + } else if opts != nil && opts.TrackTimeout > 0 { + timeout = time.Duration(opts.TrackTimeout) * time.Second + } + + trackLogs := release.TrackLogs != nil && *release.TrackLogs + if release.TrackLogs == nil && opts != nil { + trackLogs = opts.TrackLogs + } + + trackFailedLogs := release.TrackFailedLogs != nil && *release.TrackFailedLogs + if release.TrackFailedLogs == nil && opts != nil { + trackFailedLogs = opts.TrackFailedLogs + } + + filterConfig := &resource.FilterConfig{ + TrackKinds: release.TrackKinds, + SkipKinds: release.SkipKinds, + TrackResources: convertTrackResources(release.TrackResources), + } + + trackOpts := kubedog.NewTrackOptions(). + WithTimeout(timeout). + WithLogs(trackLogs). + WithFailedLogsOnly(trackFailedLogs). + WithFilterConfig(filterConfig). + WithColor(useColor) + + tracker, err := kubedog.NewTracker(&kubedog.TrackerConfig{ + Logger: st.logger, + Namespace: release.Namespace, + KubeContext: st.getKubeContext(release), + Kubeconfig: st.kubeconfig, + ReleaseName: release.Name, + TrackOptions: trackOpts, + KubedogQPS: release.KubedogQPS, + KubedogBurst: release.KubedogBurst, + }) + if err != nil { + return nil, nil, err + } + return tracker, trackOpts, nil +} + +// bufferHelmOutput swaps the helm logger for one writing into an in-memory +// buffer (so helm output can be replayed as a single block after tracking), +// and derives a release-scoped context whose cancellation drives the existing +// ShellRunner SIGINT path. The release-scoped context is what lets the +// helm-stuck safety valve interrupt a wedged helm subprocess. +func (st *HelmState) bufferHelmOutput(helm helmexec.Interface, ctx context.Context, releaseName string) (bufHelm helmexec.Interface, buf *bytes.Buffer, releaseCancel context.CancelFunc) { + buf = &bytes.Buffer{} + bufHelm = helm + if swapper, ok := helm.(helmexec.LoggerSwapper); ok { + bufHelm = swapper.WithLogger(helmexec.NewLogger(buf, "info")) + } else { + st.logger.Debugf("kubedog: helm implementation does not support logger swap; output will interleave for release %s", releaseName) + } + releaseCtx, releaseCancel := context.WithCancel(ctx) + if swapper, ok := bufHelm.(helmexec.ContextSwapper); ok { + bufHelm = swapper.WithContext(releaseCtx) + } + return +} + +func (st *HelmState) startBackgroundKubedogTracking( + ctx context.Context, release *ReleaseSpec, helm helmexec.Interface, opts *SyncOpts, +) (h *kubedogTrackingHandle, started bool) { + noop := &kubedogTrackingHandle{ + Helm: helm, + Wait: func() *ReleaseError { return nil }, + Cancel: func() {}, + NotifyHelmDone: func() {}, + FlushBufferedHelmOutput: func() {}, + WasHelmKilled: func() bool { return false }, + } + + if !st.shouldUseKubedog(release, opts) { + return noop, false + } + + useColor := false + if opts != nil { + useColor = opts.Color && !opts.NoColor + } + + // Capture the per-release preamble (header + helm template "Templating + // release=" / "wrote ..." lines + "Found N resources" line) into one + // buffer and emit it via st.logger as a single atomic entry. Without + // this, parallel releases interleave their template output and it + // becomes very hard to follow which `wrote ...` line belongs to which + // chart. Helm output during install/upgrade is buffered separately + // (helmOutputBuf below) and flushed after tracking finishes. + preambleBuf := &bytes.Buffer{} + preambleLogger := helmexec.NewLogger(preambleBuf, "info") + preambleLogger.Infof("\n%s", kubedog.HeaderDividerStyled(fmt.Sprintf("Release '%s'", release.Name), useColor)) + + tmplHelm := helm + if swapper, ok := helm.(helmexec.LoggerSwapper); ok { + tmplHelm = swapper.WithLogger(preambleLogger) + } + + flushPreamble := func() { + if preambleBuf.Len() == 0 { + return + } + st.logger.Infof("%s", strings.TrimRight(preambleBuf.String(), "\n")) + preambleBuf.Reset() + } + + resources, err := st.getReleaseResources(ctx, release, tmplHelm, preambleLogger) + if err != nil { + flushPreamble() + st.logger.Warnf("kubedog: failed to template release %s for parallel tracking, falling back to post-helm tracking: %v", release.Name, err) + return noop, false + } + if len(resources) == 0 { + flushPreamble() + st.logger.Infof("kubedog: no trackable resources templated for release %s", release.Name) + // Emit the same "Helm output for release 'X'" divider tracked + // releases get at flush time, so a hook-only release (which has + // nothing for kubedog to track) still gets the visual delimiter. + // Helm output flows live after this line — there's no buffering + // because we have nothing to interleave with. + st.logger.Infof("\n%s", kubedog.HeaderDividerStyled(fmt.Sprintf("Helm output for release '%s'", release.Name), useColor)) + return noop, true + } + + tracker, trackOpts, err := st.buildReleaseTracker(release, opts, useColor) + if err != nil { + st.logger.Warnf("kubedog: failed to initialize tracker for release %s, falling back to post-helm tracking: %v", release.Name, err) + return noop, false + } + + // Compute the kubedog filter + breakdown summary BEFORE launching the + // tracker goroutine so we can fold the "Tracking N..." and "Tracking + // breakdown: ..." lines into the same atomic preamble flush as the + // header and template output. Doing it any later means those two lines + // race with other releases' progress ticks. PreviewBreakdown is pure — + // no logging, no API calls. + _, breakdown := tracker.PreviewBreakdown(resources) + preambleLogger.Infof("Tracking %d resources from release %s with kubedog (in parallel with helm)", len(resources), release.Name) + if breakdown != "" { + preambleLogger.Infof("Tracking breakdown: %s", breakdown) + } + flushPreamble() + + // Snapshot UID + generation BEFORE handing off to helm so each tracker + // goroutine can wait until the resource actually changes. Without this, + // kubedog observes the pre-upgrade "ready" state and exits immediately. + trackOpts.WithBaselines(tracker.CaptureBaselines(ctx, resources)) + + // Buffer helm output so it doesn't interleave with kubedog progress on + // stdout. We swap the helm logger out for one that writes into the buffer; + // after tracking finishes we replay the buffer through the real logger as + // a single block. bufferHelmOutput also derives a release-scoped context + // so the helm-stuck safety valve can SIGINT helm directly. + bufHelm, helmOutputBuf, releaseCancel := st.bufferHelmOutput(helm, ctx, release.Name) + + trackCtx, trackCancel := context.WithCancel(ctx) + resultCh := make(chan error, 1) + + go func() { + resultCh <- tracker.TrackResources(trackCtx, resources) + }() + + // Two related safety valves run alongside the tracker. Both verify cluster + // state via the live API rather than trusting kubedog's resource graph. + // + // 1. Tracker-race safety valve (always runs): once helm signals success, + // wait a grace period and then poll. If the cluster confirms every + // tracked resource converged but the dyntracker is still wedged + // (kubedog race where a fast Job completion never flips ResourceStatus + // to Ready), cancel the tracker so wait() can return success instead + // of blocking until --track-timeout. + // + // 2. Helm-stuck killer (opt-in via helmStuckGrace): poll alongside helm. + // If the cluster stays converged for helmStuckGrace while helm is + // still running, send SIGINT to helm — that's the helm v4 hook waiter + // wedge (statuswait.go:195 or legacy wait.go:263), recoverable only by + // interrupting the helm subprocess. + helmDoneCh := make(chan struct{}) + var helmDoneOnce sync.Once + var safetyValveTriggered atomic.Bool + var helmKilledByUs atomic.Bool + const ( + safetyValveGrace = 60 * time.Second + safetyValveCheck = 10 * time.Second + safetyValveTrackerWindup = 30 * time.Second + ) + // safetyFiredCh is closed by either safety valve when it triggers, so + // wait() can break out of its resultCh receive without depending on + // dyntracker actually unwinding. Background: in production we hit an + // upstream-kubedog bug where one of N parallel tracker goroutines did + // not respect ctx.Done(), so TrackResources blocked on trackerWg.Wait() + // forever and never sent on resultCh. With only `trackErr := <-resultCh` + // in wait(), helmfile hung indefinitely even though the safety valve + // had already confirmed cluster convergence. + safetyFiredCh := make(chan struct{}) + var safetyFiredOnce sync.Once + signalSafetyFired := func() { + safetyFiredOnce.Do(func() { close(safetyFiredCh) }) + } + // scheduleResultChBackstop runs after a safety valve fires to guarantee + // resultCh is unblocked even if the dyntracker goroutines are wedged. + // resultCh has capacity 1: if the tracker has already (or later) sent + // its real result, our non-blocking send drops harmlessly. If the + // tracker is hung, our nil unblocks wait(). + scheduleResultChBackstop := func() { + go func() { + time.Sleep(safetyValveTrackerWindup) + select { + case resultCh <- nil: + st.logger.Warnf("kubedog: tracker for release %s did not return %s after safety valve fired; pushing nil to unblock wait() (dyntracker likely wedged)", release.Name, safetyValveTrackerWindup) + default: + } + }() + } + helmStuckGrace := st.getHelmStuckGrace(release, opts) + + go func() { + select { + case <-helmDoneCh: + case <-trackCtx.Done(): + return + } + select { + case <-time.After(safetyValveGrace): + case <-trackCtx.Done(): + return + } + ticker := time.NewTicker(safetyValveCheck) + defer ticker.Stop() + for { + if tracker.VerifyAllConverged(trackCtx, resources) { + st.logger.Infof("kubedog: cluster confirms all tracked resources converged for release %s; canceling tracker (worked around dyntracker race)", release.Name) + safetyValveTriggered.Store(true) + trackCancel() + signalSafetyFired() + scheduleResultChBackstop() + return + } + select { + case <-trackCtx.Done(): + return + case <-ticker.C: + } + } + }() + + if helmStuckGrace > 0 { + go func() { + ticker := time.NewTicker(safetyValveCheck) + defer ticker.Stop() + var firstConvergedAt time.Time + for { + select { + case <-trackCtx.Done(): + return + case <-helmDoneCh: + return // helm finished on its own; killer not needed + case <-ticker.C: + } + if !tracker.VerifyAllConverged(trackCtx, resources) { + firstConvergedAt = time.Time{} + continue + } + if firstConvergedAt.IsZero() { + firstConvergedAt = time.Now() + continue + } + if time.Since(firstConvergedAt) < helmStuckGrace { + continue + } + // Cluster has been converged for >= helmStuckGrace while helm + // is still running. Treat as the helm v4 hook waiter wedge + // and send SIGINT via the release-scoped context. Style the + // whole block in bold+yellow so it stands out from the regular + // info chatter in CI logs. + block := fmt.Sprintf("%s\nCluster has confirmed convergence for %s but helm subprocess is still running.\nSending SIGINT to recover from helm v4 hook waiter wedge.\nRelease secret may need manual cleanup: kubectl -n %s delete secret sh.helm.release.v1.%s.", + kubedog.HeaderDivider(fmt.Sprintf("WARNING: Release '%s' — helm-killer fired", release.Name)), + time.Since(firstConvergedAt).Round(time.Second), release.Namespace, release.Name) + st.logger.Warnf("\n%s", kubedog.StyleWarning(block, useColor)) + helmKilledByUs.Store(true) + releaseCancel() + signalSafetyFired() + scheduleResultChBackstop() + return + } + }() + } + + var flushOnce sync.Once + flush := func() { + flushOnce.Do(func() { + payload := strings.TrimSpace(helmOutputBuf.String()) + if payload == "" { + return + } + header := kubedog.HeaderDividerStyled(fmt.Sprintf("Helm output for release '%s'", release.Name), useColor) + st.logger.Infof("\n%s\n%s", header, payload) + }) + } + + canceled := false + wait := func() *ReleaseError { + // Race resultCh against three other paths so a wedged dyntracker + // goroutine that won't return on ctx.Done() cannot pin helmfile + // forever. In production we hit exactly this: one of N parallel + // tracker goroutines ignored cancellation, trackerWg.Wait() never + // completed, resultCh never received, and helmfile hung for 15 + // hours despite the safety valve having confirmed cluster + // convergence minutes earlier. + var trackErr error + hardTimeout := st.getReleaseHardTimeout(release, opts) + hardTimer := time.NewTimer(hardTimeout) + defer hardTimer.Stop() + + select { + case trackErr = <-resultCh: + // Normal path — tracker returned (success, error, or nil + // pushed by a safety-valve backstop). + case <-safetyFiredCh: + // A safety valve fired and already confirmed cluster + // convergence via the live API. Give dyntracker a brief + // grace to wind down on its own; otherwise proceed without + // waiting (cluster is healthy regardless). + select { + case trackErr = <-resultCh: + case <-time.After(safetyValveTrackerWindup): + st.logger.Warnf("kubedog: tracker for release %s did not return %s after safety valve fired; proceeding (cluster confirmed converged)", release.Name, safetyValveTrackerWindup) + } + case <-hardTimer.C: + // Absolute ceiling: neither dyntracker nor any safety valve + // returned within the release timeout. We have no + // confirmation of cluster state, so treat as failure rather + // than silent success. This is the seatbelt for "everything + // else failed" — the user's mental model is that a release + // should not exceed its own timeout. + st.logger.Warnf("kubedog: tracker for release %s did not return within release timeout %s; treating as failure", release.Name, hardTimeout) + trackCancel() + releaseCancel() + flush() + return newReleaseFailedError(release, fmt.Errorf("kubedog tracker did not return within release timeout %s", hardTimeout)) + } + + trackCancel() + releaseCancel() + flush() + if canceled { + // Helm failed and we canceled the tracker; treat as no-op. + return nil + } + if safetyValveTriggered.Load() || helmKilledByUs.Load() { + // We deliberately canceled the tracker after verifying via the + // live API that everything was healthy. The resulting trackErr + // (typically context.Canceled) is not a real failure. + return nil + } + if trackErr != nil { + st.logger.Warnf("kubedog tracking failed for release %s: %v", release.Name, trackErr) + if st.shouldFailOnTrackError(release, opts) { + return newReleaseFailedError(release, trackErr) + } + } + return nil + } + cancel := func() { + canceled = true + trackCancel() + releaseCancel() + } + notifyHelmDone := func() { + tracker.MarkUpstreamCompleted() + helmDoneOnce.Do(func() { close(helmDoneCh) }) + } + wasHelmKilled := func() bool { + return helmKilledByUs.Load() + } + + return &kubedogTrackingHandle{ + Helm: bufHelm, + Wait: wait, + Cancel: cancel, + NotifyHelmDone: notifyHelmDone, + FlushBufferedHelmOutput: flush, + WasHelmKilled: wasHelmKilled, + }, true +} + +// getReleaseHardTimeout returns the absolute ceiling on how long wait() +// should block. Matches the user's mental model of "a release should not +// exceed its own timeout" — the same timeout we pass to helm via --timeout. +// Priority mirrors timeoutFlags(): ops.Timeout > release.Timeout > +// HelmDefaults.Timeout. Falls back to track-timeout when no helm timeout +// is configured, and finally to 10 minutes as a safe hard default. +func (st *HelmState) getReleaseHardTimeout(release *ReleaseSpec, ops *SyncOpts) time.Duration { + if ops != nil && ops.Timeout > 0 { + return time.Duration(ops.Timeout) * time.Second + } + if release.Timeout != nil && *release.Timeout > 0 { + return time.Duration(*release.Timeout) * time.Second + } + if st.HelmDefaults.Timeout > 0 { + return time.Duration(st.HelmDefaults.Timeout) * time.Second + } + if release.TrackTimeout != nil && *release.TrackTimeout > 0 { + return time.Duration(*release.TrackTimeout) * time.Second + } + if ops != nil && ops.TrackTimeout > 0 { + return time.Duration(ops.TrackTimeout) * time.Second + } + return 10 * time.Minute +} + +// getHelmStuckGrace returns the configured helm-stuck grace period — how long +// the cluster must be confirmed converged via the live API while helm is +// still running before we send SIGINT to helm. Zero disables the killer. +// Lookup order: per-release HelmStuckGrace, SyncOpts.HelmStuckGrace, +// HelmDefaults.HelmStuckGrace. +func (st *HelmState) getHelmStuckGrace(release *ReleaseSpec, ops *SyncOpts) time.Duration { + if release.HelmStuckGrace != nil && *release.HelmStuckGrace > 0 { + return time.Duration(*release.HelmStuckGrace) * time.Second + } + if ops != nil && ops.HelmStuckGrace > 0 { + return time.Duration(ops.HelmStuckGrace) * time.Second + } + if st.HelmDefaults.HelmStuckGrace > 0 { + return time.Duration(st.HelmDefaults.HelmStuckGrace) * time.Second + } + return 0 +} + func (st *HelmState) getTrackMode(release *ReleaseSpec, ops *SyncOpts) string { trackMode := release.TrackMode if trackMode == "" && ops != nil && ops.TrackMode != "" { @@ -270,10 +750,6 @@ func (st *HelmState) getTrackMode(release *ReleaseSpec, ops *SyncOpts) string { } func (st *HelmState) appendWaitFlags(flags []string, helm helmexec.Interface, release *ReleaseSpec, ops *SyncOpts) []string { - if st.shouldUseKubedog(release, ops) { - return flags - } - shouldWait := false switch { case release.Wait != nil && *release.Wait: @@ -612,45 +1088,12 @@ func (st *HelmState) PrepareChartify(helm helmexec.Interface, release *ReleaseSp } func (st *HelmState) trackWithKubedog(ctx context.Context, release *ReleaseSpec, helm helmexec.Interface, ops *SyncOpts) error { - timeout := 5 * time.Minute - if release.TrackTimeout != nil && *release.TrackTimeout > 0 { - timeout = time.Duration(*release.TrackTimeout) * time.Second - } else if ops != nil && ops.TrackTimeout > 0 { - timeout = time.Duration(ops.TrackTimeout) * time.Second - } - - trackLogs := release.TrackLogs != nil && *release.TrackLogs - if release.TrackLogs == nil && ops != nil { - trackLogs = ops.TrackLogs - } - - filterConfig := &resource.FilterConfig{ - TrackKinds: release.TrackKinds, - SkipKinds: release.SkipKinds, - TrackResources: convertTrackResources(release.TrackResources), - } - - kubeContext := st.getKubeContext(release) - - trackOpts := kubedog.NewTrackOptions(). - WithTimeout(timeout). - WithLogs(trackLogs). - WithFilterConfig(filterConfig) - - tracker, err := kubedog.NewTracker(&kubedog.TrackerConfig{ - Logger: st.logger, - Namespace: release.Namespace, - KubeContext: kubeContext, - Kubeconfig: st.kubeconfig, - TrackOptions: trackOpts, - KubedogQPS: release.KubedogQPS, - KubedogBurst: release.KubedogBurst, - }) + tracker, _, err := st.buildReleaseTracker(release, ops, false) if err != nil { return fmt.Errorf("failed to create kubedog tracker: %w", err) } - resources, err := st.getReleaseResources(ctx, release, helm) + resources, err := st.getReleaseResources(ctx, release, helm, nil) if err != nil { return fmt.Errorf("failed to get release resources: %w", err) } @@ -661,6 +1104,9 @@ func (st *HelmState) trackWithKubedog(ctx context.Context, release *ReleaseSpec, } st.logger.Infof("Tracking %d resources from release %s with kubedog", len(resources), release.Name) + if _, breakdown := tracker.PreviewBreakdown(resources); breakdown != "" { + st.logger.Infof("Tracking breakdown: %s", breakdown) + } if err := tracker.TrackResources(ctx, resources); err != nil { return fmt.Errorf("kubedog tracking failed for release %s: %w", release.Name, err) @@ -669,7 +1115,15 @@ func (st *HelmState) trackWithKubedog(ctx context.Context, release *ReleaseSpec, return nil } -func (st *HelmState) getReleaseResources(_ context.Context, release *ReleaseSpec, helm helmexec.Interface) ([]*resource.Resource, error) { +// getReleaseResources templates a release and returns its parsed resources. +// outLogger is the logger to use for the user-visible "Found N resources" +// (and No-manifest / No-resources) messages — pass a buffered logger when +// the caller wants to flush all per-release preamble output as one atomic +// block, otherwise pass nil to use st.logger directly. +func (st *HelmState) getReleaseResources(_ context.Context, release *ReleaseSpec, helm helmexec.Interface, outLogger *zap.SugaredLogger) ([]*resource.Resource, error) { + if outLogger == nil { + outLogger = st.logger + } st.logger.Debugf("Getting resources for release %s", release.Name) manifest, namespace, err := st.getReleaseManifest(release, helm) @@ -678,7 +1132,7 @@ func (st *HelmState) getReleaseResources(_ context.Context, release *ReleaseSpec } if len(manifest) == 0 { - st.logger.Infof("No manifest found for release %s", release.Name) + outLogger.Infof("No manifest found for release %s", release.Name) return nil, nil } @@ -693,11 +1147,11 @@ func (st *HelmState) getReleaseResources(_ context.Context, release *ReleaseSpec } if len(resources) == 0 { - st.logger.Infof("No resources found in manifest for release %s", release.Name) + outLogger.Infof("No resources found in manifest for release %s", release.Name) return nil, nil } - st.logger.Infof("Found %d resources in manifest for release %s", len(resources), release.Name) + outLogger.Infof("Found %d resources in manifest for release %s", len(resources), release.Name) result := make([]*resource.Resource, len(resources)) for i := range resources { diff --git a/pkg/state/helmx_test.go b/pkg/state/helmx_test.go index 0c08bef5..6e35b0cb 100644 --- a/pkg/state/helmx_test.go +++ b/pkg/state/helmx_test.go @@ -237,11 +237,24 @@ func TestAppendWaitFlags(t *testing.T) { expected: []string{"--wait"}, }, { - name: "kubedog track mode skips --wait", + // Kubedog tracking does not customize the helm --wait strategy. + // helm runs with its default waiter — wedges are recovered by + // the safety-valve helm-killer in helmx.go, so we don't try to + // pick a "less buggy" wait strategy in advance. This keeps the + // wait-flag logic identical to upstream helmfile. + name: "kubedog track mode with Helm v4 and wait true uses plain --wait", release: &ReleaseSpec{Wait: &[]bool{true}[0], TrackMode: "kubedog"}, syncOpts: nil, helmSpec: HelmSpec{}, helm: testutil.NewVersionHelmExec("4.0.0"), + expected: []string{"--wait"}, + }, + { + name: "kubedog track mode with Helm v4 and explicit wait false skips --wait", + release: &ReleaseSpec{Wait: &[]bool{false}[0], TrackMode: "kubedog"}, + syncOpts: nil, + helmSpec: HelmSpec{Wait: true}, + helm: testutil.NewVersionHelmExec("4.0.0"), expected: []string{}, }, } @@ -573,6 +586,68 @@ func TestFormatLabels(t *testing.T) { } } +func TestGetReleaseHardTimeout(t *testing.T) { + intPtr := func(i int) *int { return &i } + tests := []struct { + name string + state HelmState + release ReleaseSpec + opts *SyncOpts + wantSecs int + }{ + { + name: "ops.Timeout wins over release.Timeout and HelmDefaults.Timeout", + state: HelmState{ReleaseSetSpec: ReleaseSetSpec{HelmDefaults: HelmSpec{Timeout: 100}}}, + release: ReleaseSpec{Timeout: intPtr(200)}, + opts: &SyncOpts{Timeout: 300}, + wantSecs: 300, + }, + { + name: "release.Timeout wins over HelmDefaults.Timeout when ops is unset", + state: HelmState{ReleaseSetSpec: ReleaseSetSpec{HelmDefaults: HelmSpec{Timeout: 100}}}, + release: ReleaseSpec{Timeout: intPtr(200)}, + opts: nil, + wantSecs: 200, + }, + { + name: "HelmDefaults.Timeout used when neither ops nor release set", + state: HelmState{ReleaseSetSpec: ReleaseSetSpec{HelmDefaults: HelmSpec{Timeout: 100}}}, + release: ReleaseSpec{}, + opts: nil, + wantSecs: 100, + }, + { + name: "falls back to release.TrackTimeout when no helm timeout configured", + state: HelmState{}, + release: ReleaseSpec{TrackTimeout: intPtr(450)}, + opts: nil, + wantSecs: 450, + }, + { + name: "falls back to ops.TrackTimeout when no other timeout configured", + state: HelmState{}, + release: ReleaseSpec{}, + opts: &SyncOpts{TrackTimeout: 450}, + wantSecs: 450, + }, + { + name: "10-minute default when nothing configured", + state: HelmState{}, + release: ReleaseSpec{}, + opts: nil, + wantSecs: 600, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.state.getReleaseHardTimeout(&tt.release, tt.opts) + if int(got.Seconds()) != tt.wantSecs { + t.Errorf("getReleaseHardTimeout() = %v, want %ds", got, tt.wantSecs) + } + }) + } +} + func strPtr(s string) *string { return &s } diff --git a/pkg/state/state.go b/pkg/state/state.go index b243a3df..9ce2327a 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -39,6 +39,7 @@ import ( "github.com/helmfile/helmfile/pkg/event" "github.com/helmfile/helmfile/pkg/filesystem" "github.com/helmfile/helmfile/pkg/helmexec" + "github.com/helmfile/helmfile/pkg/kubedog" "github.com/helmfile/helmfile/pkg/maputil" "github.com/helmfile/helmfile/pkg/remote" "github.com/helmfile/helmfile/pkg/tmpl" @@ -254,6 +255,10 @@ type HelmSpec struct { ServerSide *string `yaml:"serverSide,omitempty"` // TrackMode specifies whether to use 'helm' or 'kubedog' for tracking resources TrackMode string `yaml:"trackMode,omitempty"` + // HelmStuckGrace, when > 0, enables the helmfile-side safety-valve + // helm-killer for releases using --track-mode kubedog. See + // ReleaseSpec.HelmStuckGrace for semantics. + HelmStuckGrace int `yaml:"helmStuckGrace,omitempty"` } // RepositorySpec that defines values for a helm repo @@ -487,6 +492,17 @@ type ReleaseSpec struct { TrackTimeout *int `yaml:"trackTimeout,omitempty"` // TrackLogs enables log streaming with kubedog TrackLogs *bool `yaml:"trackLogs,omitempty"` + // TrackFailedLogs streams logs only for pods that enter a failed state + // (CrashLoopBackOff, Error, etc.). Pods that succeed produce no output. + // Has no effect when TrackLogs is true (full streaming wins). + TrackFailedLogs *bool `yaml:"trackFailedLogs,omitempty"` + // HelmStuckGrace, when > 0, enables the safety-valve helm-killer for + // kubedog tracking: if the cluster confirms every tracked resource has + // converged but the helm subprocess is still running, helmfile waits + // this many seconds before sending SIGINT to helm. Targets the helm v4 + // hook waiter wedge that --track-timeout would otherwise resolve only + // after hours. Zero (or absent) disables the killer. + HelmStuckGrace *int `yaml:"helmStuckGrace,omitempty"` // TrackKinds is a whitelist of resource kinds to track TrackKinds []string `yaml:"trackKinds,omitempty"` // SkipKinds is a blacklist of resource kinds to skip tracking @@ -973,8 +989,12 @@ type SyncOpts struct { TrackMode string TrackTimeout int TrackLogs bool + TrackFailedLogs bool + HelmStuckGrace int TrackFailOnError bool Description string + Color bool + NoColor bool } type SyncOpt interface{ Apply(*SyncOpts) } @@ -1204,23 +1224,60 @@ func (st *HelmState) SyncReleases(affectedReleases *AffectedReleases, helm helme relErr = st.trackReleaseIfEnabled(gocontext.Background(), release, helm, opts) } } else { - if err := helm.SyncRelease(context, release.Name, chart, release.Namespace, flags...); err != nil { + trackHandle, trackStarted := st.startBackgroundKubedogTracking(gocontext.Background(), release, helm, opts) + // trackHandle.Helm is a logger-scoped helm clone that + // captures output to an in-memory buffer while tracking is + // active. When tracking isn't running it's the original + // helm — either path is safe to call directly. + releaseHelm := trackHandle.Helm + helmErr := releaseHelm.SyncRelease(context, release.Name, chart, release.Namespace, flags...) + if helmErr != nil && trackStarted && trackHandle.WasHelmKilled() { + // The kubedog safety valve sent SIGINT to helm because + // the cluster confirmed convergence while helm was + // wedged on its hook waiter. Treat the resulting helm + // error as success and proceed via the post-helm-done + // path. + helmErr = nil + } + if helmErr != nil { + if trackStarted { + trackHandle.Cancel() + _ = trackHandle.Wait() + } else { + trackHandle.FlushBufferedHelmOutput() + } m.Lock() affectedReleases.Failed = append(affectedReleases.Failed, release) m.Unlock() - relErr = newReleaseFailedError(release, err) + relErr = newReleaseFailedError(release, helmErr) } else { + if trackStarted { + trackHandle.NotifyHelmDone() + } m.Lock() affectedReleases.Upgraded = append(affectedReleases.Upgraded, release) m.Unlock() - installedVersion, err := st.getDeployedVersion(context, helm, release) - if err != nil { // err is not really impacting so just log it - st.logger.Debugf("getting deployed release version failed: %v", err) - } else { - release.installedVersion = installedVersion + // Skip the deployed-version probe when helm was killed + // by the safety valve — releaseHelm's context is now + // canceled so the probe would just fail noisily, and + // the release secret is likely in pending-install + // state anyway. + if !(trackStarted && trackHandle.WasHelmKilled()) { + installedVersion, err := st.getDeployedVersion(context, releaseHelm, release) + if err != nil { // err is not really impacting so just log it + st.logger.Debugf("getting deployed release version failed: %v", err) + } else { + release.installedVersion = installedVersion + } } - if trackErr := st.trackReleaseIfEnabled(gocontext.Background(), release, helm, opts); trackErr != nil { + var trackErr *ReleaseError + if trackStarted { + trackErr = trackHandle.Wait() + } else { + trackErr = st.trackReleaseIfEnabled(gocontext.Background(), release, helm, opts) + } + if trackErr != nil { m.Lock() affectedReleases.Failed = append(affectedReleases.Failed, release) m.Unlock() @@ -4868,10 +4925,13 @@ func hideChartCredentials(chartCredentials string) (string, error) { return modifiedURL, nil } -// DisplayAffectedReleases logs the upgraded, deleted and in error releases -func (ar *AffectedReleases) DisplayAffectedReleases(logger *zap.SugaredLogger) { +// DisplayAffectedReleases logs the upgraded, deleted and in error releases. +// useColor controls whether the section headers carry the bold+blue style we +// use elsewhere for visual section breaks; pass false (e.g. when --no-color is +// set) for clean output in non-TTY consumers. +func (ar *AffectedReleases) DisplayAffectedReleases(logger *zap.SugaredLogger, useColor bool) { if len(ar.Upgraded) > 0 { - logger.Info("\nUPDATED RELEASES:") + logger.Infof("\n%s", kubedog.HeaderDividerStyled("Updated Releases", useColor)) tbl, _ := prettytable.NewTable(prettytable.Column{Header: "NAME"}, prettytable.Column{Header: "NAMESPACE", MinWidth: 6}, prettytable.Column{Header: "CHART", MinWidth: 6}, @@ -4893,7 +4953,7 @@ func (ar *AffectedReleases) DisplayAffectedReleases(logger *zap.SugaredLogger) { logger.Info(tbl.String()) } if len(ar.Reinstalled) > 0 { - logger.Info("\nREINSTALLED RELEASES:") + logger.Infof("\n%s", kubedog.HeaderDividerStyled("Reinstalled Releases", useColor)) tbl, _ := prettytable.NewTable(prettytable.Column{Header: "NAME"}, prettytable.Column{Header: "NAMESPACE", MinWidth: 6}, prettytable.Column{Header: "CHART", MinWidth: 6}, @@ -4915,7 +4975,7 @@ func (ar *AffectedReleases) DisplayAffectedReleases(logger *zap.SugaredLogger) { logger.Info(tbl.String()) } if len(ar.Deleted) > 0 { - logger.Info("\nDELETED RELEASES:") + logger.Infof("\n%s", kubedog.HeaderDividerStyled("Deleted Releases", useColor)) tbl, _ := prettytable.NewTable(prettytable.Column{Header: "NAME"}, prettytable.Column{Header: "NAMESPACE", MinWidth: 6}, prettytable.Column{Header: "DURATION", AlignRight: true}, @@ -4930,7 +4990,7 @@ func (ar *AffectedReleases) DisplayAffectedReleases(logger *zap.SugaredLogger) { logger.Info(tbl.String()) } if len(ar.Failed) > 0 { - logger.Info("\nFAILED RELEASES:") + logger.Infof("\n%s", kubedog.HeaderDividerStyled("Failed Releases", useColor)) tbl, _ := prettytable.NewTable(prettytable.Column{Header: "NAME"}, prettytable.Column{Header: "NAMESPACE", MinWidth: 6}, prettytable.Column{Header: "CHART", MinWidth: 6}, @@ -4947,7 +5007,7 @@ func (ar *AffectedReleases) DisplayAffectedReleases(logger *zap.SugaredLogger) { logger.Info(tbl.String()) } if len(ar.DeleteFailed) > 0 { - logger.Info("\nFAILED TO DELETE RELEASES:") + logger.Infof("\n%s", kubedog.HeaderDividerStyled("Failed to Delete Releases", useColor)) tbl, _ := prettytable.NewTable(prettytable.Column{Header: "NAME"}, prettytable.Column{Header: "NAMESPACE", MinWidth: 6}, prettytable.Column{Header: "DURATION", AlignRight: true}, diff --git a/pkg/state/temp_test.go b/pkg/state/temp_test.go index 44f23d68..312340f6 100644 --- a/pkg/state/temp_test.go +++ b/pkg/state/temp_test.go @@ -38,39 +38,39 @@ func TestGenerateID(t *testing.T) { run(testcase{ subject: "baseline", release: ReleaseSpec{Name: "foo", Chart: "incubator/raw"}, - want: "foo-values-56c9b878cf", + want: "foo-values-6fcd6fc479", }) run(testcase{ subject: "different bytes content", release: ReleaseSpec{Name: "foo", Chart: "incubator/raw"}, data: []byte(`{"k":"v"}`), - want: "foo-values-55d46b7dd4", + want: "foo-values-6556658b7b", }) run(testcase{ subject: "different map content", release: ReleaseSpec{Name: "foo", Chart: "incubator/raw"}, data: map[string]any{"k": "v"}, - want: "foo-values-8c574f646", + want: "foo-values-fd86f44b", }) run(testcase{ subject: "different chart", release: ReleaseSpec{Name: "foo", Chart: "stable/envoy"}, - want: "foo-values-664cdfb57d", + want: "foo-values-58bdc49774", }) run(testcase{ subject: "different name", release: ReleaseSpec{Name: "bar", Chart: "incubator/raw"}, - want: "bar-values-fbb5fd45b", + want: "bar-values-856b998888", }) run(testcase{ subject: "specific ns", release: ReleaseSpec{Name: "foo", Chart: "incubator/raw", Namespace: "myns"}, - want: "myns-foo-values-7db84f99b", + want: "myns-foo-values-7bdfd95fbd", }) for id, n := range ids { diff --git a/test/e2e/template/helmfile/testdata/snapshot/chart_need/output-helm4.yaml b/test/e2e/template/helmfile/testdata/snapshot/chart_need/output-helm4.yaml index ce3f9ba5..1617bb05 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/chart_need/output-helm4.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/chart_need/output-helm4.yaml @@ -1,6 +1,5 @@ Adding repo myrepo http://localhost:18080/ "myrepo" has been added to your repositories - Building dependency release=foo, chart=$WD/temp1/foo Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "myrepo" chart repository @@ -8,7 +7,6 @@ Update Complete. ⎈Happy Helming!⎈ Saving 1 charts Downloading raw from repo http://localhost:18080/ Deleting outdated charts - Templating release=foo, chart=$WD/temp1/foo --- # Source: raw/charts/dep/templates/resources.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/chart_need/output.yaml b/test/e2e/template/helmfile/testdata/snapshot/chart_need/output.yaml index 7954f3c1..ca1e540a 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/chart_need/output.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/chart_need/output.yaml @@ -1,6 +1,5 @@ Adding repo myrepo http://localhost:18080/ "myrepo" has been added to your repositories - Building dependency release=foo, chart=$WD/temp1/foo Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "myrepo" chart repository @@ -8,7 +7,6 @@ Update Complete. ⎈Happy Helming!⎈ Saving 1 charts Downloading raw from repo http://localhost:18080/ Deleting outdated charts - Templating release=foo, chart=$WD/temp1/foo --- # Source: raw/charts/dep/templates/resources.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/output-helm4.yaml b/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/output-helm4.yaml index ee97997b..4d61987e 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/output-helm4.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/output-helm4.yaml @@ -1,7 +1,6 @@ Live output is enabled Adding repo myrepo http://localhost:18081/ "myrepo" has been added to your repositories - Building dependency release=foo, chart=$WD/temp1/foo Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "myrepo" chart repository @@ -9,7 +8,6 @@ Update Complete. ⎈Happy Helming!⎈ Saving 1 charts Downloading raw from repo http://localhost:18081/ Deleting outdated charts - Templating release=foo, chart=$WD/temp1/foo --- # Source: raw/charts/dep/templates/resources.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/output.yaml b/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/output.yaml index 67616a32..2bda6e6b 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/output.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/chart_need_enable_live_output/output.yaml @@ -1,7 +1,6 @@ Live output is enabled Adding repo myrepo http://localhost:18081/ "myrepo" has been added to your repositories - Building dependency release=foo, chart=$WD/temp1/foo Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "myrepo" chart repository @@ -9,7 +8,6 @@ Update Complete. ⎈Happy Helming!⎈ Saving 1 charts Downloading raw from repo http://localhost:18081/ Deleting outdated charts - Templating release=foo, chart=$WD/temp1/foo --- # Source: raw/charts/dep/templates/resources.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/issue_1229_template_strategic_merge_with_lockfile/output.yaml b/test/e2e/template/helmfile/testdata/snapshot/issue_1229_template_strategic_merge_with_lockfile/output.yaml index d5d7aa26..4e2c06ed 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/issue_1229_template_strategic_merge_with_lockfile/output.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/issue_1229_template_strategic_merge_with_lockfile/output.yaml @@ -1,6 +1,5 @@ Adding repo myrepo http://localhost:18083/ "myrepo" has been added to your repositories - Templating release=raw, chart=$WD/temp1/raw/raw --- # Source: raw/templates/patched_resources.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/issue_473_oci_chart_url_fetch/output-helm4.yaml b/test/e2e/template/helmfile/testdata/snapshot/issue_473_oci_chart_url_fetch/output-helm4.yaml index c668d7c5..3202b88a 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/issue_473_oci_chart_url_fetch/output-helm4.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/issue_473_oci_chart_url_fetch/output-helm4.yaml @@ -1,4 +1,3 @@ Pulling localhost:$REGISTRY_PORT/myrepo/raw:0.1.0 Pulled: localhost:$REGISTRY_PORT/myrepo/raw:0.1.0 Digest: sha256:$DIGEST - diff --git a/test/e2e/template/helmfile/testdata/snapshot/issue_498_template_go_getter_with_selector/output.yaml b/test/e2e/template/helmfile/testdata/snapshot/issue_498_template_go_getter_with_selector/output.yaml index c5792331..adeb8133 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/issue_498_template_go_getter_with_selector/output.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/issue_498_template_go_getter_with_selector/output.yaml @@ -1,6 +1,5 @@ Adding repo istio https://istio-release.storage.googleapis.com/charts "istio" has been added to your repositories - Templating release=acme-jx, chart=$GoGetterPath diff --git a/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull/output-helm4.yaml b/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull/output-helm4.yaml index b4d00a8f..ddc37456 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull/output-helm4.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull/output-helm4.yaml @@ -1,7 +1,6 @@ Pulling localhost:$REGISTRY_PORT/myrepo/raw:0.1.0 Pulled: localhost:$REGISTRY_PORT/myrepo/raw:0.1.0 Digest: sha256:$DIGEST - Templating release=foo, chart=$HELMFILE_CACHE_HOME/myrepo/raw/0.1.0/raw --- # Source: raw/templates/resources.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull_direct/output-helm4.yaml b/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull_direct/output-helm4.yaml index 5ee316fc..b0458d92 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull_direct/output-helm4.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull_direct/output-helm4.yaml @@ -1,7 +1,6 @@ Pulling localhost:$REGISTRY_PORT/myrepo/raw:0.1.0 Pulled: localhost:$REGISTRY_PORT/myrepo/raw:0.1.0 Digest: sha256:$DIGEST - Templating release=foo, chart=$HELMFILE_CACHE_HOME/oci__localhost_$REGISTRY_PORT/myrepo/raw/0.1.0/raw --- # Source: raw/templates/resources.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull_once/output-helm4.yaml b/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull_once/output-helm4.yaml index eac963da..8e4acf53 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull_once/output-helm4.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull_once/output-helm4.yaml @@ -1,11 +1,9 @@ Pulling localhost:$REGISTRY_PORT/myrepo/raw:0.1.0 Pulled: localhost:$REGISTRY_PORT/myrepo/raw:0.1.0 Digest: sha256:$DIGEST - Pulling localhost:$REGISTRY_PORT/myrepo/raw:0.0.1 Pulled: localhost:$REGISTRY_PORT/myrepo/raw:0.0.1 Digest: sha256:$DIGEST - Templating release=foo, chart=$HELMFILE_CACHE_HOME/myrepo/raw/0.1.0/raw --- # Source: raw/templates/resources.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull_once2/output-helm4.yaml b/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull_once2/output-helm4.yaml index 7074ea9b..4b7557eb 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull_once2/output-helm4.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/oci_chart_pull_once2/output-helm4.yaml @@ -1,7 +1,6 @@ Pulling localhost:$REGISTRY_PORT/myrepo/raw:0.1.0 Pulled: localhost:$REGISTRY_PORT/myrepo/raw:0.1.0 Digest: sha256:$DIGEST - Templating release=release-0, chart=$HELMFILE_CACHE_HOME/myrepo/raw/0.1.0/raw --- # Source: raw/templates/resources.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/oci_need/output-helm4.yaml b/test/e2e/template/helmfile/testdata/snapshot/oci_need/output-helm4.yaml index f04e334b..a2f682e6 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/oci_need/output-helm4.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/oci_need/output-helm4.yaml @@ -4,7 +4,6 @@ Downloading raw from repo oci://localhost:$REGISTRY_PORT/myrepo Pulled: localhost:$REGISTRY_PORT/myrepo/raw:0.1.0 Digest: sha256:$DIGEST Deleting outdated charts - Templating release=foo, chart=$WD/temp1/foo --- # Source: raw/charts/dep/templates/resources.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/oci_need/output.yaml b/test/e2e/template/helmfile/testdata/snapshot/oci_need/output.yaml index 3c5796df..914b3ef0 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/oci_need/output.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/oci_need/output.yaml @@ -2,7 +2,6 @@ Building dependency release=foo, chart=$WD/temp1/foo Saving 1 charts Downloading raw from repo oci://localhost:$REGISTRY_PORT/myrepo Deleting outdated charts - Templating release=foo, chart=$WD/temp1/foo --- # Source: raw/charts/dep/templates/resources.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/postrenderer/output-helm4.yaml b/test/e2e/template/helmfile/testdata/snapshot/postrenderer/output-helm4.yaml index ba813e5c..0dbbe4b5 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/postrenderer/output-helm4.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/postrenderer/output-helm4.yaml @@ -5,7 +5,6 @@ Downloading raw from repo oci://localhost:$REGISTRY_PORT/myrepo Pulled: localhost:$REGISTRY_PORT/myrepo/raw:0.1.0 Digest: sha256:$DIGEST Deleting outdated charts - Templating release=foo, chart=$WD/testdata/charts/raw-0.1.0 --- # Source: generated-by-postrender-1.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/postrenderer/output.yaml b/test/e2e/template/helmfile/testdata/snapshot/postrenderer/output.yaml index d661ef51..91d81888 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/postrenderer/output.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/postrenderer/output.yaml @@ -3,7 +3,6 @@ Building dependency release=baz, chart=$WD/temp1/baz Saving 1 charts Downloading raw from repo oci://localhost:$REGISTRY_PORT/myrepo Deleting outdated charts - Templating release=foo, chart=$WD/testdata/charts/raw-0.1.0 --- # Source: raw/templates/resources.yaml diff --git a/test/e2e/template/helmfile/testdata/snapshot/release_template_inheritance/output.yaml b/test/e2e/template/helmfile/testdata/snapshot/release_template_inheritance/output.yaml index 8f4523d9..12f887a9 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/release_template_inheritance/output.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/release_template_inheritance/output.yaml @@ -1,6 +1,5 @@ Adding repo myrepo http://localhost:18084/ "myrepo" has been added to your repositories - Building dependency release=foo1, chart=../../charts/raw-0.1.0 Building dependency release=foo2, chart=../../charts/raw-0.1.0 Templating release=foo1, chart=../../charts/raw-0.1.0 diff --git a/test/e2e/template/helmfile/testdata/snapshot/templated_lockfile/output.yaml b/test/e2e/template/helmfile/testdata/snapshot/templated_lockfile/output.yaml index 18cd584e..540c5d1b 100644 --- a/test/e2e/template/helmfile/testdata/snapshot/templated_lockfile/output.yaml +++ b/test/e2e/template/helmfile/testdata/snapshot/templated_lockfile/output.yaml @@ -1,6 +1,5 @@ Adding repo myrepo http://localhost:18082/ "myrepo" has been added to your repositories - Templating release=raw, chart=myrepo/raw --- # Source: raw/templates/resources.yaml diff --git a/test/integration/test-cases/diff-args/output/apply-live-stderr b/test/integration/test-cases/diff-args/output/apply-live-stderr index 7f789424..c6e0e9be 100644 --- a/test/integration/test-cases/diff-args/output/apply-live-stderr +++ b/test/integration/test-cases/diff-args/output/apply-live-stderr @@ -4,7 +4,6 @@ Listing releases matching ^uninstalled$ Upgrading release=installed, chart=../../../charts/httpbin, namespace=helmfile-tests Listing releases matching ^installed$ - -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION diff --git a/test/integration/test-cases/diff-args/output/apply-live-stderr-helm4 b/test/integration/test-cases/diff-args/output/apply-live-stderr-helm4 index 7f789424..c6e0e9be 100644 --- a/test/integration/test-cases/diff-args/output/apply-live-stderr-helm4 +++ b/test/integration/test-cases/diff-args/output/apply-live-stderr-helm4 @@ -4,7 +4,6 @@ Listing releases matching ^uninstalled$ Upgrading release=installed, chart=../../../charts/httpbin, namespace=helmfile-tests Listing releases matching ^installed$ - -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION diff --git a/test/integration/test-cases/diff-args/output/apply-stderr b/test/integration/test-cases/diff-args/output/apply-stderr index 6a5025e3..32d71227 100644 --- a/test/integration/test-cases/diff-args/output/apply-stderr +++ b/test/integration/test-cases/diff-args/output/apply-stderr @@ -7,10 +7,8 @@ NAMESPACE: helmfile-tests STATUS: deployed REVISION: 1 TEST SUITE: None - Listing releases matching ^installed$ - -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION diff --git a/test/integration/test-cases/diff-args/output/apply-stderr-helm4 b/test/integration/test-cases/diff-args/output/apply-stderr-helm4 index 088d699a..e73cae61 100644 --- a/test/integration/test-cases/diff-args/output/apply-stderr-helm4 +++ b/test/integration/test-cases/diff-args/output/apply-stderr-helm4 @@ -8,10 +8,8 @@ STATUS: deployed REVISION: 1 DESCRIPTION: Install complete TEST SUITE: None - Listing releases matching ^installed$ - -UPDATED RELEASES: +========== Updated Releases ========== NAME NAMESPACE CHART VERSION DURATION diff --git a/test/integration/test-cases/v1-subhelmfile-multi-bases-with-array-values/output/result b/test/integration/test-cases/v1-subhelmfile-multi-bases-with-array-values/output/result index a8869435..472af46b 100644 --- a/test/integration/test-cases/v1-subhelmfile-multi-bases-with-array-values/output/result +++ b/test/integration/test-cases/v1-subhelmfile-multi-bases-with-array-values/output/result @@ -1,6 +1,5 @@ Adding repo incubator https://charts.helm.sh/incubator/ "incubator" has been added to your repositories - Templating release=helmfile-test, chart=incubator/raw --- # Source: raw/templates/resources.yaml diff --git a/test/integration/test-cases/v1-subhelmfile-multi-bases-with-array-values/output/result-live b/test/integration/test-cases/v1-subhelmfile-multi-bases-with-array-values/output/result-live index ba42d6c6..8bd30d55 100644 --- a/test/integration/test-cases/v1-subhelmfile-multi-bases-with-array-values/output/result-live +++ b/test/integration/test-cases/v1-subhelmfile-multi-bases-with-array-values/output/result-live @@ -1,7 +1,6 @@ Live output is enabled Adding repo incubator https://charts.helm.sh/incubator/ "incubator" has been added to your repositories - Templating release=helmfile-test, chart=incubator/raw --- # Source: raw/templates/resources.yaml