mirror of
https://github.com/helmfile/helmfile.git
synced 2026-10-19 09:44:35 +02:00
add trackLogsUntilReady flag (#2814)
* add trackLogsUntilReady flag Signed-off-by: pznamensky <kompastver@gmail.com> * refactor(kubedog): flatten nested conditionals in trackLogsUntilReady filter - podLogFilter.update: replace nested if-found/if-uid block with a single combined guard plus a no-op delete - podLogFilter.watch: extract cutoffTrackedKind predicate and watchNamespacePods helper; use RWTransactionErr (kubedog's own idiom) instead of capturing err in plain RWTransaction closures, removing the 'if err == nil { inf.Run() }' nesting - TrackResources: extract newStartupLogFilter guard-clause helper so the printer wiring stays flat - add table tests for the new helpers No behavior change; race tests and golangci-lint pass. Signed-off-by: yxxhero <aiopsclub@163.com> --------- Signed-off-by: pznamensky <kompastver@gmail.com> Signed-off-by: yxxhero <aiopsclub@163.com> Co-authored-by: yxxhero <aiopsclub@163.com>
This commit is contained in:
@@ -2124,6 +2124,7 @@ Do you really want to apply?
|
||||
TrackMode: c.TrackMode(),
|
||||
TrackTimeout: c.TrackTimeout(),
|
||||
TrackLogs: c.TrackLogs(),
|
||||
TrackLogsUntilReady: c.TrackLogsUntilReady(),
|
||||
TrackFailedLogs: c.TrackFailedLogs(),
|
||||
TrackLogsInterval: c.TrackLogsInterval(),
|
||||
HelmStuckGrace: c.HelmStuckGrace(),
|
||||
@@ -2622,6 +2623,7 @@ Do you really want to sync?
|
||||
TrackMode: c.TrackMode(),
|
||||
TrackTimeout: c.TrackTimeout(),
|
||||
TrackLogs: c.TrackLogs(),
|
||||
TrackLogsUntilReady: c.TrackLogsUntilReady(),
|
||||
TrackFailedLogs: c.TrackFailedLogs(),
|
||||
TrackLogsInterval: c.TrackLogsInterval(),
|
||||
HelmStuckGrace: c.HelmStuckGrace(),
|
||||
|
||||
@@ -2557,6 +2557,7 @@ type applyConfig struct {
|
||||
trackMode string
|
||||
trackTimeout int
|
||||
trackLogs bool
|
||||
trackLogsUntilReady bool
|
||||
trackLogsInterval time.Duration
|
||||
trackFailOnError bool
|
||||
|
||||
@@ -2802,6 +2803,10 @@ func (a applyConfig) TrackLogs() bool {
|
||||
return a.trackLogs
|
||||
}
|
||||
|
||||
func (a applyConfig) TrackLogsUntilReady() bool {
|
||||
return a.trackLogsUntilReady
|
||||
}
|
||||
|
||||
func (a applyConfig) TrackFailedLogs() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -103,6 +103,7 @@ type ApplyConfigProvider interface {
|
||||
TrackMode() string
|
||||
TrackTimeout() int
|
||||
TrackLogs() bool
|
||||
TrackLogsUntilReady() bool
|
||||
TrackFailedLogs() bool
|
||||
TrackLogsInterval() time.Duration
|
||||
HelmStuckGrace() int
|
||||
@@ -148,6 +149,7 @@ type SyncConfigProvider interface {
|
||||
TrackMode() string
|
||||
TrackTimeout() int
|
||||
TrackLogs() bool
|
||||
TrackLogsUntilReady() bool
|
||||
TrackFailedLogs() bool
|
||||
TrackLogsInterval() time.Duration
|
||||
HelmStuckGrace() int
|
||||
|
||||
@@ -94,6 +94,8 @@ type ApplyOptions struct {
|
||||
TrackTimeout int
|
||||
// TrackLogs enables log streaming with kubedog
|
||||
TrackLogs bool
|
||||
// TrackLogsUntilReady stops pod log streaming when each pod becomes ready. Requires TrackLogs.
|
||||
TrackLogsUntilReady bool
|
||||
// TrackFailedLogs streams logs only for pods that enter a failed state.
|
||||
TrackFailedLogs bool
|
||||
// TrackLogsInterval is the interval between kubedog log output updates.
|
||||
@@ -344,6 +346,11 @@ func (a *ApplyImpl) TrackLogs() bool {
|
||||
return a.ApplyOptions.TrackLogs
|
||||
}
|
||||
|
||||
// TrackLogsUntilReady returns the track-logs-until-ready flag.
|
||||
func (a *ApplyImpl) TrackLogsUntilReady() bool {
|
||||
return a.ApplyOptions.TrackLogsUntilReady
|
||||
}
|
||||
|
||||
// TrackFailedLogs returns the track-failed-logs flag.
|
||||
func (a *ApplyImpl) TrackFailedLogs() bool {
|
||||
return a.ApplyOptions.TrackFailedLogs
|
||||
|
||||
@@ -67,6 +67,8 @@ type SyncOptions struct {
|
||||
TrackTimeout int
|
||||
// TrackLogs enables log streaming with kubedog
|
||||
TrackLogs bool
|
||||
// TrackLogsUntilReady stops pod log streaming when each pod becomes ready. Requires TrackLogs.
|
||||
TrackLogsUntilReady bool
|
||||
// TrackFailedLogs streams logs only for pods that enter a failed state.
|
||||
TrackFailedLogs bool
|
||||
// TrackLogsInterval is the interval between kubedog log output updates.
|
||||
@@ -257,6 +259,11 @@ func (t *SyncImpl) TrackLogs() bool {
|
||||
return t.SyncOptions.TrackLogs
|
||||
}
|
||||
|
||||
// TrackLogsUntilReady returns the track-logs-until-ready flag.
|
||||
func (t *SyncImpl) TrackLogsUntilReady() bool {
|
||||
return t.SyncOptions.TrackLogsUntilReady
|
||||
}
|
||||
|
||||
// TrackFailedLogs returns the track-failed-logs flag.
|
||||
func (t *SyncImpl) TrackFailedLogs() bool {
|
||||
return t.SyncOptions.TrackFailedLogs
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
package kubedog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/werf/kubedog/pkg/informer"
|
||||
kdutil "github.com/werf/kubedog/pkg/trackers/dyntracker/util"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
)
|
||||
|
||||
type podLogCutoff struct {
|
||||
uid types.UID
|
||||
time time.Time
|
||||
}
|
||||
|
||||
// podLogFilter remembers the first Ready transition independently of kubedog's
|
||||
// workload readiness. It only filters output; pod tracking and log capture continue.
|
||||
type podLogFilter struct {
|
||||
mu sync.RWMutex
|
||||
cutoffs map[string]podLogCutoff
|
||||
}
|
||||
|
||||
func newPodLogFilter() *podLogFilter {
|
||||
return &podLogFilter{cutoffs: make(map[string]podLogCutoff)}
|
||||
}
|
||||
|
||||
// newStartupLogFilter returns a readiness-tracking log filter when log
|
||||
// streaming is limited to startup output, or nil otherwise.
|
||||
func newStartupLogFilter(options *TrackOptions, factory *kdutil.Concurrent[*informer.InformerFactory], targets []trackTarget) (*podLogFilter, error) {
|
||||
if options == nil || !options.Logs || !options.LogsUntilReady {
|
||||
return nil, nil
|
||||
}
|
||||
filter := newPodLogFilter()
|
||||
if err := filter.watch(factory, targets); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return filter, nil
|
||||
}
|
||||
|
||||
// cutoffTrackedKind reports whether pods of the target kind are subject to
|
||||
// readiness-based log cutoff. Jobs are excluded: they can be Ready while
|
||||
// still executing.
|
||||
func cutoffTrackedKind(kind string) bool {
|
||||
switch kind {
|
||||
case "deploy", "sts", "ds":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// watchNamespacePods attaches the readiness handler to the shared pod
|
||||
// informer for namespace and starts it.
|
||||
func (f *podLogFilter) watchNamespacePods(factory *kdutil.Concurrent[*informer.InformerFactory], gvr schema.GroupVersionResource, namespace string) error {
|
||||
var podInformer *kdutil.Concurrent[*informer.Informer]
|
||||
err := factory.RWTransactionErr(func(factory *informer.InformerFactory) error {
|
||||
informer, err := factory.ForNamespace(gvr, namespace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create pod log informer: %w", err)
|
||||
}
|
||||
podInformer = informer
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return podInformer.RWTransactionErr(func(inf *informer.Informer) error {
|
||||
_, err := inf.AddEventHandler(cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: f.update,
|
||||
UpdateFunc: func(_, obj any) {
|
||||
f.update(obj)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("watch pod readiness for logs: %w", err)
|
||||
}
|
||||
inf.Run()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (f *podLogFilter) watch(factory *kdutil.Concurrent[*informer.InformerFactory], targets []trackTarget) error {
|
||||
podGVR := schema.GroupVersionResource{Version: "v1", Resource: "pods"}
|
||||
namespaces := make(map[string]struct{})
|
||||
for _, target := range targets {
|
||||
if !cutoffTrackedKind(target.kind) {
|
||||
continue
|
||||
}
|
||||
if _, watched := namespaces[target.namespace]; watched {
|
||||
continue
|
||||
}
|
||||
if err := f.watchNamespacePods(factory, podGVR, target.namespace); err != nil {
|
||||
return err
|
||||
}
|
||||
namespaces[target.namespace] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *podLogFilter) update(obj any) {
|
||||
pod, ok := obj.(*unstructured.Unstructured)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
id := kdutil.ResourceID(pod.GetName(), pod.GetNamespace(), watchdogPodGVK)
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
// Keep the first Ready transition recorded for this pod instance. A
|
||||
// replacement pod reusing the name starts fresh; delete on a missing key
|
||||
// is a no-op.
|
||||
if cutoff, found := f.cutoffs[id]; found && cutoff.uid == pod.GetUID() {
|
||||
return
|
||||
}
|
||||
delete(f.cutoffs, id)
|
||||
// Job pods can be Ready while still executing; keep their complete output.
|
||||
for _, owner := range pod.GetOwnerReferences() {
|
||||
if owner.Kind == "Job" {
|
||||
return
|
||||
}
|
||||
}
|
||||
conditions, _, err := unstructured.NestedSlice(pod.Object, "status", "conditions")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, raw := range conditions {
|
||||
condition, ok := raw.(map[string]any)
|
||||
if !ok || condition["type"] != "Ready" || condition["status"] != "True" {
|
||||
continue
|
||||
}
|
||||
timestamp, _ := condition["lastTransitionTime"].(string)
|
||||
readyTime, err := time.Parse(time.RFC3339Nano, timestamp)
|
||||
if err != nil {
|
||||
readyTime = time.Now()
|
||||
} else if readyTime.Nanosecond() == 0 {
|
||||
// Kubernetes rounds Ready timestamps to seconds. Preserve startup
|
||||
// lines within that second too, even when delivered after readiness.
|
||||
readyTime = readyTime.Add(time.Second - time.Nanosecond)
|
||||
}
|
||||
f.cutoffs[id] = podLogCutoff{uid: pod.GetUID(), time: readyTime}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (f *podLogFilter) cutoff(name, namespace string, gvk schema.GroupVersionKind) time.Time {
|
||||
if f == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
f.mu.RLock()
|
||||
defer f.mu.RUnlock()
|
||||
return f.cutoffs[kdutil.ResourceID(name, namespace, gvk)].time
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package kubedog
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/werf/kubedog/pkg/informer"
|
||||
kdutil "github.com/werf/kubedog/pkg/trackers/dyntracker/util"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/dynamic/fake"
|
||||
clienttesting "k8s.io/client-go/testing"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
)
|
||||
|
||||
func newLogFilterPod(name, namespace, uid, readyStatus string, readyTime time.Time) *unstructured.Unstructured {
|
||||
pod := &unstructured.Unstructured{Object: map[string]any{
|
||||
"apiVersion": "v1",
|
||||
"kind": "Pod",
|
||||
"metadata": map[string]any{"name": name, "namespace": namespace, "uid": uid},
|
||||
"status": map[string]any{"phase": "Running"},
|
||||
}}
|
||||
if readyStatus != "" {
|
||||
condition := map[string]any{"type": "Ready", "status": readyStatus}
|
||||
if !readyTime.IsZero() {
|
||||
condition["lastTransitionTime"] = readyTime.Format(time.RFC3339Nano)
|
||||
}
|
||||
pod.Object["status"].(map[string]any)["conditions"] = []any{condition}
|
||||
}
|
||||
return pod
|
||||
}
|
||||
|
||||
func TestPodLogFilter_ReadyCondition(t *testing.T) {
|
||||
readyTime := time.Date(2026, 10, 1, 12, 0, 0, 123, time.UTC)
|
||||
for _, status := range []string{"", "False", "Unknown", "True"} {
|
||||
t.Run("Ready="+status, func(t *testing.T) {
|
||||
filter := newPodLogFilter()
|
||||
filter.update(newLogFilterPod("app", "ns", "uid", status, readyTime))
|
||||
cutoff := filter.cutoff("app", "ns", watchdogPodGVK)
|
||||
if status == "True" {
|
||||
assert.Equal(t, readyTime, cutoff)
|
||||
} else {
|
||||
assert.True(t, cutoff.IsZero(), "Running alone does not mean that a pod is ready")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodLogFilter_FirstReadyTransitionIsRetained(t *testing.T) {
|
||||
filter := newPodLogFilter()
|
||||
firstReady := time.Date(2026, 10, 1, 12, 0, 0, 123, time.UTC)
|
||||
filter.update(newLogFilterPod("app", "ns", "uid", "True", firstReady))
|
||||
filter.update(newLogFilterPod("app", "ns", "uid", "False", firstReady.Add(time.Minute)))
|
||||
filter.update(newLogFilterPod("app", "ns", "uid", "True", firstReady.Add(2*time.Minute)))
|
||||
assert.Equal(t, firstReady, filter.cutoff("app", "ns", watchdogPodGVK))
|
||||
}
|
||||
|
||||
func TestPodLogFilter_JobPodsAreExempt(t *testing.T) {
|
||||
filter := newPodLogFilter()
|
||||
pod := newLogFilterPod("job-pod", "ns", "uid", "True", time.Now())
|
||||
pod.SetOwnerReferences([]metav1.OwnerReference{{APIVersion: "batch/v1", Kind: "Job", Name: "job"}})
|
||||
filter.update(pod)
|
||||
assert.True(t, filter.cutoff("job-pod", "ns", watchdogPodGVK).IsZero())
|
||||
}
|
||||
|
||||
func TestPodLogFilter_ReplacementPodStartsLoggingAgain(t *testing.T) {
|
||||
filter := newPodLogFilter()
|
||||
firstReady := time.Date(2026, 10, 1, 12, 0, 0, 123, time.UTC)
|
||||
filter.update(newLogFilterPod("app", "ns", "old-uid", "True", firstReady))
|
||||
filter.update(newLogFilterPod("app", "ns", "new-uid", "False", time.Time{}))
|
||||
assert.True(t, filter.cutoff("app", "ns", watchdogPodGVK).IsZero())
|
||||
|
||||
nextReady := firstReady.Add(time.Minute)
|
||||
filter.update(newLogFilterPod("app", "ns", "new-uid", "True", nextReady))
|
||||
assert.Equal(t, nextReady, filter.cutoff("app", "ns", watchdogPodGVK))
|
||||
}
|
||||
|
||||
func TestPodLogFilter_NamespacesAreIndependent(t *testing.T) {
|
||||
filter := newPodLogFilter()
|
||||
readyTime := time.Date(2026, 10, 1, 12, 0, 0, 123, time.UTC)
|
||||
filter.update(newLogFilterPod("app", "ready-ns", "uid", "True", readyTime))
|
||||
filter.update(newLogFilterPod("app", "starting-ns", "uid", "False", time.Time{}))
|
||||
assert.Equal(t, readyTime, filter.cutoff("app", "ready-ns", watchdogPodGVK))
|
||||
assert.True(t, filter.cutoff("app", "starting-ns", watchdogPodGVK).IsZero())
|
||||
}
|
||||
|
||||
func TestPodLogFilter_PreservesReadinessSecond(t *testing.T) {
|
||||
filter := newPodLogFilter()
|
||||
readyTime := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
|
||||
filter.update(newLogFilterPod("app", "ns", "uid", "True", readyTime))
|
||||
assert.Equal(t, readyTime.Add(time.Second-time.Nanosecond), filter.cutoff("app", "ns", watchdogPodGVK))
|
||||
}
|
||||
|
||||
func TestPodLogFilter_MissingOrInvalidTransitionTime(t *testing.T) {
|
||||
for _, timestamp := range []string{"", "invalid"} {
|
||||
t.Run(timestamp, func(t *testing.T) {
|
||||
filter := newPodLogFilter()
|
||||
pod := newLogFilterPod("app", "ns", "uid", "True", time.Time{})
|
||||
conditions := pod.Object["status"].(map[string]any)["conditions"].([]any)
|
||||
conditions[0].(map[string]any)["lastTransitionTime"] = timestamp
|
||||
before := time.Now()
|
||||
filter.update(pod)
|
||||
after := time.Now()
|
||||
cutoff := filter.cutoff("app", "ns", watchdogPodGVK)
|
||||
assert.False(t, cutoff.Before(before))
|
||||
assert.False(t, cutoff.After(after))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodLogFilter_WatchesExistingPodsAndUpdatesUsingSharedInformer(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
podGVR := schema.GroupVersionResource{Version: "v1", Resource: "pods"}
|
||||
readyTime := time.Date(2026, 10, 1, 12, 0, 0, 123, time.UTC)
|
||||
readyPod := newLogFilterPod("ready", "ns", "ready-uid", "True", readyTime)
|
||||
startingPod := newLogFilterPod("starting", "ns", "starting-uid", "False", time.Time{})
|
||||
client := fake.NewSimpleDynamicClientWithCustomListKinds(
|
||||
runtime.NewScheme(), map[schema.GroupVersionResource]string{podGVR: "PodList"}, readyPod, startingPod,
|
||||
)
|
||||
var watches atomic.Int32
|
||||
client.PrependWatchReactor("pods", func(clienttesting.Action) (bool, watch.Interface, error) {
|
||||
watches.Add(1)
|
||||
return false, nil, nil
|
||||
})
|
||||
factory := informer.NewConcurrentInformerFactory(ctx.Done(), make(chan error, 1), client,
|
||||
informer.ConcurrentInformerFactoryOptions{})
|
||||
|
||||
// Start another consumer first, as kubedog does during resource tracking.
|
||||
var shared *kdutil.Concurrent[*informer.Informer]
|
||||
var err error
|
||||
factory.RWTransaction(func(factory *informer.InformerFactory) {
|
||||
shared, err = factory.ForNamespace(podGVR, "ns")
|
||||
})
|
||||
require.NoError(t, err)
|
||||
shared.RWTransaction(func(inf *informer.Informer) {
|
||||
_, err = inf.AddEventHandler(cache.ResourceEventHandlerFuncs{})
|
||||
if err == nil {
|
||||
inf.Run()
|
||||
}
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
filter := newPodLogFilter()
|
||||
require.NoError(t, filter.watch(factory, []trackTarget{
|
||||
{kind: "deploy", namespace: "ns"},
|
||||
{kind: "sts", namespace: "ns"},
|
||||
{kind: "job", namespace: "jobs"},
|
||||
{kind: "pvc", namespace: "storage"},
|
||||
{kind: "canary", namespace: "canaries"},
|
||||
}))
|
||||
require.Eventually(t, func() bool {
|
||||
return filter.cutoff("ready", "ns", watchdogPodGVK).Equal(readyTime)
|
||||
}, 5*time.Second, 10*time.Millisecond, "initial add must retain the pod's historical readiness time")
|
||||
assert.True(t, filter.cutoff("starting", "ns", watchdogPodGVK).IsZero())
|
||||
|
||||
nextReady := readyTime.Add(time.Minute)
|
||||
startingPod = newLogFilterPod("starting", "ns", "starting-uid", "True", nextReady)
|
||||
_, err = client.Resource(podGVR).Namespace("ns").Update(ctx, startingPod, metav1.UpdateOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Eventually(t, func() bool {
|
||||
return filter.cutoff("starting", "ns", watchdogPodGVK).Equal(nextReady)
|
||||
}, 5*time.Second, 10*time.Millisecond, "updates must stop logs independently of the other pod")
|
||||
assert.EqualValues(t, 1, watches.Load(), "consumers and repeated targets must share one pod watch")
|
||||
}
|
||||
|
||||
func TestCutoffTrackedKind(t *testing.T) {
|
||||
for kind, want := range map[string]bool{
|
||||
"deploy": true,
|
||||
"sts": true,
|
||||
"ds": true,
|
||||
"job": false,
|
||||
"pvc": false,
|
||||
"canary": false,
|
||||
} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
assert.Equal(t, want, cutoffTrackedKind(kind))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewStartupLogFilter(t *testing.T) {
|
||||
// Every combination below must yield a nil filter without touching the
|
||||
// informer factory: startup-only cutoffs apply solely to full streaming.
|
||||
nilFilterCases := []struct {
|
||||
name string
|
||||
options *TrackOptions
|
||||
}{
|
||||
{name: "nil options", options: nil},
|
||||
{name: "no log streaming", options: NewTrackOptions().WithLogsUntilReady(true)},
|
||||
{name: "streaming without cutoff", options: NewTrackOptions().WithLogs(true)},
|
||||
{name: "failed-only mode", options: NewTrackOptions().WithFailedLogsOnly(true).WithLogsUntilReady(true)},
|
||||
}
|
||||
for _, tt := range nilFilterCases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
filter, err := newStartupLogFilter(tt.options, nil, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, filter)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,9 @@ type TrackOptions struct {
|
||||
Timeout time.Duration
|
||||
// Logs enables emitting logs for every pod kubedog observes.
|
||||
Logs bool
|
||||
// LogsUntilReady limits Logs to startup output through each pod's first
|
||||
// Ready transition. Job logs are unaffected. Has no effect unless Logs is true.
|
||||
LogsUntilReady bool
|
||||
// FailedLogsOnly enables capturing logs in the background and emitting
|
||||
// them only for pods that enter a failed state (CrashLoopBackOff, Error,
|
||||
// ImagePullBackOff, etc.). Has no effect when Logs is true.
|
||||
@@ -81,6 +84,12 @@ func (o *TrackOptions) WithLogsInterval(interval time.Duration) *TrackOptions {
|
||||
return o
|
||||
}
|
||||
|
||||
// WithLogsUntilReady limits pod log output to startup when log streaming is enabled.
|
||||
func (o *TrackOptions) WithLogsUntilReady(v bool) *TrackOptions {
|
||||
o.LogsUntilReady = v
|
||||
return o
|
||||
}
|
||||
|
||||
func (o *TrackOptions) WithFilterConfig(config *resource.FilterConfig) *TrackOptions {
|
||||
o.Filter = config
|
||||
return o
|
||||
|
||||
@@ -187,6 +187,7 @@ type progressPrinter struct {
|
||||
logStore *kdutil.Concurrent[*logstore.LogStore]
|
||||
skipLogs bool
|
||||
failedLogsOnly bool
|
||||
podLogFilter *podLogFilter
|
||||
gates *gateStatuses
|
||||
skipped *skippedKeys
|
||||
useColor bool
|
||||
@@ -726,6 +727,7 @@ func (p *progressPrinter) flushLogs() {
|
||||
return
|
||||
}
|
||||
}
|
||||
cutoff := p.podLogFilter.cutoff(rl.Name(), rl.Namespace(), rl.GroupVersionKind())
|
||||
for source, lines := range rl.LogLines() {
|
||||
cursorKey := resourceKey + "|" + source
|
||||
start := p.lastCounts[cursorKey]
|
||||
@@ -733,6 +735,9 @@ func (p *progressPrinter) flushLogs() {
|
||||
continue
|
||||
}
|
||||
for _, ll := range lines[start:] {
|
||||
if !cutoff.IsZero() && ll.Time.After(cutoff) {
|
||||
continue
|
||||
}
|
||||
pending = append(pending, entry{
|
||||
resourceKey: resourceKey,
|
||||
displayLabel: displayLabel,
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
kdutil "github.com/werf/kubedog/pkg/trackers/dyntracker/util"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
@@ -511,6 +512,94 @@ func TestFlushLogs_HeaderDedupedAcrossFlushes(t *testing.T) {
|
||||
assert.Contains(t, out, "line 4")
|
||||
}
|
||||
|
||||
func TestFlushLogs_LogsUntilReady(t *testing.T) {
|
||||
for _, kind := range []string{"Deployment", "StatefulSet", "DaemonSet"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
taskStore := kdutil.NewConcurrent(statestore.NewTaskStore())
|
||||
logStore := kdutil.NewConcurrent(logstore.NewLogStore())
|
||||
gvk := schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: kind}
|
||||
ts := statestore.NewReadinessTaskState("app", "ns", gvk, statestore.ReadinessTaskStateOptions{})
|
||||
// The real dyntracker leaves individual pod states unknown until
|
||||
// the entire workload is ready, even when a replica is already Ready.
|
||||
addPodChild(t, ts, "app-ready", "ns", statestore.ResourceStatusUnknown, "Running")
|
||||
addPodChild(t, ts, "app-starting", "ns", statestore.ResourceStatusUnknown, "Running")
|
||||
taskStore.RWTransaction(func(s *statestore.TaskStore) {
|
||||
s.AddReadinessTaskState(kdutil.NewConcurrent(ts))
|
||||
})
|
||||
|
||||
readyTime := time.Unix(100, 500)
|
||||
filter := newPodLogFilter()
|
||||
filter.update(newLogFilterPod("app-ready", "ns", "ready-uid", "True", readyTime))
|
||||
filter.update(newLogFilterPod("app-starting", "ns", "starting-uid", "False", readyTime))
|
||||
logger, buf, mu := newBufferedLogger(t)
|
||||
p := newProgressPrinter(logger, "", taskStore, logStore, false, false, newGateStatuses(), newSkippedKeys(), false, 0)
|
||||
p.podLogFilter = filter
|
||||
|
||||
readyLogs := kdutil.NewConcurrent(logstore.NewResourceLogs("app-ready", "ns", podGVK))
|
||||
startingLogs := kdutil.NewConcurrent(logstore.NewResourceLogs("app-starting", "ns", podGVK))
|
||||
logStore.RWTransaction(func(s *logstore.LogStore) {
|
||||
s.AddResourceLogs(readyLogs)
|
||||
s.AddResourceLogs(startingLogs)
|
||||
})
|
||||
readyLogs.RWTransaction(func(rl *logstore.ResourceLogs) {
|
||||
rl.AddLogLine("successful startup", "container/main", readyTime.Add(-time.Second))
|
||||
rl.AddLogLine("at readiness", "container/main", readyTime)
|
||||
rl.AddLogLine("HTTP request", "container/main", readyTime.Add(time.Second))
|
||||
})
|
||||
startingLogs.RWTransaction(func(rl *logstore.ResourceLogs) {
|
||||
rl.AddLogLine("still starting", "container/main", readyTime.Add(time.Second))
|
||||
})
|
||||
p.flushLogs()
|
||||
|
||||
// Late pre-readiness chunks from another container still flush;
|
||||
// repeated traffic from the ready replica stays silent.
|
||||
readyLogs.RWTransaction(func(rl *logstore.ResourceLogs) {
|
||||
rl.AddLogLine("late startup chunk", "container/sidecar", readyTime.Add(-time.Millisecond))
|
||||
rl.AddLogLine("background job", "container/sidecar", readyTime.Add(time.Second))
|
||||
rl.AddLogLine("another HTTP request", "container/main", readyTime.Add(2*time.Second))
|
||||
})
|
||||
p.flushLogs()
|
||||
out := capturedOutput(buf, mu)
|
||||
assert.Contains(t, out, "successful startup")
|
||||
assert.Contains(t, out, "at readiness")
|
||||
assert.Contains(t, out, "late startup chunk")
|
||||
assert.Contains(t, out, "still starting")
|
||||
assert.NotContains(t, out, "HTTP request")
|
||||
assert.NotContains(t, out, "background job")
|
||||
assert.Equal(t, 4, p.lastCounts["Pod/ns/app-ready|container/main"])
|
||||
assert.Equal(t, 2, p.lastCounts["Pod/ns/app-ready|container/sidecar"])
|
||||
p.flushLogs()
|
||||
assert.Equal(t, out, capturedOutput(buf, mu), "startup lines must not be duplicated")
|
||||
|
||||
// Filtering logs must not mutate readiness or hide a later failure.
|
||||
assert.NotEqual(t, statestore.ReadinessTaskStatusReady, ts.Status())
|
||||
flipPodPhase(t, ts, "app-ready", "ns", "CrashLoopBackOff")
|
||||
p.flushProgress()
|
||||
assert.Contains(t, capturedOutput(buf, mu), "CrashLoopBackOff")
|
||||
assert.Contains(t, p.collectFailedPodIDs(), kdutil.ResourceID("app-ready", "ns", podGVK))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushLogs_LogsUntilReady_JobLogs(t *testing.T) {
|
||||
taskStore := kdutil.NewConcurrent(statestore.NewTaskStore())
|
||||
logStore := kdutil.NewConcurrent(logstore.NewLogStore())
|
||||
logger, buf, mu := newBufferedLogger(t)
|
||||
p := newProgressPrinter(logger, "", taskStore, logStore, false, false, newGateStatuses(), newSkippedKeys(), false, 0)
|
||||
p.podLogFilter = newPodLogFilter()
|
||||
pod := newLogFilterPod("job-pod", "ns", "job-uid", "True", time.Unix(0, 0))
|
||||
pod.SetOwnerReferences([]metav1.OwnerReference{{Kind: "Job", Name: "job"}})
|
||||
p.podLogFilter.update(pod)
|
||||
addPodLogs(t, logStore, "job-pod", "job started", "job still running")
|
||||
p.flushLogs()
|
||||
addPodLogs(t, logStore, "job-pod", "job completed")
|
||||
p.flushLogs()
|
||||
out := capturedOutput(buf, mu)
|
||||
assert.Contains(t, out, "job started")
|
||||
assert.Contains(t, out, "job still running")
|
||||
assert.Contains(t, out, "job completed")
|
||||
}
|
||||
|
||||
func TestHeaderDivider_FormatsBothBorders(t *testing.T) {
|
||||
assert.Equal(t, "========== title: ==========", HeaderDivider("title:"))
|
||||
}
|
||||
|
||||
@@ -454,6 +454,11 @@ func (t *Tracker) TrackResources(ctx context.Context, resources []*resource.Reso
|
||||
gateStatuses := newGateStatuses()
|
||||
printer := newProgressPrinter(t.logger, t.releaseName, taskStore, logStore, ignoreLogs,
|
||||
failedLogsOnly, gateStatuses, t.skipped, t.trackOptions.Color, t.trackOptions.LogsInterval)
|
||||
podLogFilter, err := newStartupLogFilter(t.trackOptions, informerFactory, targets)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printer.podLogFilter = podLogFilter
|
||||
printerDone := make(chan struct{})
|
||||
|
||||
// Spawn a parallel failure watchdog. It catches pods that genuinely
|
||||
|
||||
@@ -32,6 +32,7 @@ func TestNewTrackOptions(t *testing.T) {
|
||||
assert.NotNil(t, opts)
|
||||
assert.Equal(t, 5*time.Minute, opts.Timeout)
|
||||
assert.Equal(t, false, opts.Logs)
|
||||
assert.False(t, opts.LogsUntilReady)
|
||||
assert.Equal(t, 10*time.Minute, opts.LogsSince)
|
||||
assert.Equal(t, 10*time.Second, opts.LogsInterval)
|
||||
}
|
||||
@@ -57,6 +58,12 @@ func TestTrackOptions_WithLogsInterval(t *testing.T) {
|
||||
assert.Equal(t, 3*time.Second, opts.LogsInterval)
|
||||
}
|
||||
|
||||
func TestTrackOptions_WithLogsUntilReady(t *testing.T) {
|
||||
opts := NewTrackOptions().WithLogs(true).WithLogsUntilReady(true)
|
||||
assert.True(t, opts.LogsUntilReady)
|
||||
assert.True(t, opts.Logs)
|
||||
}
|
||||
|
||||
func TestTrackOptions_Chaining(t *testing.T) {
|
||||
opts := NewTrackOptions()
|
||||
opts = opts.
|
||||
|
||||
@@ -313,6 +313,11 @@ func (st *HelmState) buildReleaseTracker(release *ReleaseSpec, opts *SyncOpts, u
|
||||
trackLogs = opts.TrackLogs
|
||||
}
|
||||
|
||||
trackLogsUntilReady := release.TrackLogsUntilReady != nil && *release.TrackLogsUntilReady
|
||||
if release.TrackLogsUntilReady == nil && opts != nil {
|
||||
trackLogsUntilReady = opts.TrackLogsUntilReady
|
||||
}
|
||||
|
||||
trackFailedLogs := release.TrackFailedLogs != nil && *release.TrackFailedLogs
|
||||
if release.TrackFailedLogs == nil && opts != nil {
|
||||
trackFailedLogs = opts.TrackFailedLogs
|
||||
@@ -327,6 +332,7 @@ func (st *HelmState) buildReleaseTracker(release *ReleaseSpec, opts *SyncOpts, u
|
||||
trackOpts := kubedog.NewTrackOptions().
|
||||
WithTimeout(timeout).
|
||||
WithLogs(trackLogs).
|
||||
WithLogsUntilReady(trackLogsUntilReady).
|
||||
WithFailedLogsOnly(trackFailedLogs).
|
||||
WithFilterConfig(filterConfig).
|
||||
WithColor(useColor)
|
||||
|
||||
@@ -2,12 +2,15 @@ package state
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/helmfile/helmfile/pkg/helmexec"
|
||||
"github.com/helmfile/helmfile/pkg/testutil"
|
||||
"github.com/helmfile/helmfile/pkg/yaml"
|
||||
)
|
||||
|
||||
func TestAppendWaitForJobsFlags(t *testing.T) {
|
||||
@@ -797,3 +800,62 @@ func TestAppendServerSideFlagsForUpgrade(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildReleaseTracker_LogsUntilReady(t *testing.T) {
|
||||
kubeconfig := filepath.Join(t.TempDir(), "kubeconfig")
|
||||
require.NoError(t, os.WriteFile(kubeconfig, []byte(`apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- name: test
|
||||
cluster:
|
||||
server: http://127.0.0.1:1
|
||||
contexts:
|
||||
- name: test
|
||||
context:
|
||||
cluster: test
|
||||
current-context: test
|
||||
`), 0600))
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
release string
|
||||
opts *SyncOpts
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "disabled by default",
|
||||
},
|
||||
{
|
||||
name: "cli enables startup logs",
|
||||
opts: &SyncOpts{TrackLogs: true, TrackLogsUntilReady: true},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "release enables startup logs",
|
||||
release: "trackLogs: true\ntrackLogsUntilReady: true\n",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "release false overrides cli true",
|
||||
release: "trackLogsUntilReady: false\n",
|
||||
opts: &SyncOpts{TrackLogs: true, TrackLogsUntilReady: true},
|
||||
},
|
||||
{
|
||||
name: "release true overrides cli false",
|
||||
release: "trackLogsUntilReady: true\n",
|
||||
opts: &SyncOpts{TrackLogs: true},
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var release ReleaseSpec
|
||||
require.NoError(t, yaml.Unmarshal([]byte(tt.release), &release))
|
||||
st := &HelmState{kubeconfig: kubeconfig}
|
||||
_, opts, err := st.buildReleaseTracker(&release, tt.opts, false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tt.want, opts.LogsUntilReady)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+4
-1
@@ -561,9 +561,11 @@ type ReleaseSpec struct {
|
||||
TrackTimeout *int `yaml:"trackTimeout,omitempty"`
|
||||
// TrackLogs enables log streaming with kubedog
|
||||
TrackLogs *bool `yaml:"trackLogs,omitempty"`
|
||||
// TrackLogsUntilReady stops pod log streaming when each pod becomes ready. Requires TrackLogs.
|
||||
TrackLogsUntilReady *bool `yaml:"trackLogsUntilReady,omitempty"`
|
||||
// TrackFailedLogs streams logs only for pods that enter a failed state
|
||||
// (CrashLoopBackOff, Error, etc.). Pods that succeed produce no output.
|
||||
// Has no effect when TrackLogs is true (full streaming wins).
|
||||
// Has no effect when TrackLogs is true (TrackLogs wins).
|
||||
TrackFailedLogs *bool `yaml:"trackFailedLogs,omitempty"`
|
||||
// HelmStuckGrace, when > 0, enables the safety-valve helm-killer for
|
||||
// kubedog tracking: if the cluster confirms every tracked resource has
|
||||
@@ -1082,6 +1084,7 @@ type SyncOpts struct {
|
||||
TrackMode string
|
||||
TrackTimeout int
|
||||
TrackLogs bool
|
||||
TrackLogsUntilReady bool
|
||||
TrackFailedLogs bool
|
||||
TrackLogsInterval time.Duration
|
||||
HelmStuckGrace int
|
||||
|
||||
@@ -42,39 +42,39 @@ func TestGenerateID(t *testing.T) {
|
||||
run(testcase{
|
||||
subject: "baseline",
|
||||
release: ReleaseSpec{Name: "foo", Chart: "incubator/raw"},
|
||||
want: "foo-values-565b4d4448",
|
||||
want: "foo-values-699f8dfb8f",
|
||||
})
|
||||
|
||||
run(testcase{
|
||||
subject: "different bytes content",
|
||||
release: ReleaseSpec{Name: "foo", Chart: "incubator/raw"},
|
||||
data: []byte(`{"k":"v"}`),
|
||||
want: "foo-values-5bf759b9cf",
|
||||
want: "foo-values-8484b97b54",
|
||||
})
|
||||
|
||||
run(testcase{
|
||||
subject: "different map content",
|
||||
release: ReleaseSpec{Name: "foo", Chart: "incubator/raw"},
|
||||
data: map[string]any{"k": "v"},
|
||||
want: "foo-values-cbd47bb9c",
|
||||
want: "foo-values-6d74854796",
|
||||
})
|
||||
|
||||
run(testcase{
|
||||
subject: "different chart",
|
||||
release: ReleaseSpec{Name: "foo", Chart: "stable/envoy"},
|
||||
want: "foo-values-84bcbdd8f9",
|
||||
want: "foo-values-6c75d94b64",
|
||||
})
|
||||
|
||||
run(testcase{
|
||||
subject: "different name",
|
||||
release: ReleaseSpec{Name: "bar", Chart: "incubator/raw"},
|
||||
want: "bar-values-757bdb6889",
|
||||
want: "bar-values-fdc66db9d",
|
||||
})
|
||||
|
||||
run(testcase{
|
||||
subject: "specific ns",
|
||||
release: ReleaseSpec{Name: "foo", Chart: "incubator/raw", Namespace: "myns"},
|
||||
want: "myns-foo-values-f4bcf8c77",
|
||||
want: "myns-foo-values-74d9576cdb",
|
||||
})
|
||||
|
||||
for id, n := range ids {
|
||||
|
||||
Reference in New Issue
Block a user