diff --git a/pkg/kubedog/display.go b/pkg/kubedog/display.go new file mode 100644 index 00000000..bcd234e7 --- /dev/null +++ b/pkg/kubedog/display.go @@ -0,0 +1,307 @@ +package kubedog + +import ( + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/werf/kubedog/pkg/tracker/daemonset" + "github.com/werf/kubedog/pkg/tracker/deployment" + "github.com/werf/kubedog/pkg/tracker/indicators" + "github.com/werf/kubedog/pkg/tracker/job" + "github.com/werf/kubedog/pkg/tracker/pod" + "github.com/werf/kubedog/pkg/tracker/statefulset" + "github.com/werf/kubedog/pkg/utils" +) + +var statusProgressTableRatio = []float64{.58, .11, .12, .19} +var statusProgressSubTableRatio = []float64{.40, .15, .20, .25} + +func writeOut(out io.Writer, s string) { + _, _ = fmt.Fprint(out, s) +} + +func displayDeploymentStatusProgress(out io.Writer, resourceCaption string, status deployment.DeploymentStatus, prevStatus *deployment.DeploymentStatus) { + t := utils.NewTable(statusProgressTableRatio...) + t.SetWidth(termWidth()) + + showProgress := status.StatusGeneration > prevStatus.StatusGeneration + + replicas := "-" + if status.ReplicasIndicator != nil { + replicas = status.ReplicasIndicator.FormatTableElem(prevStatus.ReplicasIndicator, indicators.FormatTableElemOptions{ + ShowProgress: showProgress, + WithTargetValue: true, + }) + } + available := "-" + if status.AvailableIndicator != nil { + available = status.AvailableIndicator.FormatTableElem(prevStatus.AvailableIndicator, indicators.FormatTableElemOptions{ + ShowProgress: showProgress, + }) + } + uptodate := "-" + if status.UpToDateIndicator != nil { + uptodate = status.UpToDateIndicator.FormatTableElem(prevStatus.UpToDateIndicator, indicators.FormatTableElemOptions{ + ShowProgress: showProgress, + }) + } + + t.Header("DEPLOYMENT", "REPLICAS", "AVAILABLE", "UP-TO-DATE") + + args := []interface{}{resourceCaption, replicas, available, uptodate} + if status.IsFailed { + args = append(args, formatResourceError(status.FailedReason)) + } + t.Row(args...) + + displayChildPodsAndWaiting(&t, prevStatus.Pods, status.Pods, status.NewPodsNames, status.WaitingForMessages) + + writeOut(out, t.Render()) +} + +func displayStatefulSetStatusProgress(out io.Writer, resourceCaption string, status statefulset.StatefulSetStatus, prevStatus *statefulset.StatefulSetStatus) { + t := utils.NewTable(statusProgressTableRatio...) + t.SetWidth(termWidth()) + + showProgress := status.StatusGeneration > prevStatus.StatusGeneration + + replicas := "-" + if status.ReplicasIndicator != nil { + replicas = status.ReplicasIndicator.FormatTableElem(prevStatus.ReplicasIndicator, indicators.FormatTableElemOptions{ + ShowProgress: showProgress, + WithTargetValue: true, + }) + } + ready := "-" + if status.ReadyIndicator != nil { + ready = status.ReadyIndicator.FormatTableElem(prevStatus.ReadyIndicator, indicators.FormatTableElemOptions{ + ShowProgress: showProgress, + }) + } + uptodate := "-" + if status.UpToDateIndicator != nil { + uptodate = status.UpToDateIndicator.FormatTableElem(prevStatus.UpToDateIndicator, indicators.FormatTableElemOptions{ + ShowProgress: showProgress, + }) + } + + t.Header("STATEFULSET", "REPLICAS", "READY", "UP-TO-DATE") + + args := []interface{}{resourceCaption, replicas, ready, uptodate} + if status.IsFailed { + args = append(args, formatResourceError(status.FailedReason)) + } else { + for _, w := range status.WarningMessages { + args = append(args, formatResourceWarning(w)) + } + } + t.Row(args...) + + displayChildPodsAndWaiting(&t, prevStatus.Pods, status.Pods, status.NewPodsNames, status.WaitingForMessages) + + writeOut(out, t.Render()) +} + +func displayDaemonSetStatusProgress(out io.Writer, resourceCaption string, status daemonset.DaemonSetStatus, prevStatus *daemonset.DaemonSetStatus) { + t := utils.NewTable(statusProgressTableRatio...) + t.SetWidth(termWidth()) + + showProgress := status.StatusGeneration > prevStatus.StatusGeneration + + replicas := "-" + if status.ReplicasIndicator != nil { + replicas = status.ReplicasIndicator.FormatTableElem(prevStatus.ReplicasIndicator, indicators.FormatTableElemOptions{ + ShowProgress: showProgress, + WithTargetValue: true, + }) + } + available := "-" + if status.AvailableIndicator != nil { + available = status.AvailableIndicator.FormatTableElem(prevStatus.AvailableIndicator, indicators.FormatTableElemOptions{ + ShowProgress: showProgress, + }) + } + uptodate := "-" + if status.UpToDateIndicator != nil { + uptodate = status.UpToDateIndicator.FormatTableElem(prevStatus.UpToDateIndicator, indicators.FormatTableElemOptions{ + ShowProgress: showProgress, + }) + } + + t.Header("DAEMONSET", "REPLICAS", "AVAILABLE", "UP-TO-DATE") + + args := []interface{}{resourceCaption, replicas, available, uptodate} + if status.IsFailed { + args = append(args, formatResourceError(status.FailedReason)) + } + t.Row(args...) + + displayChildPodsAndWaiting(&t, prevStatus.Pods, status.Pods, status.NewPodsNames, status.WaitingForMessages) + + writeOut(out, t.Render()) +} + +func displayJobStatusProgress(out io.Writer, resourceCaption string, status job.JobStatus, prevStatus *job.JobStatus) { + t := utils.NewTable(statusProgressTableRatio...) + t.SetWidth(termWidth()) + + showProgress := status.StatusGeneration > prevStatus.StatusGeneration + + succeeded := "-" + if status.SucceededIndicator != nil { + succeeded = status.SucceededIndicator.FormatTableElem(prevStatus.SucceededIndicator, indicators.FormatTableElemOptions{ + ShowProgress: showProgress, + }) + } + + t.Header("JOB", "ACTIVE", "DURATION", "SUCCEEDED/FAILED") + + var active interface{} = "-" + if status.Active != 0 { + active = status.Active + } + failed := fmt.Sprintf("%d", status.Failed) + + args := []interface{}{resourceCaption, active, status.Age, strings.Join([]string{succeeded, failed}, "/")} + if status.IsFailed { + args = append(args, formatResourceError(status.FailedReason)) + } + t.Row(args...) + + if len(status.Pods) > 0 { + st := displayChildPodsStatusProgress(&t, prevStatus.Pods, status.Pods, nil, showProgress) + extraMsg := "" + if len(status.WaitingForMessages) > 0 { + extraMsg += "---\n" + extraMsg += utils.BlueF("Waiting for: %s", strings.Join(status.WaitingForMessages, ", ")) + } + st.Commit(extraMsg) + } + + writeOut(out, t.Render()) +} + +func displayChildPodsAndWaiting(t *utils.Table, prevPods, pods map[string]pod.PodStatus, newPodsNames []string, waitingForMessages []string) { + if len(pods) > 0 { + st := displayChildPodsStatusProgress(t, prevPods, pods, newPodsNames, true) + extraMsg := "" + if len(waitingForMessages) > 0 { + extraMsg += "---\n" + extraMsg += utils.BlueF("Waiting for: %s", strings.Join(waitingForMessages, ", ")) + } + st.Commit(extraMsg) + } +} + +func displayChildPodsStatusProgress(t *utils.Table, prevPods, pods map[string]pod.PodStatus, newPodsNames []string, showProgress bool) *utils.Table { + subT := t.SubTable(statusProgressSubTableRatio...) + st := &subT + + st.Header("POD", "READY", "RESTARTS", "STATUS") + + podsNames := make([]string, 0, len(pods)) + for podName := range pods { + podsNames = append(podsNames, podName) + } + sort.Strings(podsNames) + + var podRows [][]interface{} + for _, podName := range podsNames { + var podRow []interface{} + + isPodNew := false + for _, newPodName := range newPodsNames { + if newPodName == podName { + isPodNew = true + } + } + + prevPodStatus := prevPods[podName] + podStatus := pods[podName] + + isReady := false + if podStatus.StatusIndicator != nil { + isReady = podStatus.StatusIndicator.IsReady() + } + + resource := formatPodResourceCaption(podName, isReady, podStatus.IsFailed, isPodNew) + ready := fmt.Sprintf("%d/%d", podStatus.ReadyContainers, podStatus.TotalContainers) + + status := "-" + if podStatus.StatusIndicator != nil { + status = podStatus.StatusIndicator.FormatTableElem(prevPodStatus.StatusIndicator, indicators.FormatTableElemOptions{ + ShowProgress: showProgress, + IsResourceNew: isPodNew, + }) + } + + podRow = append(podRow, resource, ready, podStatus.Restarts, status) + if podStatus.IsFailed { + podRow = append(podRow, formatResourceError(podStatus.FailedReason)) + } + + podRows = append(podRows, podRow) + } + + st.Rows(podRows...) + + return st +} + +func formatResourceCaption(caption string, isReady, isFailed bool) string { + switch { + case isReady: + return utils.GreenF("%s", caption) + case isFailed: + return utils.RedF("%s", caption) + default: + return utils.YellowF("%s", caption) + } +} + +func formatPodResourceCaption(podName string, isReady, isFailed, isNew bool) string { + if !isNew { + return podName + } + return formatResourceCaption(podName, isReady, isFailed) +} + +func formatResourceError(reason string) string { + return utils.RedF("error: %s", reason) +} + +func formatResourceWarning(reason string) string { + return utils.YellowF("warning: %s", reason) +} + +func termWidth() int { + return 140 +} + +func displayCanaryStatus(out io.Writer, resourceCaption string, status CanaryStatusView) { + var parts []string + if status.Phase != "" { + parts = append(parts, fmt.Sprintf("phase %s", status.Phase)) + } + if status.Age != "" { + parts = append(parts, fmt.Sprintf("age %s", status.Age)) + } + msg := fmt.Sprintf("%s: %s", resourceCaption, strings.Join(parts, ", ")) + if status.IsFailed { + msg = utils.RedF("%s", msg) + } + _, _ = fmt.Fprintln(out, msg) +} + +type CanaryStatusView struct { + Phase string + Age string + IsFailed bool +} + +func statusOutput() io.Writer { + return os.Stderr +} diff --git a/pkg/kubedog/tracker.go b/pkg/kubedog/tracker.go index 867116f8..679be994 100644 --- a/pkg/kubedog/tracker.go +++ b/pkg/kubedog/tracker.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/werf/kubedog/pkg/display" "github.com/werf/kubedog/pkg/informer" "github.com/werf/kubedog/pkg/tracker" "github.com/werf/kubedog/pkg/tracker/canary" @@ -234,6 +235,7 @@ func (t *Tracker) TrackResources(ctx context.Context, resources []*resource.Reso ParentContext: ctx, Timeout: t.trackOptions.Timeout, LogsFromTime: time.Now().Add(-t.trackOptions.LogsSince), + IgnoreLogs: !t.trackOptions.Logs, } var wg sync.WaitGroup @@ -312,13 +314,33 @@ func (t *Tracker) runDeploymentTracker(ctx context.Context, tr *deployment.Track } func (t *Tracker) waitDeploymentTracker(ctx context.Context, tr *deployment.Tracker, trackErrCh <-chan error, doneCh <-chan struct{}) error { + var prevStatus deployment.DeploymentStatus + out := statusOutput() + caption := formatResourceCaption(fmt.Sprintf("deploy/%s", tr.ResourceName), false, false) + for { select { + case status := <-tr.Added: + displayDeploymentStatusProgress(out, caption, status, &prevStatus) + prevStatus = status case <-tr.Ready: - t.logger.Debugf("Deployment %s/%s is ready", tr.Namespace, tr.ResourceName) + t.logger.Infof("Deployment %s/%s is ready", tr.Namespace, tr.ResourceName) return nil case status := <-tr.Failed: return fmt.Errorf("deployment %s/%s failed: %s", tr.Namespace, tr.ResourceName, status.FailedReason) + case status := <-tr.Status: + if status.StatusGeneration > prevStatus.StatusGeneration { + displayDeploymentStatusProgress(out, caption, status, &prevStatus) + prevStatus = status + } + case msg := <-tr.EventMsg: + t.logger.Infof("deploy/%s: %s", tr.ResourceName, msg) + case chunk := <-tr.PodLogChunk: + t.logPodLogChunk(chunk.PodName, chunk.LogLines) + case report := <-tr.PodError: + t.logger.Warnf("deploy/%s pod %s: %s: %s", tr.ResourceName, report.ReplicaSetPodError.PodName, report.ReplicaSetPodError.ContainerName, report.ReplicaSetPodError.Message) + case <-tr.AddedReplicaSet: + case <-tr.AddedPod: case err := <-trackErrCh: return err case <-doneCh: @@ -338,13 +360,32 @@ func (t *Tracker) runStatefulSetTracker(ctx context.Context, tr *statefulset.Tra } func (t *Tracker) waitStatefulSetTracker(ctx context.Context, tr *statefulset.Tracker, trackErrCh <-chan error, doneCh <-chan struct{}) error { + var prevStatus statefulset.StatefulSetStatus + out := statusOutput() + caption := formatResourceCaption(fmt.Sprintf("sts/%s", tr.ResourceName), false, false) + for { select { + case status := <-tr.Added: + displayStatefulSetStatusProgress(out, caption, status, &prevStatus) + prevStatus = status case <-tr.Ready: - t.logger.Debugf("StatefulSet %s/%s is ready", tr.Namespace, tr.ResourceName) + t.logger.Infof("StatefulSet %s/%s is ready", tr.Namespace, tr.ResourceName) return nil case status := <-tr.Failed: return fmt.Errorf("statefulset %s/%s failed: %s", tr.Namespace, tr.ResourceName, status.FailedReason) + case status := <-tr.Status: + if status.StatusGeneration > prevStatus.StatusGeneration { + displayStatefulSetStatusProgress(out, caption, status, &prevStatus) + prevStatus = status + } + case msg := <-tr.EventMsg: + t.logger.Infof("sts/%s: %s", tr.ResourceName, msg) + case chunk := <-tr.PodLogChunk: + t.logPodLogChunk(chunk.PodName, chunk.LogLines) + case report := <-tr.PodError: + t.logger.Warnf("sts/%s pod %s: %s: %s", tr.ResourceName, report.ReplicaSetPodError.PodName, report.ReplicaSetPodError.ContainerName, report.ReplicaSetPodError.Message) + case <-tr.AddedPod: case err := <-trackErrCh: return err case <-doneCh: @@ -364,13 +405,32 @@ func (t *Tracker) runDaemonSetTracker(ctx context.Context, tr *daemonset.Tracker } func (t *Tracker) waitDaemonSetTracker(ctx context.Context, tr *daemonset.Tracker, trackErrCh <-chan error, doneCh <-chan struct{}) error { + var prevStatus daemonset.DaemonSetStatus + out := statusOutput() + caption := formatResourceCaption(fmt.Sprintf("ds/%s", tr.ResourceName), false, false) + for { select { + case status := <-tr.Added: + displayDaemonSetStatusProgress(out, caption, status, &prevStatus) + prevStatus = status case <-tr.Ready: - t.logger.Debugf("DaemonSet %s/%s is ready", tr.Namespace, tr.ResourceName) + t.logger.Infof("DaemonSet %s/%s is ready", tr.Namespace, tr.ResourceName) return nil case status := <-tr.Failed: return fmt.Errorf("daemonset %s/%s failed: %s", tr.Namespace, tr.ResourceName, status.FailedReason) + case status := <-tr.Status: + if status.StatusGeneration > prevStatus.StatusGeneration { + displayDaemonSetStatusProgress(out, caption, status, &prevStatus) + prevStatus = status + } + case msg := <-tr.EventMsg: + t.logger.Infof("ds/%s: %s", tr.ResourceName, msg) + case chunk := <-tr.PodLogChunk: + t.logPodLogChunk(chunk.PodName, chunk.LogLines) + case report := <-tr.PodError: + t.logger.Warnf("ds/%s pod %s: %s: %s", tr.ResourceName, report.PodError.PodName, report.PodError.ContainerName, report.PodError.Message) + case <-tr.AddedPod: case err := <-trackErrCh: return err case <-doneCh: @@ -390,13 +450,32 @@ func (t *Tracker) runJobTracker(ctx context.Context, tr *job.Tracker, errCh chan } func (t *Tracker) waitJobTracker(ctx context.Context, tr *job.Tracker, trackErrCh <-chan error, doneCh <-chan struct{}) error { + var prevStatus job.JobStatus + out := statusOutput() + caption := formatResourceCaption(fmt.Sprintf("job/%s", tr.ResourceName), false, false) + for { select { + case status := <-tr.Added: + displayJobStatusProgress(out, caption, status, &prevStatus) + prevStatus = status case <-tr.Succeeded: - t.logger.Debugf("Job %s/%s succeeded", tr.Namespace, tr.ResourceName) + t.logger.Infof("Job %s/%s succeeded", tr.Namespace, tr.ResourceName) return nil case status := <-tr.Failed: return fmt.Errorf("job %s/%s failed: %s", tr.Namespace, tr.ResourceName, status.FailedReason) + case status := <-tr.Status: + if status.StatusGeneration > prevStatus.StatusGeneration { + displayJobStatusProgress(out, caption, status, &prevStatus) + prevStatus = status + } + case msg := <-tr.EventMsg: + t.logger.Infof("job/%s: %s", tr.ResourceName, msg) + case chunk := <-tr.PodLogChunk: + t.logPodLogChunk(chunk.PodName, chunk.LogLines) + case report := <-tr.PodError: + t.logger.Warnf("job/%s pod %s: %s: %s", tr.ResourceName, report.PodError.PodName, report.PodError.ContainerName, report.PodError.Message) + case <-tr.AddedPod: case err := <-trackErrCh: return err case <-doneCh: @@ -416,13 +495,34 @@ func (t *Tracker) runCanaryTracker(ctx context.Context, tr *canary.Tracker, errC } func (t *Tracker) waitCanaryTracker(ctx context.Context, tr *canary.Tracker, trackErrCh <-chan error, doneCh <-chan struct{}) error { + out := statusOutput() + caption := formatResourceCaption(fmt.Sprintf("canary/%s", tr.ResourceName), false, false) + for { select { + case status := <-tr.Added: + displayCanaryStatus(out, caption, CanaryStatusView{ + Phase: string(status.CanaryStatus.Phase), + IsFailed: status.IsFailed, + }) case <-tr.Succeeded: - t.logger.Debugf("Canary %s/%s succeeded", tr.Namespace, tr.ResourceName) + t.logger.Infof("Canary %s/%s succeeded", tr.Namespace, tr.ResourceName) return nil case status := <-tr.Failed: return fmt.Errorf("canary %s/%s failed: %s", tr.Namespace, tr.ResourceName, status.FailedReason) + case status := <-tr.Status: + displayCanaryStatus(out, caption, CanaryStatusView{ + Phase: func() string { + if status.StatusIndicator != nil { + return status.StatusIndicator.Value + } + return "" + }(), + Age: status.Age, + IsFailed: status.IsFailed, + }) + case msg := <-tr.EventMsg: + t.logger.Infof("canary/%s: %s", tr.ResourceName, msg) case err := <-trackErrCh: return err case <-doneCh: @@ -433,6 +533,12 @@ func (t *Tracker) waitCanaryTracker(ctx context.Context, tr *canary.Tracker, tra } } +func (t *Tracker) logPodLogChunk(podName string, logLines []display.LogLine) { + for _, line := range logLines { + t.logger.Infof("po/%s [%s] %s", podName, line.Timestamp, line.Message) + } +} + func (t *Tracker) buildTargets(resources []*resource.Resource) []trackTarget { var targets []trackTarget for _, res := range resources {