Improved notification messages

cmd/manager/main.go

@@ -13,6 +13,7 @@ import (
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/notifications"
 	"github.com/jenkinsci/kubernetes-operator/pkg/event"
 	"github.com/jenkinsci/kubernetes-operator/pkg/log"
 	"github.com/jenkinsci/kubernetes-operator/version"
@@ -118,8 +119,11 @@ func main() {
 		fatal(errors.Wrap(err, "failed to create Kubernetes client set"), *debug)
 	}
 
+	c := make(chan notifications.Event)
+	go notifications.Listen(c, mgr.GetClient())
+
 	// setup Jenkins controller
-	if err := jenkins.Add(mgr, *local, *minikube, events, *clientSet, *cfg); err != nil {
+	if err := jenkins.Add(mgr, *local, *minikube, events, *clientSet, *cfg, &c); err != nil {
 		fatal(errors.Wrap(err, "failed to setup controllers"), *debug)
 	}
 

pkg/apis/jenkins/v1alpha2/jenkins_types.go

@@ -17,9 +17,9 @@ type JenkinsSpec struct {
 	// +optional
 	SeedJobs []SeedJob `json:"seedJobs,omitempty"`
 
-	/*	// Notifications defines list of a services which are used to inform about Jenkins status
+	// Notifications defines list of a services which are used to inform about Jenkins status
-		// Can be used to integrate chat services like Slack, Microsoft MicrosoftTeams or Mailgun
+	// Can be used to integrate chat services like Slack, Microsoft Teams or Mailgun
-		Notifications []Notification `json:"notifications,omitempty"`*/
+	Notifications []Notification `json:"notifications,omitempty"`
 
 	// Service is Kubernetes service of Jenkins master HTTP pod
 	// Defaults to :

pkg/apis/jenkins/v1alpha2/zz_generated.deepcopy.go

@@ -342,6 +342,13 @@ func (in *JenkinsSpec) DeepCopyInto(out *JenkinsSpec) {
 		*out = make([]SeedJob, len(*in))
 		copy(*out, *in)
 	}
+	if in.Notifications != nil {
+		in, out := &in.Notifications, &out.Notifications
+		*out = make([]Notification, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	in.Service.DeepCopyInto(&out.Service)
 	in.SlaveService.DeepCopyInto(&out.SlaveService)
 	in.Backup.DeepCopyInto(&out.Backup)

pkg/controller/jenkins/configuration/base/reconcile.go

@@ -124,6 +124,7 @@ func (r *ReconcileJenkinsBaseConfiguration) Reconcile() (reconcile.Result, jenki
 	}
 
 	result, err = r.ensureBaseConfiguration(jenkinsClient)
+
 	return result, jenkinsClient, err
 }
 

pkg/controller/jenkins/jenkins_controller.go

@@ -3,6 +3,7 @@ package jenkins
 import (
 	"context"
 	"fmt"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/notifications"
 	"reflect"
 
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
@@ -52,20 +53,21 @@ var reconcileErrors = map[string]reconcileError{}
 
 // Add creates a new Jenkins Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
-func Add(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config) error {
+func Add(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config, notificationEvents *chan notifications.Event) error {
-	return add(mgr, newReconciler(mgr, local, minikube, events, clientSet, config))
+	return add(mgr, newReconciler(mgr, local, minikube, events, clientSet, config, notificationEvents))
 }
 
 // newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config) reconcile.Reconciler {
+func newReconciler(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config, notificationEvents *chan notifications.Event) reconcile.Reconciler {
 	return &ReconcileJenkins{
-		client:    mgr.GetClient(),
+		client:             mgr.GetClient(),
-		scheme:    mgr.GetScheme(),
+		scheme:             mgr.GetScheme(),
-		local:     local,
+		local:              local,
-		minikube:  minikube,
+		minikube:           minikube,
-		events:    events,
+		events:             events,
-		clientSet: clientSet,
+		clientSet:          clientSet,
-		config:    config,
+		config:             config,
+		notificationEvents: notificationEvents,
 	}
 }
 
@@ -119,19 +121,34 @@ var _ reconcile.Reconciler = &ReconcileJenkins{}
 
 // ReconcileJenkins reconciles a Jenkins object
 type ReconcileJenkins struct {
-	client          client.Client
+	client             client.Client
-	scheme          *runtime.Scheme
+	scheme             *runtime.Scheme
-	local, minikube bool
+	local, minikube    bool
-	events          event.Recorder
+	events             event.Recorder
-	clientSet       kubernetes.Clientset
+	clientSet          kubernetes.Clientset
-	config          rest.Config
+	config             rest.Config
+	notificationEvents *chan notifications.Event
 }
 
 // Reconcile it's a main reconciliation loop which maintain desired state based on Jenkins.Spec
 func (r *ReconcileJenkins) Reconcile(request reconcile.Request) (reconcile.Result, error) {
+	reconcileFailLimit := uint64(10)
 	logger := r.buildLogger(request.Name)
 	logger.V(log.VDebug).Info("Reconciling Jenkins")
 
+	jenkins := &v1alpha2.Jenkins{}
+	err := r.client.Get(context.TODO(), request.NamespacedName, jenkins)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			// Request object not found, could have been deleted after reconcile request.
+			// Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
+			// Return and don't requeue
+			return reconcile.Result{}, nil
+		}
+		// Error reading the object - requeue the request.
+		return reconcile.Result{}, errors.WithStack(err)
+	}
+
 	result, err := r.reconcile(request, logger)
 	if err != nil && apierrors.IsConflict(err) {
 		return reconcile.Result{Requeue: true}, nil
@@ -151,11 +168,19 @@ func (r *ReconcileJenkins) Reconcile(request reconcile.Request) (reconcile.Resul
 			}
 		}
 		reconcileErrors[request.Name] = lastErrors
-		if lastErrors.counter >= 15 {
+		if lastErrors.counter >= reconcileFailLimit {
 			if log.Debug {
-				logger.V(log.VWarn).Info(fmt.Sprintf("Reconcile loop failed ten times with the same error, giving up: %+v", err))
+				logger.V(log.VWarn).Info(fmt.Sprintf("Reconcile loop failed %d times with the same error, giving up: %+v", reconcileFailLimit, err))
 			} else {
-				logger.V(log.VWarn).Info(fmt.Sprintf("Reconcile loop failed ten times with the same error, giving up: %s", err))
+				logger.V(log.VWarn).Info(fmt.Sprintf("Reconcile loop failed %d times with the same error, giving up: %s", reconcileFailLimit, err))
+			}
+
+			*r.notificationEvents <- notifications.Event{
+				Jenkins:           *jenkins,
+				ConfigurationType: notifications.ConfigurationTypeUnknown,
+				LogLevel:          v1alpha2.NotificationLogLevelWarning,
+				Message:           fmt.Sprintf("Reconcile loop failed ten times with the same error, giving up: %s", err),
+				MessageVerbose:    fmt.Sprintf("Reconcile loop failed ten times with the same error, giving up: %+v", err),
 			}
 			return reconcile.Result{Requeue: false}, nil
 		}
@@ -203,8 +228,16 @@ func (r *ReconcileJenkins) reconcile(request reconcile.Request, logger logr.Logg
 		return reconcile.Result{}, err
 	}
 	if !valid {
+		message := "Validation of base configuration failed, please correct Jenkins CR"
+		*r.notificationEvents <- notifications.Event{
+			Jenkins:           *jenkins,
+			ConfigurationType: notifications.ConfigurationTypeBase,
+			LogLevel:          v1alpha2.NotificationLogLevelWarning,
+			Message:           message,
+			MessageVerbose:    message,
+		}
 		r.events.Emit(jenkins, event.TypeWarning, reasonCRValidationFailure, "Base CR validation failed")
-		logger.V(log.VWarn).Info("Validation of base configuration failed, please correct Jenkins CR")
+		logger.V(log.VWarn).Info(message)
 		return reconcile.Result{}, nil // don't requeue
 	}
 
@@ -226,8 +259,17 @@ func (r *ReconcileJenkins) reconcile(request reconcile.Request, logger logr.Logg
 		if err != nil {
 			return reconcile.Result{}, errors.WithStack(err)
 		}
-		logger.Info(fmt.Sprintf("Base configuration phase is complete, took %s",
+
-			jenkins.Status.BaseConfigurationCompletedTime.Sub(jenkins.Status.ProvisionStartTime.Time)))
+		message := fmt.Sprintf("Base configuration phase is complete, took %s",
+			jenkins.Status.BaseConfigurationCompletedTime.Sub(jenkins.Status.ProvisionStartTime.Time))
+		*r.notificationEvents <- notifications.Event{
+			Jenkins:           *jenkins,
+			ConfigurationType: notifications.ConfigurationTypeBase,
+			LogLevel:          v1alpha2.NotificationLogLevelInfo,
+			Message:           message,
+			MessageVerbose:    message,
+		}
+		logger.Info(message)
 		r.events.Emit(jenkins, event.TypeNormal, reasonBaseConfigurationSuccess, "Base configuration completed")
 	}
 	// Reconcile user configuration
@@ -238,7 +280,15 @@ func (r *ReconcileJenkins) reconcile(request reconcile.Request, logger logr.Logg
 		return reconcile.Result{}, err
 	}
 	if !valid {
-		logger.V(log.VWarn).Info("Validation of user configuration failed, please correct Jenkins CR")
+		message := fmt.Sprintf("Validation of user configuration failed, please correct Jenkins CR")
+		*r.notificationEvents <- notifications.Event{
+			Jenkins:           *jenkins,
+			ConfigurationType: notifications.ConfigurationTypeUser,
+			LogLevel:          v1alpha2.NotificationLogLevelWarning,
+			Message:           message,
+			MessageVerbose:    message,
+		}
+		logger.V(log.VWarn).Info(message)
 		r.events.Emit(jenkins, event.TypeWarning, reasonCRValidationFailure, "User CR validation failed")
 		return reconcile.Result{}, nil // don't requeue
 	}
@@ -258,8 +308,16 @@ func (r *ReconcileJenkins) reconcile(request reconcile.Request, logger logr.Logg
 		if err != nil {
 			return reconcile.Result{}, errors.WithStack(err)
 		}
-		logger.Info(fmt.Sprintf("User configuration phase is complete, took %s",
+		message := fmt.Sprintf("User configuration phase is complete, took %s",
-			jenkins.Status.UserConfigurationCompletedTime.Sub(jenkins.Status.ProvisionStartTime.Time)))
+			jenkins.Status.UserConfigurationCompletedTime.Sub(jenkins.Status.ProvisionStartTime.Time))
+		*r.notificationEvents <- notifications.Event{
+			Jenkins:           *jenkins,
+			ConfigurationType: notifications.ConfigurationTypeUser,
+			LogLevel:          v1alpha2.NotificationLogLevelInfo,
+			Message:           message,
+			MessageVerbose:    message,
+		}
+		logger.Info(message)
 		r.events.Emit(jenkins, event.TypeNormal, reasonUserConfigurationSuccess, "User configuration completed")
 	}
 

pkg/controller/jenkins/notifications/mailgun.go

@@ -20,8 +20,8 @@ const content = `
 <html>
 <head></head>
 <body>
-		<h1 style="background-color: %s; color: white; padding: 3px 10px;">Jenkins Operator Reconciled</h1>
+		<h1 style="background-color: %s; color: white; padding: 3px 10px;">%s</h1>
-		<h3>Failed to do something</h3>
+		<h3>%s</h3>
 		<table>
 			<tr>
 				<td><b>CR name:</b></td>
@@ -31,10 +31,6 @@ const content = `
 				<td><b>Configuration type:</b></td>
 				<td>%s</td>
 			</tr>
-			<tr>
-				<td><b>Status:</b></td>
-				<td><b style="color: %s;">%s</b></td>
-			</tr>
 		</table>
 		<h6 style="font-size: 11px; color: grey; margin-top: 15px;">Powered by Jenkins Operator <3</h6>
 </body>
@@ -74,9 +70,17 @@ func (m MailGun) Send(event Event, config v1alpha2.Notification) error {
 
 	mg := mailgun.NewMailgun(config.Mailgun.Domain, secretValue)
 
-	htmlMessage := fmt.Sprintf(content, m.getStatusColor(event.LogLevel), event.Jenkins.Name, event.ConfigurationType, m.getStatusColor(event.LogLevel), string(event.LogLevel))
+	var statusMessage string
 
-	msg := mg.NewMessage(fmt.Sprintf("Jenkins Operator Notifier <%s>", config.Mailgun.From), "Jenkins Operator Status", "", config.Mailgun.Recipient)
+	if config.Verbose {
+		statusMessage = event.MessageVerbose
+	} else {
+		statusMessage = event.Message
+	}
+
+	htmlMessage := fmt.Sprintf(content, m.getStatusColor(event.LogLevel), statusMessage, event.Jenkins.Name, event.ConfigurationType, m.getStatusColor(event.LogLevel))
+
+	msg := mg.NewMessage(fmt.Sprintf("Jenkins Operator Notifier <%s>", config.Mailgun.From), notificationTitle(event), "", config.Mailgun.Recipient)
 	msg.SetHtml(htmlMessage)
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 	defer cancel()

pkg/controller/jenkins/notifications/msteams.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
@@ -26,6 +27,7 @@ type TeamsMessage struct {
 	ThemeColor StatusColor    `json:"themeColor"`
 	Title      string         `json:"title"`
 	Sections   []TeamsSection `json:"sections"`
+	Summary    string         `json:"summary"`
 }
 
 // TeamsSection is MS Teams message section
@@ -67,11 +69,10 @@ func (t Teams) Send(event Event, config v1alpha2.Notification) error {
 		return errors.Errorf("Microsoft Teams webhook URL is empty in secret '%s/%s[%s]", event.Jenkins.Namespace, selector.Name, selector.Key)
 	}
 
-	msg, err := json.Marshal(TeamsMessage{
+	tm := &TeamsMessage{
 		Type:       "MessageCard",
 		Context:    "https://schema.org/extensions",
 		ThemeColor: t.getStatusColor(event.LogLevel),
-		Title:      titleText,
 		Sections: []TeamsSection{
 			{
 				Facts: []TeamsFact{
@@ -79,14 +80,6 @@ func (t Teams) Send(event Event, config v1alpha2.Notification) error {
 						Name:  crNameFieldName,
 						Value: event.Jenkins.Name,
 					},
-					{
-						Name:  configurationTypeFieldName,
-						Value: event.ConfigurationType,
-					},
-					{
-						Name:  loggingLevelFieldName,
-						Value: string(event.LogLevel),
-					},
 					{
 						Name:  namespaceFieldName,
 						Value: event.Jenkins.Namespace,
@@ -95,7 +88,24 @@ func (t Teams) Send(event Event, config v1alpha2.Notification) error {
 				Text: event.Message,
 			},
 		},
-	})
+		Summary: event.Message,
+	}
+
+	tm.Title = notificationTitle(event)
+
+	if config.Verbose {
+		tm.Sections[0].Text = event.MessageVerbose
+		tm.Summary = event.MessageVerbose
+	}
+
+	if event.ConfigurationType != ConfigurationTypeUnknown {
+		tm.Sections[0].Facts = append(tm.Sections[0].Facts, TeamsFact{
+			Name:  configurationTypeFieldName,
+			Value: event.ConfigurationType,
+		})
+	}
+
+	msg, err := json.Marshal(tm)
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -110,6 +120,9 @@ func (t Teams) Send(event Event, config v1alpha2.Notification) error {
 		return errors.WithStack(err)
 	}
 
+	if resp.StatusCode != http.StatusOK {
+		return errors.New(fmt.Sprintf("Invalid response from server: %s", resp.Status))
+	}
 	defer func() { _ = resp.Body.Close() }()
 
 	return nil

pkg/controller/jenkins/notifications/msteams_test.go

@@ -42,7 +42,7 @@ func TestTeams_Send(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		assert.Equal(t, message.Title, titleText)
+		assert.Equal(t, message.Title, notificationTitle(event))
 		assert.Equal(t, message.ThemeColor, teams.getStatusColor(event.LogLevel))
 
 		mainSection := message.Sections[0]

pkg/controller/jenkins/notifications/sender.go

@@ -1,13 +1,19 @@
 package notifications
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
+	"github.com/jenkinsci/kubernetes-operator/pkg/log"
+
+	"github.com/pkg/errors"
+	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
-	titleText                  = "Operator reconciled."
+	infoTitleText              = "Jenkins Operator reconciliation info"
+	warnTitleText              = "Jenkins Operator reconciliation warning"
 	messageFieldName           = "Message"
 	loggingLevelFieldName      = "Logging Level"
 	crNameFieldName            = "CR Name"
@@ -16,6 +22,17 @@ const (
 	footerContent              = "Powered by Jenkins Operator"
 )
 
+const (
+	// ConfigurationTypeBase is core configuration of Jenkins provided by the Operator
+	ConfigurationTypeBase = "base"
+
+	// ConfigurationTypeUser is user-defined configuration of Jenkins
+	ConfigurationTypeUser = "user"
+
+	// ConfigurationTypeUnknown is untraceable type of configuration
+	ConfigurationTypeUnknown = "unknown"
+)
+
 var (
 	testConfigurationType = "test-configuration"
 	testCrName            = "test-cr"
@@ -42,7 +59,7 @@ type Event struct {
 	MessageVerbose    string
 }
 
-/*type service interface {
+type service interface {
 	Send(event Event, notificationConfig v1alpha2.Notification) error
 }
 
@@ -61,7 +78,7 @@ func Listen(events chan Event, k8sClient k8sclient.Client) {
 			} else if notificationConfig.Mailgun != nil {
 				svc = MailGun{k8sClient: k8sClient}
 			} else {
-				logger.V(log.VWarn).Info(fmt.Sprintf("Unexpected notification `%+v`", notificationConfig))
+				logger.V(log.VWarn).Info(fmt.Sprintf("Unknown notification service `%+v`", notificationConfig))
 				continue
 			}
 
@@ -85,6 +102,15 @@ func notify(svc service, event Event, manifest v1alpha2.Notification) error {
 	if event.LogLevel == v1alpha2.NotificationLogLevelInfo && manifest.LoggingLevel == v1alpha2.NotificationLogLevelWarning {
 		return nil
 	}
-
 	return svc.Send(event, manifest)
-}*/
+}
+
+func notificationTitle(event Event) string {
+	if event.LogLevel == v1alpha2.NotificationLogLevelInfo {
+		return infoTitleText
+	} else if event.LogLevel == v1alpha2.NotificationLogLevelWarning {
+		return warnTitleText
+	} else {
+		return "undefined"
+	}
+}

pkg/controller/jenkins/notifications/slack.go

@@ -64,12 +64,11 @@ func (s Slack) Send(event Event, config v1alpha2.Notification) error {
 		return err
 	}
 
-	slackMessage, err := json.Marshal(SlackMessage{
+	sm := &SlackMessage{
 		Attachments: []SlackAttachment{
 			{
 				Fallback: "",
 				Color:    s.getStatusColor(event.LogLevel),
-				Text:     titleText,
 				Fields: []SlackField{
 					{
 						Title: messageFieldName,
@@ -81,16 +80,6 @@ func (s Slack) Send(event Event, config v1alpha2.Notification) error {
 						Value: event.Jenkins.Name,
 						Short: true,
 					},
-					{
-						Title: configurationTypeFieldName,
-						Value: event.ConfigurationType,
-						Short: true,
-					},
-					{
-						Title: loggingLevelFieldName,
-						Value: string(event.LogLevel),
-						Short: true,
-					},
 					{
 						Title: namespaceFieldName,
 						Value: event.Jenkins.Namespace,
@@ -100,17 +89,35 @@ func (s Slack) Send(event Event, config v1alpha2.Notification) error {
 				Footer: footerContent,
 			},
 		},
-	})
+	}
+
+	mainAttachment := sm.Attachments[0]
+
+	mainAttachment.Title = notificationTitle(event)
+
+	if config.Verbose {
+		// TODO: or for title == message
+		mainAttachment.Fields[0].Value = event.MessageVerbose
+	}
+
+	if event.ConfigurationType != ConfigurationTypeUnknown {
+		mainAttachment.Fields = append(mainAttachment.Fields, SlackField{
+			Title: configurationTypeFieldName,
+			Value: event.ConfigurationType,
+			Short: true,
+		})
+	}
+
+	slackMessage, err := json.Marshal(sm)
+	if err != nil {
+		return err
+	}
 
 	secretValue := string(secret.Data[selector.Key])
 	if secretValue == "" {
 		return errors.Errorf("SecretValue %s is empty", selector.Name)
 	}
 
-	if err != nil {
-		return err
-	}
-
 	request, err := http.NewRequest("POST", secretValue, bytes.NewBuffer(slackMessage))
 	if err != nil {
 		return err

pkg/controller/jenkins/notifications/slack_test.go

@@ -45,7 +45,7 @@ func TestSlack_Send(t *testing.T) {
 
 		mainAttachment := message.Attachments[0]
 
-		assert.Equal(t, mainAttachment.Text, titleText)
+		assert.Equal(t, mainAttachment.Title, notificationTitle(event))
 		for _, field := range mainAttachment.Fields {
 			switch field.Title {
 			case configurationTypeFieldName: