public_git
/
kubernetes-operator
mirror of https://github.com/jenkinsci/kubernetes-operator.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Improved notification messages

This commit is contained in:
Jakub Al-Khalili 2019-10-01 09:50:38 +02:00
parent bad8236104
commit b7c153f40c
11 changed files with 192 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/manager/main.go
View File

@ -13,6 +13,7 @@ import (
"github.com/jenkinsci/kubernetes-operator/pkg/apis"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/notifications"
"github.com/jenkinsci/kubernetes-operator/pkg/event"
"github.com/jenkinsci/kubernetes-operator/pkg/log"
"github.com/jenkinsci/kubernetes-operator/version"
@ -118,8 +119,11 @@ func main() {
fatal(errors.Wrap(err, "failed to create Kubernetes client set"), *debug)
}
c := make(chan notifications.Event)
go notifications.Listen(c, mgr.GetClient())
// setup Jenkins controller
if err := jenkins.Add(mgr, *local, *minikube, events, *clientSet, *cfg); err != nil {
if err := jenkins.Add(mgr, *local, *minikube, events, *clientSet, *cfg, &c); err != nil {
fatal(errors.Wrap(err, "failed to setup controllers"), *debug)
}

6
pkg/apis/jenkins/v1alpha2/jenkins_types.go
View File

@ -17,9 +17,9 @@ type JenkinsSpec struct {
// +optional
SeedJobs []SeedJob `json:"seedJobs,omitempty"`
/* // Notifications defines list of a services which are used to inform about Jenkins status
// Can be used to integrate chat services like Slack, Microsoft MicrosoftTeams or Mailgun
Notifications []Notification `json:"notifications,omitempty"`*/
// Notifications defines list of a services which are used to inform about Jenkins status
// Can be used to integrate chat services like Slack, Microsoft Teams or Mailgun
Notifications []Notification `json:"notifications,omitempty"`
// Service is Kubernetes service of Jenkins master HTTP pod
// Defaults to :

7
pkg/apis/jenkins/v1alpha2/zz_generated.deepcopy.go
View File

@ -342,6 +342,13 @@ func (in *JenkinsSpec) DeepCopyInto(out *JenkinsSpec) {
*out = make([]SeedJob, len(*in))
copy(*out, *in)
}
if in.Notifications != nil {
in, out := &in.Notifications, &out.Notifications
*out = make([]Notification, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
in.Service.DeepCopyInto(&out.Service)
in.SlaveService.DeepCopyInto(&out.SlaveService)
in.Backup.DeepCopyInto(&out.Backup)

1
pkg/controller/jenkins/configuration/base/reconcile.go
View File

@ -124,6 +124,7 @@ func (r *ReconcileJenkinsBaseConfiguration) Reconcile() (reconcile.Result, jenki
}
result, err = r.ensureBaseConfiguration(jenkinsClient)
return result, jenkinsClient, err
}

108
pkg/controller/jenkins/jenkins_controller.go
View File

@ -3,6 +3,7 @@ package jenkins
import (
"context"
"fmt"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/notifications"
"reflect"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
@ -52,20 +53,21 @@ var reconcileErrors = map[string]reconcileError{}
// Add creates a new Jenkins Controller and adds it to the Manager. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config) error {
return add(mgr, newReconciler(mgr, local, minikube, events, clientSet, config))
func Add(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config, notificationEvents *chan notifications.Event) error {
return add(mgr, newReconciler(mgr, local, minikube, events, clientSet, config, notificationEvents))
}
// newReconciler returns a new reconcile.Reconciler
func newReconciler(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config) reconcile.Reconciler {
func newReconciler(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config, notificationEvents *chan notifications.Event) reconcile.Reconciler {
return &ReconcileJenkins{
client: mgr.GetClient(),
scheme: mgr.GetScheme(),
local: local,
minikube: minikube,
events: events,
clientSet: clientSet,
config: config,
client: mgr.GetClient(),
scheme: mgr.GetScheme(),
local: local,
minikube: minikube,
events: events,
clientSet: clientSet,
config: config,
notificationEvents: notificationEvents,
}
}
@ -119,19 +121,34 @@ var _ reconcile.Reconciler = &ReconcileJenkins{}
// ReconcileJenkins reconciles a Jenkins object
type ReconcileJenkins struct {
client client.Client
scheme *runtime.Scheme
local, minikube bool
events event.Recorder
clientSet kubernetes.Clientset
config rest.Config
client client.Client
scheme *runtime.Scheme
local, minikube bool
events event.Recorder
clientSet kubernetes.Clientset
config rest.Config
notificationEvents *chan notifications.Event
}
// Reconcile it's a main reconciliation loop which maintain desired state based on Jenkins.Spec
func (r *ReconcileJenkins) Reconcile(request reconcile.Request) (reconcile.Result, error) {
reconcileFailLimit := uint64(10)
logger := r.buildLogger(request.Name)
logger.V(log.VDebug).Info("Reconciling Jenkins")
jenkins := &v1alpha2.Jenkins{}
err := r.client.Get(context.TODO(), request.NamespacedName, jenkins)
if err != nil {
if apierrors.IsNotFound(err) {
// Request object not found, could have been deleted after reconcile request.
// Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
// Return and don't requeue
return reconcile.Result{}, nil
}
// Error reading the object - requeue the request.
return reconcile.Result{}, errors.WithStack(err)
}
result, err := r.reconcile(request, logger)
if err != nil && apierrors.IsConflict(err) {
return reconcile.Result{Requeue: true}, nil
@ -151,11 +168,19 @@ func (r *ReconcileJenkins) Reconcile(request reconcile.Request) (reconcile.Resul
}
}
reconcileErrors[request.Name] = lastErrors
if lastErrors.counter >= 15 {
if lastErrors.counter >= reconcileFailLimit {
if log.Debug {
logger.V(log.VWarn).Info(fmt.Sprintf("Reconcile loop failed ten times with the same error, giving up: %+v", err))
logger.V(log.VWarn).Info(fmt.Sprintf("Reconcile loop failed %d times with the same error, giving up: %+v", reconcileFailLimit, err))
} else {
logger.V(log.VWarn).Info(fmt.Sprintf("Reconcile loop failed ten times with the same error, giving up: %s", err))
logger.V(log.VWarn).Info(fmt.Sprintf("Reconcile loop failed %d times with the same error, giving up: %s", reconcileFailLimit, err))
}
*r.notificationEvents <- notifications.Event{
Jenkins: *jenkins,
ConfigurationType: notifications.ConfigurationTypeUnknown,
LogLevel: v1alpha2.NotificationLogLevelWarning,
Message: fmt.Sprintf("Reconcile loop failed ten times with the same error, giving up: %s", err),
MessageVerbose: fmt.Sprintf("Reconcile loop failed ten times with the same error, giving up: %+v", err),
}
return reconcile.Result{Requeue: false}, nil
}
@ -203,8 +228,16 @@ func (r *ReconcileJenkins) reconcile(request reconcile.Request, logger logr.Logg
return reconcile.Result{}, err
}
if !valid {
message := "Validation of base configuration failed, please correct Jenkins CR"
*r.notificationEvents <- notifications.Event{
Jenkins: *jenkins,
ConfigurationType: notifications.ConfigurationTypeBase,
LogLevel: v1alpha2.NotificationLogLevelWarning,
Message: message,
MessageVerbose: message,
}
r.events.Emit(jenkins, event.TypeWarning, reasonCRValidationFailure, "Base CR validation failed")
logger.V(log.VWarn).Info("Validation of base configuration failed, please correct Jenkins CR")
logger.V(log.VWarn).Info(message)
return reconcile.Result{}, nil // don't requeue
}
@ -226,8 +259,17 @@ func (r *ReconcileJenkins) reconcile(request reconcile.Request, logger logr.Logg
if err != nil {
return reconcile.Result{}, errors.WithStack(err)
}
logger.Info(fmt.Sprintf("Base configuration phase is complete, took %s",
jenkins.Status.BaseConfigurationCompletedTime.Sub(jenkins.Status.ProvisionStartTime.Time)))
message := fmt.Sprintf("Base configuration phase is complete, took %s",
jenkins.Status.BaseConfigurationCompletedTime.Sub(jenkins.Status.ProvisionStartTime.Time))
*r.notificationEvents <- notifications.Event{
Jenkins: *jenkins,
ConfigurationType: notifications.ConfigurationTypeBase,
LogLevel: v1alpha2.NotificationLogLevelInfo,
Message: message,
MessageVerbose: message,
}
logger.Info(message)
r.events.Emit(jenkins, event.TypeNormal, reasonBaseConfigurationSuccess, "Base configuration completed")
}
// Reconcile user configuration
@ -238,7 +280,15 @@ func (r *ReconcileJenkins) reconcile(request reconcile.Request, logger logr.Logg
return reconcile.Result{}, err
}
if !valid {
logger.V(log.VWarn).Info("Validation of user configuration failed, please correct Jenkins CR")
message := fmt.Sprintf("Validation of user configuration failed, please correct Jenkins CR")
*r.notificationEvents <- notifications.Event{
Jenkins: *jenkins,
ConfigurationType: notifications.ConfigurationTypeUser,
LogLevel: v1alpha2.NotificationLogLevelWarning,
Message: message,
MessageVerbose: message,
}
logger.V(log.VWarn).Info(message)
r.events.Emit(jenkins, event.TypeWarning, reasonCRValidationFailure, "User CR validation failed")
return reconcile.Result{}, nil // don't requeue
}
@ -258,8 +308,16 @@ func (r *ReconcileJenkins) reconcile(request reconcile.Request, logger logr.Logg
if err != nil {
return reconcile.Result{}, errors.WithStack(err)
}
logger.Info(fmt.Sprintf("User configuration phase is complete, took %s",
jenkins.Status.UserConfigurationCompletedTime.Sub(jenkins.Status.ProvisionStartTime.Time)))
message := fmt.Sprintf("User configuration phase is complete, took %s",
jenkins.Status.UserConfigurationCompletedTime.Sub(jenkins.Status.ProvisionStartTime.Time))
*r.notificationEvents <- notifications.Event{
Jenkins: *jenkins,
ConfigurationType: notifications.ConfigurationTypeUser,
LogLevel: v1alpha2.NotificationLogLevelInfo,
Message: message,
MessageVerbose: message,
}
logger.Info(message)
r.events.Emit(jenkins, event.TypeNormal, reasonUserConfigurationSuccess, "User configuration completed")
}

20
pkg/controller/jenkins/notifications/mailgun.go
View File

@ -20,8 +20,8 @@ const content = `
<html>
<head></head>
<body>
<h1 style="background-color: %s; color: white; padding: 3px 10px;">Jenkins Operator Reconciled</h1>
<h3>Failed to do something</h3>
<h1 style="background-color: %s; color: white; padding: 3px 10px;">%s</h1>
<h3>%s</h3>
<table>
<tr>
<td><b>CR name:</b></td>
@ -31,10 +31,6 @@ const content = `
<td><b>Configuration type:</b></td>
<td>%s</td>
</tr>
<tr>
<td><b>Status:</b></td>
<td><b style="color: %s;">%s</b></td>
</tr>
</table>
<h6 style="font-size: 11px; color: grey; margin-top: 15px;">Powered by Jenkins Operator <3</h6>
</body>
@ -74,9 +70,17 @@ func (m MailGun) Send(event Event, config v1alpha2.Notification) error {
mg := mailgun.NewMailgun(config.Mailgun.Domain, secretValue)
htmlMessage := fmt.Sprintf(content, m.getStatusColor(event.LogLevel), event.Jenkins.Name, event.ConfigurationType, m.getStatusColor(event.LogLevel), string(event.LogLevel))
var statusMessage string
msg := mg.NewMessage(fmt.Sprintf("Jenkins Operator Notifier <%s>", config.Mailgun.From), "Jenkins Operator Status", "", config.Mailgun.Recipient)
if config.Verbose {
statusMessage = event.MessageVerbose
} else {
statusMessage = event.Message
}
htmlMessage := fmt.Sprintf(content, m.getStatusColor(event.LogLevel), statusMessage, event.Jenkins.Name, event.ConfigurationType, m.getStatusColor(event.LogLevel))
msg := mg.NewMessage(fmt.Sprintf("Jenkins Operator Notifier <%s>", config.Mailgun.From), notificationTitle(event), "", config.Mailgun.Recipient)
msg.SetHtml(htmlMessage)
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()

35
pkg/controller/jenkins/notifications/msteams.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
@ -26,6 +27,7 @@ type TeamsMessage struct {
ThemeColor StatusColor `json:"themeColor"`
Title string `json:"title"`
Sections []TeamsSection `json:"sections"`
Summary string `json:"summary"`
}
// TeamsSection is MS Teams message section
@ -67,11 +69,10 @@ func (t Teams) Send(event Event, config v1alpha2.Notification) error {
return errors.Errorf("Microsoft Teams webhook URL is empty in secret '%s/%s[%s]", event.Jenkins.Namespace, selector.Name, selector.Key)
}
msg, err := json.Marshal(TeamsMessage{
tm := &TeamsMessage{
Type: "MessageCard",
Context: "https://schema.org/extensions",
ThemeColor: t.getStatusColor(event.LogLevel),
Title: titleText,
Sections: []TeamsSection{
{
Facts: []TeamsFact{
@ -79,14 +80,6 @@ func (t Teams) Send(event Event, config v1alpha2.Notification) error {
Name: crNameFieldName,
Value: event.Jenkins.Name,
},
{
Name: configurationTypeFieldName,
Value: event.ConfigurationType,
},
{
Name: loggingLevelFieldName,
Value: string(event.LogLevel),
},
{
Name: namespaceFieldName,
Value: event.Jenkins.Namespace,
@ -95,7 +88,24 @@ func (t Teams) Send(event Event, config v1alpha2.Notification) error {
Text: event.Message,
},
},
})
Summary: event.Message,
}
tm.Title = notificationTitle(event)
if config.Verbose {
tm.Sections[0].Text = event.MessageVerbose
tm.Summary = event.MessageVerbose
}
if event.ConfigurationType != ConfigurationTypeUnknown {
tm.Sections[0].Facts = append(tm.Sections[0].Facts, TeamsFact{
Name: configurationTypeFieldName,
Value: event.ConfigurationType,
})
}
msg, err := json.Marshal(tm)
if err != nil {
return errors.WithStack(err)
}
@ -110,6 +120,9 @@ func (t Teams) Send(event Event, config v1alpha2.Notification) error {
return errors.WithStack(err)
}
if resp.StatusCode != http.StatusOK {
return errors.New(fmt.Sprintf("Invalid response from server: %s", resp.Status))
}
defer func() { _ = resp.Body.Close() }()
return nil

2
pkg/controller/jenkins/notifications/msteams_test.go
View File

@ -42,7 +42,7 @@ func TestTeams_Send(t *testing.T) {
t.Fatal(err)
}
assert.Equal(t, message.Title, titleText)
assert.Equal(t, message.Title, notificationTitle(event))
assert.Equal(t, message.ThemeColor, teams.getStatusColor(event.LogLevel))
mainSection := message.Sections[0]

36
pkg/controller/jenkins/notifications/sender.go
View File

@ -1,13 +1,19 @@
package notifications
import (
"fmt"
"net/http"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
"github.com/jenkinsci/kubernetes-operator/pkg/log"
"github.com/pkg/errors"
k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
)
const (
titleText = "Operator reconciled."
infoTitleText = "Jenkins Operator reconciliation info"
warnTitleText = "Jenkins Operator reconciliation warning"
messageFieldName = "Message"
loggingLevelFieldName = "Logging Level"
crNameFieldName = "CR Name"
@ -16,6 +22,17 @@ const (
footerContent = "Powered by Jenkins Operator"
)
const (
// ConfigurationTypeBase is core configuration of Jenkins provided by the Operator
ConfigurationTypeBase = "base"
// ConfigurationTypeUser is user-defined configuration of Jenkins
ConfigurationTypeUser = "user"
// ConfigurationTypeUnknown is untraceable type of configuration
ConfigurationTypeUnknown = "unknown"
)
var (
testConfigurationType = "test-configuration"
testCrName = "test-cr"
@ -42,7 +59,7 @@ type Event struct {
MessageVerbose string
}
/*type service interface {
type service interface {
Send(event Event, notificationConfig v1alpha2.Notification) error
}
@ -61,7 +78,7 @@ func Listen(events chan Event, k8sClient k8sclient.Client) {
} else if notificationConfig.Mailgun != nil {
svc = MailGun{k8sClient: k8sClient}
} else {
logger.V(log.VWarn).Info(fmt.Sprintf("Unexpected notification `%+v`", notificationConfig))
logger.V(log.VWarn).Info(fmt.Sprintf("Unknown notification service `%+v`", notificationConfig))
continue
}
@ -85,6 +102,15 @@ func notify(svc service, event Event, manifest v1alpha2.Notification) error {
if event.LogLevel == v1alpha2.NotificationLogLevelInfo && manifest.LoggingLevel == v1alpha2.NotificationLogLevelWarning {
return nil
}
return svc.Send(event, manifest)
}*/
}
func notificationTitle(event Event) string {
if event.LogLevel == v1alpha2.NotificationLogLevelInfo {
return infoTitleText
} else if event.LogLevel == v1alpha2.NotificationLogLevelWarning {
return warnTitleText
} else {
return "undefined"
}
}

41
pkg/controller/jenkins/notifications/slack.go
View File

@ -64,12 +64,11 @@ func (s Slack) Send(event Event, config v1alpha2.Notification) error {
return err
}
slackMessage, err := json.Marshal(SlackMessage{
sm := &SlackMessage{
Attachments: []SlackAttachment{
{
Fallback: "",
Color: s.getStatusColor(event.LogLevel),
Text: titleText,
Fields: []SlackField{
{
Title: messageFieldName,
@ -81,16 +80,6 @@ func (s Slack) Send(event Event, config v1alpha2.Notification) error {
Value: event.Jenkins.Name,
Short: true,
},
{
Title: configurationTypeFieldName,
Value: event.ConfigurationType,
Short: true,
},
{
Title: loggingLevelFieldName,
Value: string(event.LogLevel),
Short: true,
},
{
Title: namespaceFieldName,
Value: event.Jenkins.Namespace,
@ -100,17 +89,35 @@ func (s Slack) Send(event Event, config v1alpha2.Notification) error {
Footer: footerContent,
},
},
})
}
mainAttachment := sm.Attachments[0]
mainAttachment.Title = notificationTitle(event)
if config.Verbose {
// TODO: or for title == message
mainAttachment.Fields[0].Value = event.MessageVerbose
}
if event.ConfigurationType != ConfigurationTypeUnknown {
mainAttachment.Fields = append(mainAttachment.Fields, SlackField{
Title: configurationTypeFieldName,
Value: event.ConfigurationType,
Short: true,
})
}
slackMessage, err := json.Marshal(sm)
if err != nil {
return err
}
secretValue := string(secret.Data[selector.Key])
if secretValue == "" {
return errors.Errorf("SecretValue %s is empty", selector.Name)
}
if err != nil {
return err
}
request, err := http.NewRequest("POST", secretValue, bytes.NewBuffer(slackMessage))
if err != nil {
return err

2
pkg/controller/jenkins/notifications/slack_test.go
View File

@ -45,7 +45,7 @@ func TestSlack_Send(t *testing.T) {
mainAttachment := message.Attachments[0]
assert.Equal(t, mainAttachment.Text, titleText)
assert.Equal(t, mainAttachment.Title, notificationTitle(event))
for _, field := range mainAttachment.Fields {
switch field.Title {
case configurationTypeFieldName: