diff --git a/pkg/apis/virtuslab/v1alpha1/jenkins_types.go b/pkg/apis/virtuslab/v1alpha1/jenkins_types.go index d9f4a33f..a3738130 100644 --- a/pkg/apis/virtuslab/v1alpha1/jenkins_types.go +++ b/pkg/apis/virtuslab/v1alpha1/jenkins_types.go @@ -51,6 +51,7 @@ type JenkinsMaster struct { type JenkinsStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster // Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file + BackupRestored bool `json:"backupRestored,omitempty"` BaseConfigurationCompletedTime *metav1.Time `json:"baseConfigurationCompletedTime,omitempty"` UserConfigurationCompletedTime *metav1.Time `json:"userConfigurationCompletedTime,omitempty"` Builds []Build `json:"builds,omitempty"` diff --git a/pkg/controller/jenkins/backup/backup.go b/pkg/controller/jenkins/backup/backup.go new file mode 100644 index 00000000..895f14d4 --- /dev/null +++ b/pkg/controller/jenkins/backup/backup.go @@ -0,0 +1,160 @@ +package backup + +import ( + "context" + "fmt" + "time" + + virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/backup/nobackup" + jenkinsclient "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/client" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/base/resources" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/constants" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/jobs" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/plugins" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + k8s "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + restoreJobName = constants.OperatorName + "-restore-backup" +) + +// Provider defines API of backup providers +type Provider interface { + GetRestoreJobXML(jenkins virtuslabv1alpha1.Jenkins) (string, error) + GetBackupJobXML(jenkins virtuslabv1alpha1.Jenkins) (string, error) + IsConfigurationValidForBasePhase(jenkins virtuslabv1alpha1.Jenkins, logger logr.Logger) bool + IsConfigurationValidForUserPhase(k8sClient k8s.Client, jenkins virtuslabv1alpha1.Jenkins, logger logr.Logger) (bool, error) + GetRequiredPlugins() map[string][]plugins.Plugin +} + +// Backup defines backup manager which is responsible of backup jobs history +type Backup struct { + jenkins *virtuslabv1alpha1.Jenkins + k8sClient k8s.Client + logger logr.Logger + jenkinsClient jenkinsclient.Jenkins +} + +// New returns instance of backup manager +func New(jenkins *virtuslabv1alpha1.Jenkins, k8sClient k8s.Client, logger logr.Logger, jenkinsClient jenkinsclient.Jenkins) *Backup { + return &Backup{jenkins: jenkins, k8sClient: k8sClient, logger: logger, jenkinsClient: jenkinsClient} +} + +// EnsureRestoreJob creates and updates Jenkins job used to restore backup +func (b *Backup) EnsureRestoreJob() error { + if b.jenkins.Status.UserConfigurationCompletedTime == nil { + provider, err := GetBackupProvider(b.jenkins.Spec.Backup) + if err != nil { + return err + } + restoreJobXML, err := provider.GetRestoreJobXML(*b.jenkins) + if err != nil { + return err + } + _, created, err := b.jenkinsClient.CreateOrUpdateJob(restoreJobXML, restoreJobName) + if err != nil { + return err + } + if created { + b.logger.Info(fmt.Sprintf("'%s' job has been created", restoreJobName)) + } + + return nil + } + + return nil +} + +// RestoreBackup restores backup +func (b *Backup) RestoreBackup() (reconcile.Result, error) { + if !b.jenkins.Status.BackupRestored && b.jenkins.Status.UserConfigurationCompletedTime == nil { + jobsClient := jobs.New(b.jenkinsClient, b.k8sClient, b.logger) + + hash := "hash-restore" // it can be hardcoded because restore job can be run only once + done, err := jobsClient.EnsureBuildJob(restoreJobName, hash, map[string]string{}, b.jenkins, true) + if err != nil { + // build failed and can be recovered - retry build and requeue reconciliation loop with timeout + if err == jobs.ErrorBuildFailed { + return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10}, nil + } + // build failed and cannot be recovered + if err == jobs.ErrorUnrecoverableBuildFailed { + b.logger.Info(fmt.Sprintf("Restore backup can not be performed. Please check backup configuration in CR and credentials in secret '%s'.", resources.GetBackupCredentialsSecretName(b.jenkins))) + b.logger.Info(fmt.Sprintf("You can also check '%s' job logs in Jenkins", constants.BackupJobName)) + return reconcile.Result{}, nil + } + // unexpected error - requeue reconciliation loop + return reconcile.Result{}, err + } + // build not finished yet - requeue reconciliation loop with timeout + if !done { + return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10}, nil + } + + b.jenkins.Status.BackupRestored = true + err = b.k8sClient.Update(context.TODO(), b.jenkins) + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +// EnsureBackupJob creates and updates Jenkins job used to backup +func (b *Backup) EnsureBackupJob() error { + provider, err := GetBackupProvider(b.jenkins.Spec.Backup) + if err != nil { + return err + } + backupJobXML, err := provider.GetBackupJobXML(*b.jenkins) + if err != nil { + return err + } + _, created, err := b.jenkinsClient.CreateOrUpdateJob(backupJobXML, constants.BackupJobName) + if err != nil { + return err + } + if created { + b.logger.Info(fmt.Sprintf("'%s' job has been created", constants.BackupJobName)) + } + + return nil +} + +// GetBackupProvider returns backup provider by type +func GetBackupProvider(backupType virtuslabv1alpha1.JenkinsBackup) (Provider, error) { + switch backupType { + case virtuslabv1alpha1.JenkinsBackupTypeNoBackup: + return &nobackup.NoBackup{}, nil + default: + return nil, errors.Errorf("Invalid BackupManager type '%s'", backupType) + } +} + +// GetPluginsRequiredByAllBackupProviders returns plugins required by all backup providers +func GetPluginsRequiredByAllBackupProviders() map[string][]plugins.Plugin { + allPlugins := map[string][]plugins.Plugin{} + for _, provider := range getAllProviders() { + for key, value := range provider.GetRequiredPlugins() { + allPlugins[key] = func() []plugins.Plugin { + var pluginsNameWithVersion []plugins.Plugin + for _, plugin := range value { + pluginsNameWithVersion = append(pluginsNameWithVersion, plugin) + } + return pluginsNameWithVersion + }() + } + } + + return allPlugins +} + +func getAllProviders() []Provider { + return []Provider{ + &nobackup.NoBackup{}, + } +} diff --git a/pkg/controller/jenkins/backup/nobackup/nobackup.go b/pkg/controller/jenkins/backup/nobackup/nobackup.go new file mode 100644 index 00000000..04f2eee3 --- /dev/null +++ b/pkg/controller/jenkins/backup/nobackup/nobackup.go @@ -0,0 +1,52 @@ +package nobackup + +import ( + virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/plugins" + + "github.com/go-logr/logr" + k8s "sigs.k8s.io/controller-runtime/pkg/client" +) + +// NoBackup is a backup strategy where there is no backup +type NoBackup struct{} + +var emptyJob = ` + + + + false + + + + false + + + false + +` + +// GetRestoreJobXML returns Jenkins restore backup job config XML +func (b *NoBackup) GetRestoreJobXML(jenkins virtuslabv1alpha1.Jenkins) (string, error) { + return emptyJob, nil +} + +// GetBackupJobXML returns Jenkins backup job config XML +func (b *NoBackup) GetBackupJobXML(jenkins virtuslabv1alpha1.Jenkins) (string, error) { + return emptyJob, nil +} + +// IsConfigurationValidForBasePhase validates if user provided valid configuration of backup for base phase +func (b *NoBackup) IsConfigurationValidForBasePhase(jenkins virtuslabv1alpha1.Jenkins, logger logr.Logger) bool { + return true +} + +// IsConfigurationValidForUserPhase validates if user provided valid configuration of backup for user phase +func (b *NoBackup) IsConfigurationValidForUserPhase(k8sClient k8s.Client, jenkins virtuslabv1alpha1.Jenkins, logger logr.Logger) (bool, error) { + return true, nil +} + +// GetRequiredPlugins returns all required Jenkins plugins by this backup strategy +func (b *NoBackup) GetRequiredPlugins() map[string][]plugins.Plugin { + return map[string][]plugins.Plugin{} +} diff --git a/pkg/controller/jenkins/configuration/base/reconcile.go b/pkg/controller/jenkins/configuration/base/reconcile.go index e6035b99..95ec1470 100644 --- a/pkg/controller/jenkins/configuration/base/reconcile.go +++ b/pkg/controller/jenkins/configuration/base/reconcile.go @@ -7,6 +7,7 @@ import ( "time" virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/backup" jenkinsclient "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/client" "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/base/resources" "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/constants" @@ -61,7 +62,16 @@ func (r *ReconcileJenkinsBaseConfiguration) Reconcile() (reconcile.Result, jenki return reconcile.Result{}, nil, err } - result, err := r.ensureJenkinsMasterPod(metaObject) + pluginsRequiredByAllBackupProviders := backup.GetPluginsRequiredByAllBackupProviders() + result, err := r.ensurePluginsRequiredByAllBackupProviders(pluginsRequiredByAllBackupProviders) + if err != nil { + return reconcile.Result{}, nil, err + } + if result.Requeue { + return result, nil, nil + } + + result, err = r.ensureJenkinsMasterPod(metaObject) if err != nil { return reconcile.Result{}, nil, err } @@ -85,7 +95,7 @@ func (r *ReconcileJenkinsBaseConfiguration) Reconcile() (reconcile.Result, jenki } r.logger.V(log.VDebug).Info("Jenkins API client set") - ok, err := r.verifyBasePlugins(jenkinsClient) + ok, err := r.verifyPlugins(jenkinsClient, plugins.BasePluginsMap, pluginsRequiredByAllBackupProviders) if err != nil { return reconcile.Result{}, nil, err } @@ -142,7 +152,7 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureResourcesRequiredForJenkinsPod return nil } -func (r *ReconcileJenkinsBaseConfiguration) verifyBasePlugins(jenkinsClient jenkinsclient.Jenkins) (bool, error) { +func (r *ReconcileJenkinsBaseConfiguration) verifyPlugins(jenkinsClient jenkinsclient.Jenkins, allRequiredPlugins ...map[string][]plugins.Plugin) (bool, error) { allPluginsInJenkins, err := jenkinsClient.GetPlugins(fetchAllPlugins) if err != nil { return false, err @@ -157,17 +167,19 @@ func (r *ReconcileJenkinsBaseConfiguration) verifyBasePlugins(jenkinsClient jenk r.logger.V(log.VDebug).Info(fmt.Sprintf("Installed plugins '%+v'", installedPlugins)) status := true - for rootPluginName, p := range plugins.BasePluginsMap { - rootPlugin, _ := plugins.New(rootPluginName) - if found, ok := isPluginInstalled(allPluginsInJenkins, *rootPlugin); !ok { - r.logger.V(log.VWarn).Info(fmt.Sprintf("Missing plugin '%s', actual '%+v'", rootPlugin, found)) - status = false - } - for _, requiredPlugin := range p { - if found, ok := isPluginInstalled(allPluginsInJenkins, requiredPlugin); !ok { - r.logger.V(log.VWarn).Info(fmt.Sprintf("Missing plugin '%s', actual '%+v'", requiredPlugin, found)) + for _, requiredPlugins := range allRequiredPlugins { + for rootPluginName, p := range requiredPlugins { + rootPlugin, _ := plugins.New(rootPluginName) + if found, ok := isPluginInstalled(allPluginsInJenkins, *rootPlugin); !ok { + r.logger.V(log.VWarn).Info(fmt.Sprintf("Missing plugin '%s', actual '%+v'", rootPlugin, found)) status = false } + for _, requiredPlugin := range p { + if found, ok := isPluginInstalled(allPluginsInJenkins, requiredPlugin); !ok { + r.logger.V(log.VWarn).Info(fmt.Sprintf("Missing plugin '%s', actual '%+v'", requiredPlugin, found)) + status = false + } + } } } @@ -488,3 +500,28 @@ func (r *ReconcileJenkinsBaseConfiguration) verifyLabelsForWatchedResource(objec return true } + +func (r *ReconcileJenkinsBaseConfiguration) ensurePluginsRequiredByAllBackupProviders(requiredPlugins map[string][]plugins.Plugin) (reconcile.Result, error) { + copiedPlugins := map[string][]string{} + for key, value := range r.jenkins.Spec.Master.Plugins { + copiedPlugins[key] = value + } + for key, value := range requiredPlugins { + copiedPlugins[key] = func() []string { + var pluginsWithVersion []string + for _, plugin := range value { + pluginsWithVersion = append(pluginsWithVersion, plugin.String()) + } + return pluginsWithVersion + }() + } + + if !reflect.DeepEqual(r.jenkins.Spec.Master.Plugins, copiedPlugins) { + r.logger.Info("Adding plugins required by backup providers to '.spec.master.plugins'") + r.jenkins.Spec.Master.Plugins = copiedPlugins + err := r.k8sClient.Update(context.TODO(), r.jenkins) + return reconcile.Result{Requeue: true}, err + } + + return reconcile.Result{}, nil +} diff --git a/pkg/controller/jenkins/configuration/base/reconcile_test.go b/pkg/controller/jenkins/configuration/base/reconcile_test.go new file mode 100644 index 00000000..20854cae --- /dev/null +++ b/pkg/controller/jenkins/configuration/base/reconcile_test.go @@ -0,0 +1,116 @@ +package base + +import ( + "context" + "testing" + + virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/plugins" + + "github.com/stretchr/testify/assert" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" +) + +func TestReconcileJenkinsBaseConfiguration_ensurePluginsRequiredByAllBackupProviders(t *testing.T) { + tests := []struct { + name string + jenkins *virtuslabv1alpha1.Jenkins + requiredPlugins map[string][]plugins.Plugin + want reconcile.Result + wantErr bool + }{ + { + name: "happy, no required plugins", + jenkins: &virtuslabv1alpha1.Jenkins{ + Spec: virtuslabv1alpha1.JenkinsSpec{ + Master: virtuslabv1alpha1.JenkinsMaster{ + Plugins: map[string][]string{ + "first-plugin:0.0.1": {"second-plugin:0.0.1"}, + }, + }, + }, + }, + want: reconcile.Result{Requeue: false}, + wantErr: false, + }, + { + name: "happy, required plugins are set", + jenkins: &virtuslabv1alpha1.Jenkins{ + Spec: virtuslabv1alpha1.JenkinsSpec{ + Master: virtuslabv1alpha1.JenkinsMaster{ + Plugins: map[string][]string{ + "first-plugin:0.0.1": {"second-plugin:0.0.1"}, + }, + }, + }, + }, + requiredPlugins: map[string][]plugins.Plugin{ + "first-plugin:0.0.1": {plugins.Must(plugins.New("second-plugin:0.0.1"))}, + }, + want: reconcile.Result{Requeue: false}, + wantErr: false, + }, + { + name: "happy, jenkins CR must be updated", + jenkins: &virtuslabv1alpha1.Jenkins{ + Spec: virtuslabv1alpha1.JenkinsSpec{ + Master: virtuslabv1alpha1.JenkinsMaster{ + Plugins: map[string][]string{ + "first-plugin:0.0.1": {"second-plugin:0.0.1"}, + }, + }, + }, + }, + requiredPlugins: map[string][]plugins.Plugin{ + "first-plugin:0.0.1": {plugins.Must(plugins.New("second-plugin:0.0.1"))}, + "third-plugin:0.0.1": {}, + }, + want: reconcile.Result{Requeue: true}, + wantErr: false, + }, + { + name: "happy, jenkins CR must be updated", + jenkins: &virtuslabv1alpha1.Jenkins{ + Spec: virtuslabv1alpha1.JenkinsSpec{ + Master: virtuslabv1alpha1.JenkinsMaster{ + Plugins: map[string][]string{ + "first-plugin:0.0.1": {"second-plugin:0.0.1"}, + }, + }, + }, + }, + requiredPlugins: map[string][]plugins.Plugin{ + "first-plugin:0.0.1": {plugins.Must(plugins.New("second-plugin:0.0.1"))}, + "third-plugin:0.0.1": {plugins.Must(plugins.New("fourth-plugin:0.0.1"))}, + }, + want: reconcile.Result{Requeue: true}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := virtuslabv1alpha1.SchemeBuilder.AddToScheme(scheme.Scheme) + assert.NoError(t, err) + r := &ReconcileJenkinsBaseConfiguration{ + k8sClient: fake.NewFakeClient(), + scheme: nil, + logger: logf.ZapLogger(false), + jenkins: tt.jenkins, + local: false, + minikube: false, + } + err = r.k8sClient.Create(context.TODO(), tt.jenkins) + assert.NoError(t, err) + got, err := r.ensurePluginsRequiredByAllBackupProviders(tt.requiredPlugins) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/controller/jenkins/configuration/user/reconcile.go b/pkg/controller/jenkins/configuration/user/reconcile.go index 51eb48c5..10a0f545 100644 --- a/pkg/controller/jenkins/configuration/user/reconcile.go +++ b/pkg/controller/jenkins/configuration/user/reconcile.go @@ -6,6 +6,7 @@ import ( "time" virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/backup" jenkinsclient "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/client" "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/base/resources" "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/user/seedjobs" @@ -41,8 +42,12 @@ func New(k8sClient k8s.Client, jenkinsClient jenkinsclient.Jenkins, logger logr. // Reconcile it's a main reconciliation loop for user supplied configuration func (r *ReconcileUserConfiguration) Reconcile() (reconcile.Result, error) { - // reconcile seed jobs - result, err := r.ensureSeedJobs() + backupManager := backup.New(r.jenkins, r.k8sClient, r.logger, r.jenkinsClient) + if err := backupManager.EnsureRestoreJob(); err != nil { + return reconcile.Result{}, err + } + + result, err := backupManager.RestoreBackup() if err != nil { return reconcile.Result{}, err } @@ -50,7 +55,29 @@ func (r *ReconcileUserConfiguration) Reconcile() (reconcile.Result, error) { return result, nil } - return r.ensureUserConfiguration(r.jenkinsClient) + // reconcile seed jobs + result, err = r.ensureSeedJobs() + if err != nil { + return reconcile.Result{}, err + } + if result.Requeue { + return result, nil + } + + result, err = r.ensureUserConfiguration(r.jenkinsClient) + if err != nil { + return reconcile.Result{}, err + } + if result.Requeue { + return result, nil + } + + err = backupManager.EnsureBackupJob() + if err != nil { + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil } func (r *ReconcileUserConfiguration) ensureSeedJobs() (reconcile.Result, error) { diff --git a/pkg/controller/jenkins/constants/constants.go b/pkg/controller/jenkins/constants/constants.go index b8d36631..b42a6535 100644 --- a/pkg/controller/jenkins/constants/constants.go +++ b/pkg/controller/jenkins/constants/constants.go @@ -13,4 +13,6 @@ const ( BackupAmazonS3SecretAccessKey = "access-key" // BackupAmazonS3SecretSecretKey is the Amazon user secret key used to Amazon S3 backup BackupAmazonS3SecretSecretKey = "secret-key" + // BackupJobName is the Jenkins job name used to backup jobs history + BackupJobName = OperatorName + "-backup" )