#4 [WIP] Backup and restore

pkg/controller/jenkins/configuration/base/reconcile.go

@@ -360,9 +360,9 @@ func (r *ReconcileJenkinsBaseConfiguration) createService(meta metav1.ObjectMeta
 }
 
 func (r *ReconcileJenkinsBaseConfiguration) getJenkinsMasterPod(meta metav1.ObjectMeta) (*corev1.Pod, error) {
-	jenkinsMasterPod := resources.NewJenkinsMasterPod(meta, r.jenkins)
+	jenkinsMasterPodName := resources.GetJenkinsMasterPodName(*r.jenkins)
 	currentJenkinsMasterPod := &corev1.Pod{}
-	err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: jenkinsMasterPod.Name, Namespace: jenkinsMasterPod.Namespace}, currentJenkinsMasterPod)
+	err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: jenkinsMasterPodName, Namespace: r.jenkins.Namespace}, currentJenkinsMasterPod)
 	if err != nil {
 		return nil, err // don't wrap error
 	}
@@ -382,6 +382,8 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureJenkinsMasterPod(meta metav1.O
 		now := metav1.Now()
 		r.jenkins.Status = v1alpha2.JenkinsStatus{
 			ProvisionStartTime: &now,
+			LastBackup:         r.jenkins.Status.LastBackup,
+			PendingBackup:      r.jenkins.Status.LastBackup,
 		}
 		err = r.updateResource(r.jenkins)
 		if err != nil {
@@ -407,6 +409,11 @@ func isPodTerminating(pod corev1.Pod) bool {
 }
 
 func (r *ReconcileJenkinsBaseConfiguration) isRecreatePodNeeded(currentJenkinsMasterPod corev1.Pod) bool {
+	if r.jenkins.Spec.Restore.RecoveryOnce != 0 {
+		r.logger.Info(fmt.Sprintf("spec.restore.recoveryOnce is set, recreating pod"))
+		return true
+	}
+
 	if version.Version != r.jenkins.Status.OperatorVersion {
 		r.logger.Info(fmt.Sprintf("Jenkins Operator version has changed, actual '%+v' new '%+v', recreating pod",
 			r.jenkins.Status.OperatorVersion, version.Version))

pkg/controller/jenkins/configuration/base/resources/pod.go

@@ -14,7 +14,7 @@ const (
 	// JenkinsMasterContainerName is the Jenkins master container name in pod
 	JenkinsMasterContainerName = "jenkins-master"
 	// JenkinsHomeVolumeName is the Jenkins home volume name
-	JenkinsHomeVolumeName = "home"
+	JenkinsHomeVolumeName = "jenkins-home"
 	jenkinsPath           = "/var/jenkins"
 	jenkinsHomePath       = jenkinsPath + "/home"
 

pkg/controller/jenkins/configuration/user/backuprestore/backuprestore.go

@@ -0,0 +1,191 @@
+package backuprestore
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
+	jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
+	"github.com/jenkinsci/kubernetes-operator/pkg/log"
+
+	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/remotecommand"
+	k8s "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// BackupAndRestore represents Jenkins backup and restore client
+type BackupAndRestore struct {
+	config    rest.Config
+	k8sClient k8s.Client
+	clientSet kubernetes.Clientset
+
+	jenkinsClient jenkinsclient.Jenkins
+	logger        logr.Logger
+	jenkins       *v1alpha2.Jenkins
+}
+
+// New returns Jenkins backup and restore client
+func New(k8sClient k8s.Client, clientSet kubernetes.Clientset, jenkinsClient jenkinsclient.Jenkins,
+	logger logr.Logger, jenkins *v1alpha2.Jenkins, config rest.Config) *BackupAndRestore {
+	return &BackupAndRestore{k8sClient: k8sClient, clientSet: clientSet, jenkinsClient: jenkinsClient, logger: logger, jenkins: jenkins, config: config}
+}
+
+// Validate validates backup and restore configuration
+func (bar *BackupAndRestore) Validate() bool {
+	valid := true
+	allContainers := map[string]v1alpha2.Container{}
+	for _, container := range bar.jenkins.Spec.Master.Containers {
+		allContainers[container.Name] = container
+	}
+
+	restore := bar.jenkins.Spec.Restore
+	if len(restore.ContainerName) > 0 {
+		_, found := allContainers[restore.ContainerName]
+		if !found {
+			valid = false
+			bar.logger.V(log.VWarn).Info(fmt.Sprintf("restore container '%s' not found in CR spec.master.containers", restore.ContainerName))
+		}
+		if restore.Action.Exec == nil {
+			valid = false
+			bar.logger.V(log.VWarn).Info(fmt.Sprintf("spec.restore.action.exec is not configured"))
+		}
+	}
+
+	backup := bar.jenkins.Spec.Backup
+	if len(backup.ContainerName) > 0 {
+		_, found := allContainers[backup.ContainerName]
+		if !found {
+			valid = false
+			bar.logger.V(log.VWarn).Info(fmt.Sprintf("backup container '%s' not found in CR spec.master.containers", backup.ContainerName))
+		}
+		if backup.Action.Exec == nil {
+			valid = false
+			bar.logger.V(log.VWarn).Info(fmt.Sprintf("spec.backup.action.exec is not configured"))
+		}
+		if backup.Interval == 0 {
+			valid = false
+			bar.logger.V(log.VWarn).Info(fmt.Sprintf("spec.backup.interval is not configured"))
+		}
+	}
+
+	if len(restore.ContainerName) > 0 && len(backup.ContainerName) == 0 {
+		valid = false
+		bar.logger.V(log.VWarn).Info("spec.backup.containerName is not configured")
+	}
+	if len(backup.ContainerName) > 0 && len(restore.ContainerName) == 0 {
+		valid = false
+		bar.logger.V(log.VWarn).Info("spec.restore.containerName is not configured")
+	}
+
+	return valid
+}
+
+// Restore performs Jenkins restore backup operation
+func (bar *BackupAndRestore) Restore() error {
+	jenkins := bar.jenkins
+	if jenkins.Status.RestoredBackup != 0 {
+		bar.logger.V(log.VDebug).Info("Skipping restore backup, backup already restored")
+		return nil
+	}
+	if jenkins.Status.LastBackup == 0 {
+		bar.logger.Info("Skipping restore backup")
+		if jenkins.Status.PendingBackup == 0 {
+			jenkins.Status.PendingBackup = 1
+			return bar.k8sClient.Update(context.TODO(), jenkins)
+		}
+		return nil
+	}
+
+	var backupNumber uint64
+	if jenkins.Spec.Restore.RecoveryOnce != 0 {
+		backupNumber = jenkins.Spec.Restore.RecoveryOnce
+	} else {
+		backupNumber = jenkins.Status.LastBackup
+	}
+	bar.logger.Info(fmt.Sprintf("Restoring backup '%d'", backupNumber))
+	podName := resources.GetJenkinsMasterPodName(*jenkins)
+	command := jenkins.Spec.Restore.Action.Exec.Command
+	command = append(command, fmt.Sprintf("%d", backupNumber))
+	_, _, err := bar.exec(podName, jenkins.Spec.Restore.ContainerName, command)
+
+	if err == nil {
+		jenkins.Spec.Restore.RecoveryOnce = 0
+		jenkins.Status.RestoredBackup = backupNumber
+		jenkins.Status.PendingBackup = backupNumber + 1
+		return bar.k8sClient.Update(context.TODO(), jenkins)
+	}
+
+	//TODO reload?
+	//TODO after 3 fails stop
+
+	return err
+}
+
+// Backup performs Jenkins backup operation
+func (bar *BackupAndRestore) Backup() error {
+	jenkins := bar.jenkins
+	if jenkins.Status.PendingBackup == jenkins.Status.LastBackup {
+		bar.logger.V(log.VDebug).Info("Skipping backup")
+		return nil
+	}
+	backupNumber := jenkins.Status.PendingBackup
+	bar.logger.Info(fmt.Sprintf("Performing backup '%d'", backupNumber))
+	podName := resources.GetJenkinsMasterPodName(*jenkins)
+	command := jenkins.Spec.Backup.Action.Exec.Command
+	command = append(command, fmt.Sprintf("%d", backupNumber))
+	_, _, err := bar.exec(podName, jenkins.Spec.Backup.ContainerName, command)
+
+	if err == nil {
+		if jenkins.Status.RestoredBackup == 0 {
+			jenkins.Status.RestoredBackup = backupNumber
+		}
+		jenkins.Status.LastBackup = backupNumber
+		jenkins.Status.PendingBackup = backupNumber
+		return bar.k8sClient.Update(context.TODO(), jenkins)
+	}
+
+	//TODO after 3 fails stop
+
+	return err
+}
+
+func (bar *BackupAndRestore) exec(podName, containerName string, command []string) (stdout, stderr bytes.Buffer, err error) {
+	req := bar.clientSet.CoreV1().RESTClient().Post().
+		Resource("pods").
+		Name(podName).
+		Namespace(bar.jenkins.Namespace).
+		SubResource("exec")
+	req.VersionedParams(&corev1.PodExecOptions{
+		Command:   command,
+		Container: containerName,
+		Stdin:     false,
+		Stdout:    true,
+		Stderr:    true,
+		TTY:       false,
+	}, scheme.ParameterCodec)
+
+	exec, err := remotecommand.NewSPDYExecutor(&bar.config, "POST", req.URL())
+	if err != nil {
+		return stdout, stderr, errors.Wrap(err, "pod exec error while creating Executor")
+	}
+
+	err = exec.Stream(remotecommand.StreamOptions{
+		Stdin:  nil,
+		Stdout: &stdout,
+		Stderr: &stderr,
+		Tty:    false,
+	})
+	bar.logger.V(log.VDebug).Info(fmt.Sprintf("pod exec: stdout '%s' stderr '%s'", stdout.String(), stderr.String()))
+	if err != nil {
+		return stdout, stderr, errors.Wrapf(err, "pod exec error operation on stream: stdout '%s' stderr '%s'", stdout.String(), stderr.String())
+	}
+
+	return
+}

pkg/controller/jenkins/configuration/user/reconcile.go

@@ -7,6 +7,7 @@ import (
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
 	jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/backuprestore"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/casc"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/seedjobs"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
@@ -17,6 +18,8 @@ import (
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	k8s "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
@@ -27,22 +30,27 @@ type ReconcileUserConfiguration struct {
 	jenkinsClient jenkinsclient.Jenkins
 	logger        logr.Logger
 	jenkins       *v1alpha2.Jenkins
+	clientSet     kubernetes.Clientset
+	config        rest.Config
 }
 
 // New create structure which takes care of user configuration
 func New(k8sClient k8s.Client, jenkinsClient jenkinsclient.Jenkins, logger logr.Logger,
-	jenkins *v1alpha2.Jenkins) *ReconcileUserConfiguration {
+	jenkins *v1alpha2.Jenkins, clientSet kubernetes.Clientset, config rest.Config) *ReconcileUserConfiguration {
 	return &ReconcileUserConfiguration{
 		k8sClient:     k8sClient,
 		jenkinsClient: jenkinsClient,
 		logger:        logger,
 		jenkins:       jenkins,
+		clientSet:     clientSet,
+		config:        config,
 	}
 }
 
 // Reconcile it's a main reconciliation loop for user supplied configuration
 func (r *ReconcileUserConfiguration) Reconcile() (reconcile.Result, error) {
-	// reconcile seed jobs
+	backupAndRestore := backuprestore.New(r.k8sClient, r.clientSet, r.jenkinsClient, r.logger, r.jenkins, r.config)
+
 	result, err := r.ensureSeedJobs()
 	if err != nil {
 		return reconcile.Result{}, err
@@ -51,6 +59,10 @@ func (r *ReconcileUserConfiguration) Reconcile() (reconcile.Result, error) {
 		return result, nil
 	}
 
+	if err := backupAndRestore.Restore(); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	result, err = r.ensureUserConfiguration(r.jenkinsClient)
 	if err != nil {
 		return reconcile.Result{}, err
@@ -59,6 +71,11 @@ func (r *ReconcileUserConfiguration) Reconcile() (reconcile.Result, error) {
 		return result, nil
 	}
 
+	if err := backupAndRestore.Backup(); err != nil {
+		return reconcile.Result{}, err
+	}
+	//TODO backup Goroutine
+
 	return reconcile.Result{}, nil
 }
 

pkg/controller/jenkins/configuration/user/validate.go

@@ -2,11 +2,17 @@ package user
 
 import (
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/backuprestore"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/seedjobs"
 )
 
 // Validate validates Jenkins CR Spec section
 func (r *ReconcileUserConfiguration) Validate(jenkins *v1alpha2.Jenkins) (bool, error) {
+	backupAndRestore := backuprestore.New(r.k8sClient, r.clientSet, r.jenkinsClient, r.logger, r.jenkins, r.config)
+	if ok := backupAndRestore.Validate(); !ok {
+		return false, nil
+	}
+
 	seedJobs := seedjobs.New(r.jenkinsClient, r.k8sClient, r.logger)
 	return seedJobs.ValidateSeedJobs(*jenkins)
 }

pkg/controller/jenkins/jenkins_controller.go

@@ -23,6 +23,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -42,18 +44,20 @@ const (
 
 // Add creates a new Jenkins Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
-func Add(mgr manager.Manager, local, minikube bool, events event.Recorder) error {
-	return add(mgr, newReconciler(mgr, local, minikube, events))
+func Add(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config) error {
+	return add(mgr, newReconciler(mgr, local, minikube, events, clientSet, config))
 }
 
 // newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager, local, minikube bool, events event.Recorder) reconcile.Reconciler {
+func newReconciler(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config) reconcile.Reconciler {
 	return &ReconcileJenkins{
-		client:   mgr.GetClient(),
-		scheme:   mgr.GetScheme(),
-		local:    local,
-		minikube: minikube,
-		events:   events,
+		client:    mgr.GetClient(),
+		scheme:    mgr.GetScheme(),
+		local:     local,
+		minikube:  minikube,
+		events:    events,
+		clientSet: clientSet,
+		config:    config,
 	}
 }
 
@@ -103,6 +107,8 @@ type ReconcileJenkins struct {
 	scheme          *runtime.Scheme
 	local, minikube bool
 	events          event.Recorder
+	clientSet       kubernetes.Clientset
+	config          rest.Config
 }
 
 // Reconcile it's a main reconciliation loop which maintain desired state based on Jenkins.Spec
@@ -181,7 +187,7 @@ func (r *ReconcileJenkins) reconcile(request reconcile.Request, logger logr.Logg
 		r.events.Emit(jenkins, event.TypeNormal, reasonBaseConfigurationSuccess, "Base configuration completed")
 	}
 	// Reconcile user configuration
-	userConfiguration := user.New(r.client, jenkinsClient, logger, jenkins)
+	userConfiguration := user.New(r.client, jenkinsClient, logger, jenkins, r.clientSet, r.config)
 
 	valid, err = userConfiguration.Validate(jenkins)
 	if err != nil {
@@ -334,13 +340,25 @@ func (r *ReconcileJenkins) setDefaults(jenkins *v1alpha2.Jenkins, logger logr.Lo
 	}
 	if len(jenkins.Spec.Master.Containers) > 1 {
 		for i, container := range jenkins.Spec.Master.Containers[1:] {
-			if setDefaultsForContainer(jenkins, i, logger.WithValues("container", container.Name)) {
+			if setDefaultsForContainer(jenkins, i+1, logger.WithValues("container", container.Name)) {
 				changed = true
 			}
 		}
 	}
+	if len(jenkins.Spec.Backup.ContainerName) > 0 && jenkins.Spec.Backup.Interval == 0 {
+		logger.Info("Setting default backup interval")
+		changed = true
+		jenkins.Spec.Backup.Interval = 30
+	}
 
-	jenkins.Spec.Master.Containers = []v1alpha2.Container{jenkinsContainer}
+	if len(jenkins.Spec.Master.Containers) == 0 || len(jenkins.Spec.Master.Containers) == 1 {
+		jenkins.Spec.Master.Containers = []v1alpha2.Container{jenkinsContainer}
+	} else {
+		noJenkinsContainers := jenkins.Spec.Master.Containers[1:]
+		containers := []v1alpha2.Container{jenkinsContainer}
+		containers = append(containers, noJenkinsContainers...)
+		jenkins.Spec.Master.Containers = containers
+	}
 
 	if changed {
 		return errors.WithStack(r.client.Update(context.TODO(), jenkins))