#4 [WIP] Backup and restore

- add backup trigger
- fix recoveryOnce logic
This commit is contained in:
Tomasz Sęk 2019-06-16 22:40:41 +02:00
parent 144d0245f2
commit 2264e11a6d
No known key found for this signature in database
GPG Key ID: DC356D23F6A644D0
10 changed files with 210 additions and 35 deletions

View File

@ -9,6 +9,23 @@ import (
runtime "k8s.io/apimachinery/pkg/runtime"
)
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Backup) DeepCopyInto(out *Backup) {
*out = *in
in.Action.DeepCopyInto(&out.Action)
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Backup.
func (in *Backup) DeepCopy() *Backup {
if in == nil {
return nil
}
out := new(Backup)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Build) DeepCopyInto(out *Build) {
*out = *in
@ -106,6 +123,27 @@ func (in *Container) DeepCopy() *Container {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Handler) DeepCopyInto(out *Handler) {
*out = *in
if in.Exec != nil {
in, out := &in.Exec, &out.Exec
*out = new(v1.ExecAction)
(*in).DeepCopyInto(*out)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Handler.
func (in *Handler) DeepCopy() *Handler {
if in == nil {
return nil
}
out := new(Handler)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Jenkins) DeepCopyInto(out *Jenkins) {
*out = *in
@ -232,6 +270,8 @@ func (in *JenkinsSpec) DeepCopyInto(out *JenkinsSpec) {
}
in.Service.DeepCopyInto(&out.Service)
in.SlaveService.DeepCopyInto(&out.SlaveService)
in.Backup.DeepCopyInto(&out.Backup)
in.Restore.DeepCopyInto(&out.Restore)
return
}
@ -296,6 +336,23 @@ func (in *Plugin) DeepCopy() *Plugin {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Restore) DeepCopyInto(out *Restore) {
*out = *in
in.Action.DeepCopyInto(&out.Action)
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Restore.
func (in *Restore) DeepCopy() *Restore {
if in == nil {
return nil
}
out := new(Restore)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SeedJob) DeepCopyInto(out *SeedJob) {
*out = *in

View File

@ -96,11 +96,21 @@ func schema_pkg_apis_jenkins_v1alpha2_JenkinsSpec(ref common.ReferenceCallback)
Ref: ref("github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.Service"),
},
},
"backup": {
SchemaProps: spec.SchemaProps{
Ref: ref("github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.Backup"),
},
},
"restore": {
SchemaProps: spec.SchemaProps{
Ref: ref("github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.Restore"),
},
},
},
},
},
Dependencies: []string{
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.JenkinsMaster", "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.SeedJob", "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.Service"},
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.Backup", "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.JenkinsMaster", "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.Restore", "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.SeedJob", "github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.Service"},
}
}
@ -144,6 +154,24 @@ func schema_pkg_apis_jenkins_v1alpha2_JenkinsStatus(ref common.ReferenceCallback
},
},
},
"restoredBackup": {
SchemaProps: spec.SchemaProps{
Type: []string{"integer"},
Format: "int64",
},
},
"lastBackup": {
SchemaProps: spec.SchemaProps{
Type: []string{"integer"},
Format: "int64",
},
},
"pendingBackup": {
SchemaProps: spec.SchemaProps{
Type: []string{"integer"},
Format: "int64",
},
},
},
},
},

View File

@ -4,6 +4,7 @@ import (
"bytes"
"context"
"fmt"
"time"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
@ -13,6 +14,7 @@ import (
"github.com/go-logr/logr"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
@ -20,21 +22,27 @@ import (
k8s "sigs.k8s.io/controller-runtime/pkg/client"
)
type backupTrigger struct {
interval uint64
ticker *time.Ticker
}
var backupTriggers = map[string]backupTrigger{}
// BackupAndRestore represents Jenkins backup and restore client
type BackupAndRestore struct {
config rest.Config
k8sClient k8s.Client
clientSet kubernetes.Clientset
jenkinsClient jenkinsclient.Jenkins
logger logr.Logger
jenkins *v1alpha2.Jenkins
logger logr.Logger
jenkins *v1alpha2.Jenkins
}
// New returns Jenkins backup and restore client
func New(k8sClient k8s.Client, clientSet kubernetes.Clientset, jenkinsClient jenkinsclient.Jenkins,
func New(k8sClient k8s.Client, clientSet kubernetes.Clientset,
logger logr.Logger, jenkins *v1alpha2.Jenkins, config rest.Config) *BackupAndRestore {
return &BackupAndRestore{k8sClient: k8sClient, clientSet: clientSet, jenkinsClient: jenkinsClient, logger: logger, jenkins: jenkins, config: config}
return &BackupAndRestore{k8sClient: k8sClient, clientSet: clientSet, logger: logger, jenkins: jenkins, config: config}
}
// Validate validates backup and restore configuration
@ -88,7 +96,7 @@ func (bar *BackupAndRestore) Validate() bool {
}
// Restore performs Jenkins restore backup operation
func (bar *BackupAndRestore) Restore() error {
func (bar *BackupAndRestore) Restore(jenkinsClient jenkinsclient.Jenkins) error {
jenkins := bar.jenkins
if jenkins.Status.RestoredBackup != 0 {
bar.logger.V(log.VDebug).Info("Skipping restore backup, backup already restored")
@ -116,13 +124,17 @@ func (bar *BackupAndRestore) Restore() error {
_, _, err := bar.exec(podName, jenkins.Spec.Restore.ContainerName, command)
if err == nil {
_, err := jenkinsClient.ExecuteScript("Jenkins.instance.reload()")
if err != nil {
return err
}
jenkins.Spec.Restore.RecoveryOnce = 0
jenkins.Status.RestoredBackup = backupNumber
jenkins.Status.PendingBackup = backupNumber + 1
return bar.k8sClient.Update(context.TODO(), jenkins)
}
//TODO reload?
//TODO after 3 fails stop
return err
@ -156,6 +168,70 @@ func (bar *BackupAndRestore) Backup() error {
return err
}
func triggerBackup(ticker *time.Ticker, k8sClient k8s.Client, logger logr.Logger, namespace, name string) {
for range ticker.C {
jenkins := &v1alpha2.Jenkins{}
err := k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, jenkins)
if err != nil {
logger.V(log.VWarn).Info(fmt.Sprintf("backup trigger, error when fetching CR: %s", err))
}
if jenkins.Status.LastBackup == jenkins.Status.PendingBackup {
jenkins.Status.PendingBackup = jenkins.Status.PendingBackup + 1
err = k8sClient.Update(context.TODO(), jenkins)
if err != nil {
logger.V(log.VWarn).Info(fmt.Sprintf("backup trigger, error when updating CR: %s", err))
}
}
}
}
// EnsureBackupTrigger creates or update trigger which update CR to make backup
func (bar *BackupAndRestore) EnsureBackupTrigger() error {
jenkins := bar.jenkins
trigger, found := backupTriggers[jenkins.Name]
if len(jenkins.Spec.Backup.ContainerName) == 0 || jenkins.Spec.Backup.Interval == 0 {
bar.logger.V(log.VDebug).Info("Skipping create backup trigger")
if found {
bar.stopBackupTrigger(trigger)
}
return nil
}
if !found {
bar.startBackupTrigger()
}
if found && jenkins.Spec.Backup.Interval != trigger.interval {
bar.stopBackupTrigger(trigger)
bar.startBackupTrigger()
}
return nil
}
// StopBackupTrigger stops trigger which update CR to make backup
func (bar *BackupAndRestore) StopBackupTrigger() {
trigger, found := backupTriggers[bar.jenkins.Name]
if found {
bar.stopBackupTrigger(trigger)
}
}
func (bar *BackupAndRestore) startBackupTrigger() {
bar.logger.Info("Starting backup trigger")
ticker := time.NewTicker(time.Duration(bar.jenkins.Spec.Backup.Interval) * time.Second)
backupTriggers[bar.jenkins.Name] = backupTrigger{
interval: bar.jenkins.Spec.Backup.Interval,
ticker: ticker,
}
go triggerBackup(ticker, bar.k8sClient, bar.logger, bar.jenkins.Namespace, bar.jenkins.Name)
}
func (bar *BackupAndRestore) stopBackupTrigger(trigger backupTrigger) {
bar.logger.Info("Stopping backup trigger")
trigger.ticker.Stop()
delete(backupTriggers, bar.jenkins.Name)
}
func (bar *BackupAndRestore) exec(podName, containerName string, command []string) (stdout, stderr bytes.Buffer, err error) {
req := bar.clientSet.CoreV1().RESTClient().Post().
Resource("pods").

View File

@ -0,0 +1,2 @@
// Package backuprestore is responsible for create Jenkins master backup and restore it
package backuprestore

View File

@ -9,6 +9,7 @@ import (
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/backuprestore"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/groovy"
@ -25,6 +26,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)
@ -40,11 +43,13 @@ type ReconcileJenkinsBaseConfiguration struct {
logger logr.Logger
jenkins *v1alpha2.Jenkins
local, minikube bool
clientSet *kubernetes.Clientset
config *rest.Config
}
// New create structure which takes care of base configuration
func New(client client.Client, scheme *runtime.Scheme, logger logr.Logger,
jenkins *v1alpha2.Jenkins, local, minikube bool) *ReconcileJenkinsBaseConfiguration {
jenkins *v1alpha2.Jenkins, local, minikube bool, clientSet *kubernetes.Clientset, config *rest.Config) *ReconcileJenkinsBaseConfiguration {
return &ReconcileJenkinsBaseConfiguration{
k8sClient: client,
scheme: scheme,
@ -52,6 +57,8 @@ func New(client client.Client, scheme *runtime.Scheme, logger logr.Logger,
jenkins: jenkins,
local: local,
minikube: minikube,
clientSet: clientSet,
config: config,
}
}
@ -395,6 +402,9 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureJenkinsMasterPod(meta metav1.O
}
if currentJenkinsMasterPod != nil && isPodTerminating(*currentJenkinsMasterPod) {
backupAndRestore := backuprestore.New(r.k8sClient, *r.clientSet, r.logger, r.jenkins, *r.config)
backupAndRestore.StopBackupTrigger()
//TODO backup before pod deletion?
return reconcile.Result{Requeue: true}, nil
}
if currentJenkinsMasterPod != nil && r.isRecreatePodNeeded(*currentJenkinsMasterPod) {
@ -409,7 +419,7 @@ func isPodTerminating(pod corev1.Pod) bool {
}
func (r *ReconcileJenkinsBaseConfiguration) isRecreatePodNeeded(currentJenkinsMasterPod corev1.Pod) bool {
if r.jenkins.Spec.Restore.RecoveryOnce != 0 {
if r.jenkins.Spec.Restore.RecoveryOnce != 0 && r.jenkins.Status.RestoredBackup != 0 {
r.logger.Info(fmt.Sprintf("spec.restore.recoveryOnce is set, recreating pod"))
return true
}

View File

@ -98,7 +98,7 @@ func TestCompareVolumes(t *testing.T) {
Volumes: resources.GetJenkinsMasterPodBaseVolumes(jenkins),
},
}
reconciler := New(nil, nil, nil, jenkins, false, false)
reconciler := New(nil, nil, nil, jenkins, false, false, nil, nil)
got := reconciler.compareVolumes(pod)
@ -122,7 +122,7 @@ func TestCompareVolumes(t *testing.T) {
Volumes: resources.GetJenkinsMasterPodBaseVolumes(jenkins),
},
}
reconciler := New(nil, nil, nil, jenkins, false, false)
reconciler := New(nil, nil, nil, jenkins, false, false, nil, nil)
got := reconciler.compareVolumes(pod)
@ -146,7 +146,7 @@ func TestCompareVolumes(t *testing.T) {
Volumes: append(resources.GetJenkinsMasterPodBaseVolumes(jenkins), corev1.Volume{Name: "added"}),
},
}
reconciler := New(nil, nil, nil, jenkins, false, false)
reconciler := New(nil, nil, nil, jenkins, false, false, nil, nil)
got := reconciler.compareVolumes(pod)

View File

@ -20,7 +20,7 @@ import (
func TestValidatePlugins(t *testing.T) {
log.SetupLogger(true)
baseReconcileLoop := New(nil, nil, log.Log,
nil, false, false)
nil, false, false, nil, nil)
t.Run("empty", func(t *testing.T) {
var requiredBasePlugins []plugins.Plugin
var basePlugins []v1alpha2.Plugin
@ -150,7 +150,7 @@ func TestValidateJenkinsMasterPodEnvs(t *testing.T) {
},
}
baseReconcileLoop := New(nil, nil, logf.ZapLogger(false),
&jenkins, false, false)
&jenkins, false, false, nil, nil)
got := baseReconcileLoop.validateJenkinsMasterPodEnvs()
assert.Equal(t, true, got)
})
@ -172,7 +172,7 @@ func TestValidateJenkinsMasterPodEnvs(t *testing.T) {
},
}
baseReconcileLoop := New(nil, nil, logf.ZapLogger(false),
&jenkins, false, false)
&jenkins, false, false, nil, nil)
got := baseReconcileLoop.validateJenkinsMasterPodEnvs()
assert.Equal(t, false, got)
})
@ -192,7 +192,7 @@ func TestValidateReservedVolumes(t *testing.T) {
},
}
baseReconcileLoop := New(nil, nil, logf.ZapLogger(false),
&jenkins, false, false)
&jenkins, false, false, nil, nil)
got := baseReconcileLoop.validateReservedVolumes()
assert.Equal(t, true, got)
})
@ -209,7 +209,7 @@ func TestValidateReservedVolumes(t *testing.T) {
},
}
baseReconcileLoop := New(nil, nil, logf.ZapLogger(false),
&jenkins, false, false)
&jenkins, false, false, nil, nil)
got := baseReconcileLoop.validateReservedVolumes()
assert.Equal(t, false, got)
})
@ -223,7 +223,7 @@ func TestValidateContainerVolumeMounts(t *testing.T) {
},
}
baseReconcileLoop := New(nil, nil, logf.ZapLogger(false),
&jenkins, false, false)
&jenkins, false, false, nil, nil)
got := baseReconcileLoop.validateContainerVolumeMounts(v1alpha2.Container{})
assert.Equal(t, true, got)
})
@ -250,7 +250,7 @@ func TestValidateContainerVolumeMounts(t *testing.T) {
},
}
baseReconcileLoop := New(nil, nil, logf.ZapLogger(false),
&jenkins, false, false)
&jenkins, false, false, nil, nil)
got := baseReconcileLoop.validateContainerVolumeMounts(jenkins.Spec.Master.Containers[0])
assert.Equal(t, true, got)
})
@ -277,7 +277,7 @@ func TestValidateContainerVolumeMounts(t *testing.T) {
},
}
baseReconcileLoop := New(nil, nil, logf.ZapLogger(false),
&jenkins, false, false)
&jenkins, false, false, nil, nil)
got := baseReconcileLoop.validateContainerVolumeMounts(jenkins.Spec.Master.Containers[0])
assert.Equal(t, false, got)
})
@ -299,7 +299,7 @@ func TestValidateContainerVolumeMounts(t *testing.T) {
},
}
baseReconcileLoop := New(nil, nil, logf.ZapLogger(false),
&jenkins, false, false)
&jenkins, false, false, nil, nil)
got := baseReconcileLoop.validateContainerVolumeMounts(jenkins.Spec.Master.Containers[0])
assert.Equal(t, false, got)
})
@ -319,7 +319,7 @@ func TestValidateConfigMapVolume(t *testing.T) {
}
fakeClient := fake.NewFakeClient()
baseReconcileLoop := New(fakeClient, nil, logf.ZapLogger(false),
nil, false, false)
nil, false, false, nil, nil)
got, err := baseReconcileLoop.validateConfigMapVolume(volume)
@ -345,7 +345,7 @@ func TestValidateConfigMapVolume(t *testing.T) {
err := fakeClient.Create(context.TODO(), &configMap)
assert.NoError(t, err)
baseReconcileLoop := New(fakeClient, nil, logf.ZapLogger(false),
jenkins, false, false)
jenkins, false, false, nil, nil)
got, err := baseReconcileLoop.validateConfigMapVolume(volume)
@ -369,7 +369,7 @@ func TestValidateConfigMapVolume(t *testing.T) {
}
fakeClient := fake.NewFakeClient()
baseReconcileLoop := New(fakeClient, nil, logf.ZapLogger(false),
jenkins, false, false)
jenkins, false, false, nil, nil)
got, err := baseReconcileLoop.validateConfigMapVolume(volume)
@ -392,7 +392,7 @@ func TestValidateSecretVolume(t *testing.T) {
}
fakeClient := fake.NewFakeClient()
baseReconcileLoop := New(fakeClient, nil, logf.ZapLogger(false),
nil, false, false)
nil, false, false, nil, nil)
got, err := baseReconcileLoop.validateSecretVolume(volume)
@ -416,7 +416,7 @@ func TestValidateSecretVolume(t *testing.T) {
err := fakeClient.Create(context.TODO(), &secret)
assert.NoError(t, err)
baseReconcileLoop := New(fakeClient, nil, logf.ZapLogger(false),
jenkins, false, false)
jenkins, false, false, nil, nil)
got, err := baseReconcileLoop.validateSecretVolume(volume)
@ -438,7 +438,7 @@ func TestValidateSecretVolume(t *testing.T) {
}
fakeClient := fake.NewFakeClient()
baseReconcileLoop := New(fakeClient, nil, logf.ZapLogger(false),
jenkins, false, false)
jenkins, false, false, nil, nil)
got, err := baseReconcileLoop.validateSecretVolume(volume)

View File

@ -6,8 +6,8 @@ import (
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/backuprestore"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/backuprestore"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/casc"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/seedjobs"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
@ -49,7 +49,7 @@ func New(k8sClient k8s.Client, jenkinsClient jenkinsclient.Jenkins, logger logr.
// Reconcile it's a main reconciliation loop for user supplied configuration
func (r *ReconcileUserConfiguration) Reconcile() (reconcile.Result, error) {
backupAndRestore := backuprestore.New(r.k8sClient, r.clientSet, r.jenkinsClient, r.logger, r.jenkins, r.config)
backupAndRestore := backuprestore.New(r.k8sClient, r.clientSet, r.logger, r.jenkins, r.config)
result, err := r.ensureSeedJobs()
if err != nil {
@ -59,7 +59,7 @@ func (r *ReconcileUserConfiguration) Reconcile() (reconcile.Result, error) {
return result, nil
}
if err := backupAndRestore.Restore(); err != nil {
if err := backupAndRestore.Restore(r.jenkinsClient); err != nil {
return reconcile.Result{}, err
}
@ -74,7 +74,9 @@ func (r *ReconcileUserConfiguration) Reconcile() (reconcile.Result, error) {
if err := backupAndRestore.Backup(); err != nil {
return reconcile.Result{}, err
}
//TODO backup Goroutine
if err := backupAndRestore.EnsureBackupTrigger(); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
}

View File

@ -2,13 +2,13 @@ package user
import (
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/backuprestore"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/backuprestore"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/seedjobs"
)
// Validate validates Jenkins CR Spec section
func (r *ReconcileUserConfiguration) Validate(jenkins *v1alpha2.Jenkins) (bool, error) {
backupAndRestore := backuprestore.New(r.k8sClient, r.clientSet, r.jenkinsClient, r.logger, r.jenkins, r.config)
backupAndRestore := backuprestore.New(r.k8sClient, r.clientSet, r.logger, r.jenkins, r.config)
if ok := backupAndRestore.Validate(); !ok {
return false, nil
}

View File

@ -152,7 +152,7 @@ func (r *ReconcileJenkins) reconcile(request reconcile.Request, logger logr.Logg
}
// Reconcile base configuration
baseConfiguration := base.New(r.client, r.scheme, logger, jenkins, r.local, r.minikube)
baseConfiguration := base.New(r.client, r.scheme, logger, jenkins, r.local, r.minikube, &r.clientSet, &r.config)
valid, err := baseConfiguration.Validate(jenkins)
if err != nil {