public_git
/
kubernetes-operator
mirror of https://github.com/jenkinsci/kubernetes-operator.git
1
0
Fork 0
Code Issues Releases Wiki Activity

#4 [WIP] Backup and restore

This commit is contained in:
Tomasz Sęk 2019-06-16 08:21:23 +02:00
parent 66e9512c80
commit 2d501b00d5
No known key found for this signature in database
GPG Key ID: DC356D23F6A644D0
11 changed files with 293 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/manager/main.go
View File

@ -25,6 +25,7 @@ import (
sdkVersion "github.com/operator-framework/operator-sdk/version"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"k8s.io/client-go/kubernetes"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/controller-runtime/pkg/manager"
@ -107,8 +108,13 @@ func main() {
fatal(errors.Wrap(err, "failed to create manager"), *debug)
}
clientSet, err := kubernetes.NewForConfig(cfg)
if err != nil {
fatal(errors.Wrap(err, "failed to create Kubernetes client set"), *debug)
}
// setup Jenkins controller
if err := jenkins.Add(mgr, *local, *minikube, events); err != nil {
if err := jenkins.Add(mgr, *local, *minikube, events, *clientSet, *cfg); err != nil {
fatal(errors.Wrap(err, "failed to setup controllers"), *debug)
}

1
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/coreos/prometheus-operator v0.26.0 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
github.com/docker/distribution v2.7.1+incompatible
github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
github.com/emicklei/go-restful v2.8.1+incompatible // indirect
github.com/go-logr/logr v0.1.0
github.com/go-logr/zapr v0.1.0

2
go.sum
View File

@ -35,6 +35,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s=
github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=

27
pkg/apis/jenkins/v1alpha2/jenkins_types.go
View File

@ -17,6 +17,8 @@ type JenkinsSpec struct {
SeedJobs []SeedJob `json:"seedJobs,omitempty"`
Service Service `json:"service,omitempty"`
SlaveService Service `json:"slaveService,omitempty"`
Backup Backup `json:"backup,omitempty"`
Restore Restore `json:"restore,omitempty"`
}
// Container defines Kubernetes container attributes
@ -80,6 +82,9 @@ type JenkinsStatus struct {
BaseConfigurationCompletedTime *metav1.Time `json:"baseConfigurationCompletedTime,omitempty"`
UserConfigurationCompletedTime *metav1.Time `json:"userConfigurationCompletedTime,omitempty"`
Builds []Build `json:"builds,omitempty"`
RestoredBackup uint64 `json:"restoredBackup,omitempty"`
LastBackup uint64 `json:"lastBackup,omitempty"`
PendingBackup uint64 `json:"pendingBackup,omitempty"`
}
// BuildStatus defines type of Jenkins build job status
@ -154,7 +159,7 @@ var AllowedJenkinsCredentialMap = map[string]string{
string(UsernamePasswordCredentialType): "",
}
// SeedJob defined configuration for seed jobs and deploy keys
// SeedJob defines configuration for seed jobs and deploy keys
type SeedJob struct {
ID string `json:"id,omitempty"`
CredentialID string `json:"credentialID,omitempty"`
@ -165,6 +170,22 @@ type SeedJob struct {
JenkinsCredentialType JenkinsCredentialType `json:"credentialType,omitempty"`
}
func init() {
SchemeBuilder.Register(&Jenkins{}, &JenkinsList{})
// Handler defines a specific action that should be taken
type Handler struct {
// Exec specifies the action to take.
Exec *corev1.ExecAction `json:"exec,omitempty"`
}
// Backup defines configuration of Jenkins backup
type Backup struct {
ContainerName string `json:"containerName"`
Action Handler `json:"action"`
Interval uint64 `json:"interval"`
}
// Restore defines configuration of Jenkins backup restore
type Restore struct {
ContainerName string `json:"containerName"`
Action Handler `json:"action"`
RecoveryOnce uint64 `json:"recoveryOnce,omitempty"`
}

4
pkg/apis/jenkins/v1alpha2/register.go
View File

@ -25,3 +25,7 @@ var (
// GetObjectKind returns Jenkins object kind
func (in *Jenkins) GetObjectKind() schema.ObjectKind { return in }
func init() {
SchemeBuilder.Register(&Jenkins{}, &JenkinsList{})
}

11
pkg/controller/jenkins/configuration/base/reconcile.go
View File

@ -360,9 +360,9 @@ func (r *ReconcileJenkinsBaseConfiguration) createService(meta metav1.ObjectMeta
}
func (r *ReconcileJenkinsBaseConfiguration) getJenkinsMasterPod(meta metav1.ObjectMeta) (*corev1.Pod, error) {
jenkinsMasterPod := resources.NewJenkinsMasterPod(meta, r.jenkins)
jenkinsMasterPodName := resources.GetJenkinsMasterPodName(*r.jenkins)
currentJenkinsMasterPod := &corev1.Pod{}
err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: jenkinsMasterPod.Name, Namespace: jenkinsMasterPod.Namespace}, currentJenkinsMasterPod)
err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: jenkinsMasterPodName, Namespace: r.jenkins.Namespace}, currentJenkinsMasterPod)
if err != nil {
return nil, err // don't wrap error
}
@ -382,6 +382,8 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureJenkinsMasterPod(meta metav1.O
now := metav1.Now()
r.jenkins.Status = v1alpha2.JenkinsStatus{
ProvisionStartTime: &now,
LastBackup: r.jenkins.Status.LastBackup,
PendingBackup: r.jenkins.Status.LastBackup,
}
err = r.updateResource(r.jenkins)
if err != nil {
@ -407,6 +409,11 @@ func isPodTerminating(pod corev1.Pod) bool {
}
func (r *ReconcileJenkinsBaseConfiguration) isRecreatePodNeeded(currentJenkinsMasterPod corev1.Pod) bool {
if r.jenkins.Spec.Restore.RecoveryOnce != 0 {
r.logger.Info(fmt.Sprintf("spec.restore.recoveryOnce is set, recreating pod"))
return true
}
if version.Version != r.jenkins.Status.OperatorVersion {
r.logger.Info(fmt.Sprintf("Jenkins Operator version has changed, actual '%+v' new '%+v', recreating pod",
r.jenkins.Status.OperatorVersion, version.Version))

2
pkg/controller/jenkins/configuration/base/resources/pod.go
View File

@ -14,7 +14,7 @@ const (
// JenkinsMasterContainerName is the Jenkins master container name in pod
JenkinsMasterContainerName = "jenkins-master"
// JenkinsHomeVolumeName is the Jenkins home volume name
JenkinsHomeVolumeName = "home"
JenkinsHomeVolumeName = "jenkins-home"
jenkinsPath = "/var/jenkins"
jenkinsHomePath = jenkinsPath + "/home"

191
pkg/controller/jenkins/configuration/user/backuprestore/backuprestore.go Normal file
View File

@ -0,0 +1,191 @@
package backuprestore
import (
"bytes"
"context"
"fmt"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
"github.com/jenkinsci/kubernetes-operator/pkg/log"
"github.com/go-logr/logr"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/remotecommand"
k8s "sigs.k8s.io/controller-runtime/pkg/client"
)
// BackupAndRestore represents Jenkins backup and restore client
type BackupAndRestore struct {
config rest.Config
k8sClient k8s.Client
clientSet kubernetes.Clientset
jenkinsClient jenkinsclient.Jenkins
logger logr.Logger
jenkins *v1alpha2.Jenkins
}
// New returns Jenkins backup and restore client
func New(k8sClient k8s.Client, clientSet kubernetes.Clientset, jenkinsClient jenkinsclient.Jenkins,
logger logr.Logger, jenkins *v1alpha2.Jenkins, config rest.Config) *BackupAndRestore {
return &BackupAndRestore{k8sClient: k8sClient, clientSet: clientSet, jenkinsClient: jenkinsClient, logger: logger, jenkins: jenkins, config: config}
}
// Validate validates backup and restore configuration
func (bar *BackupAndRestore) Validate() bool {
valid := true
allContainers := map[string]v1alpha2.Container{}
for _, container := range bar.jenkins.Spec.Master.Containers {
allContainers[container.Name] = container
}
restore := bar.jenkins.Spec.Restore
if len(restore.ContainerName) > 0 {
_, found := allContainers[restore.ContainerName]
if !found {
valid = false
bar.logger.V(log.VWarn).Info(fmt.Sprintf("restore container '%s' not found in CR spec.master.containers", restore.ContainerName))
}
if restore.Action.Exec == nil {
valid = false
bar.logger.V(log.VWarn).Info(fmt.Sprintf("spec.restore.action.exec is not configured"))
}
}
backup := bar.jenkins.Spec.Backup
if len(backup.ContainerName) > 0 {
_, found := allContainers[backup.ContainerName]
if !found {
valid = false
bar.logger.V(log.VWarn).Info(fmt.Sprintf("backup container '%s' not found in CR spec.master.containers", backup.ContainerName))
}
if backup.Action.Exec == nil {
valid = false
bar.logger.V(log.VWarn).Info(fmt.Sprintf("spec.backup.action.exec is not configured"))
}
if backup.Interval == 0 {
valid = false
bar.logger.V(log.VWarn).Info(fmt.Sprintf("spec.backup.interval is not configured"))
}
}
if len(restore.ContainerName) > 0 && len(backup.ContainerName) == 0 {
valid = false
bar.logger.V(log.VWarn).Info("spec.backup.containerName is not configured")
}
if len(backup.ContainerName) > 0 && len(restore.ContainerName) == 0 {
valid = false
bar.logger.V(log.VWarn).Info("spec.restore.containerName is not configured")
}
return valid
}
// Restore performs Jenkins restore backup operation
func (bar *BackupAndRestore) Restore() error {
jenkins := bar.jenkins
if jenkins.Status.RestoredBackup != 0 {
bar.logger.V(log.VDebug).Info("Skipping restore backup, backup already restored")
return nil
}
if jenkins.Status.LastBackup == 0 {
bar.logger.Info("Skipping restore backup")
if jenkins.Status.PendingBackup == 0 {
jenkins.Status.PendingBackup = 1
return bar.k8sClient.Update(context.TODO(), jenkins)
}
return nil
}
var backupNumber uint64
if jenkins.Spec.Restore.RecoveryOnce != 0 {
backupNumber = jenkins.Spec.Restore.RecoveryOnce
} else {
backupNumber = jenkins.Status.LastBackup
}
bar.logger.Info(fmt.Sprintf("Restoring backup '%d'", backupNumber))
podName := resources.GetJenkinsMasterPodName(*jenkins)
command := jenkins.Spec.Restore.Action.Exec.Command
command = append(command, fmt.Sprintf("%d", backupNumber))
_, _, err := bar.exec(podName, jenkins.Spec.Restore.ContainerName, command)
if err == nil {
jenkins.Spec.Restore.RecoveryOnce = 0
jenkins.Status.RestoredBackup = backupNumber
jenkins.Status.PendingBackup = backupNumber + 1
return bar.k8sClient.Update(context.TODO(), jenkins)
}
//TODO reload?
//TODO after 3 fails stop
return err
}
// Backup performs Jenkins backup operation
func (bar *BackupAndRestore) Backup() error {
jenkins := bar.jenkins
if jenkins.Status.PendingBackup == jenkins.Status.LastBackup {
bar.logger.V(log.VDebug).Info("Skipping backup")
return nil
}
backupNumber := jenkins.Status.PendingBackup
bar.logger.Info(fmt.Sprintf("Performing backup '%d'", backupNumber))
podName := resources.GetJenkinsMasterPodName(*jenkins)
command := jenkins.Spec.Backup.Action.Exec.Command
command = append(command, fmt.Sprintf("%d", backupNumber))
_, _, err := bar.exec(podName, jenkins.Spec.Backup.ContainerName, command)
if err == nil {
if jenkins.Status.RestoredBackup == 0 {
jenkins.Status.RestoredBackup = backupNumber
}
jenkins.Status.LastBackup = backupNumber
jenkins.Status.PendingBackup = backupNumber
return bar.k8sClient.Update(context.TODO(), jenkins)
}
//TODO after 3 fails stop
return err
}
func (bar *BackupAndRestore) exec(podName, containerName string, command []string) (stdout, stderr bytes.Buffer, err error) {
req := bar.clientSet.CoreV1().RESTClient().Post().
Resource("pods").
Name(podName).
Namespace(bar.jenkins.Namespace).
SubResource("exec")
req.VersionedParams(&corev1.PodExecOptions{
Command: command,
Container: containerName,
Stdin: false,
Stdout: true,
Stderr: true,
TTY: false,
}, scheme.ParameterCodec)
exec, err := remotecommand.NewSPDYExecutor(&bar.config, "POST", req.URL())
if err != nil {
return stdout, stderr, errors.Wrap(err, "pod exec error while creating Executor")
}
err = exec.Stream(remotecommand.StreamOptions{
Stdin: nil,
Stdout: &stdout,
Stderr: &stderr,
Tty: false,
})
bar.logger.V(log.VDebug).Info(fmt.Sprintf("pod exec: stdout '%s' stderr '%s'", stdout.String(), stderr.String()))
if err != nil {
return stdout, stderr, errors.Wrapf(err, "pod exec error operation on stream: stdout '%s' stderr '%s'", stdout.String(), stderr.String())
}
return
}

21
pkg/controller/jenkins/configuration/user/reconcile.go
View File

@ -7,6 +7,7 @@ import (
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/backuprestore"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/casc"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/seedjobs"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
@ -17,6 +18,8 @@ import (
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
k8s "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)
@ -27,22 +30,27 @@ type ReconcileUserConfiguration struct {
jenkinsClient jenkinsclient.Jenkins
logger logr.Logger
jenkins *v1alpha2.Jenkins
clientSet kubernetes.Clientset
config rest.Config
}
// New create structure which takes care of user configuration
func New(k8sClient k8s.Client, jenkinsClient jenkinsclient.Jenkins, logger logr.Logger,
jenkins *v1alpha2.Jenkins) *ReconcileUserConfiguration {
jenkins *v1alpha2.Jenkins, clientSet kubernetes.Clientset, config rest.Config) *ReconcileUserConfiguration {
return &ReconcileUserConfiguration{
k8sClient: k8sClient,
jenkinsClient: jenkinsClient,
logger: logger,
jenkins: jenkins,
clientSet: clientSet,
config: config,
}
}
// Reconcile it's a main reconciliation loop for user supplied configuration
func (r *ReconcileUserConfiguration) Reconcile() (reconcile.Result, error) {
// reconcile seed jobs
backupAndRestore := backuprestore.New(r.k8sClient, r.clientSet, r.jenkinsClient, r.logger, r.jenkins, r.config)
result, err := r.ensureSeedJobs()
if err != nil {
return reconcile.Result{}, err
@ -51,6 +59,10 @@ func (r *ReconcileUserConfiguration) Reconcile() (reconcile.Result, error) {
return result, nil
}
if err := backupAndRestore.Restore(); err != nil {
return reconcile.Result{}, err
}
result, err = r.ensureUserConfiguration(r.jenkinsClient)
if err != nil {
return reconcile.Result{}, err
@ -59,6 +71,11 @@ func (r *ReconcileUserConfiguration) Reconcile() (reconcile.Result, error) {
return result, nil
}
if err := backupAndRestore.Backup(); err != nil {
return reconcile.Result{}, err
}
//TODO backup Goroutine
return reconcile.Result{}, nil
}

6
pkg/controller/jenkins/configuration/user/validate.go
View File

@ -2,11 +2,17 @@ package user
import (
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/backuprestore"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/seedjobs"
)
// Validate validates Jenkins CR Spec section
func (r *ReconcileUserConfiguration) Validate(jenkins *v1alpha2.Jenkins) (bool, error) {
backupAndRestore := backuprestore.New(r.k8sClient, r.clientSet, r.jenkinsClient, r.logger, r.jenkins, r.config)
if ok := backupAndRestore.Validate(); !ok {
return false, nil
}
seedJobs := seedjobs.New(r.jenkinsClient, r.k8sClient, r.logger)
return seedJobs.ValidateSeedJobs(*jenkins)
}

40
pkg/controller/jenkins/jenkins_controller.go
View File

@ -23,6 +23,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/handler"
@ -42,18 +44,20 @@ const (
// Add creates a new Jenkins Controller and adds it to the Manager. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager, local, minikube bool, events event.Recorder) error {
return add(mgr, newReconciler(mgr, local, minikube, events))
func Add(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config) error {
return add(mgr, newReconciler(mgr, local, minikube, events, clientSet, config))
}
// newReconciler returns a new reconcile.Reconciler
func newReconciler(mgr manager.Manager, local, minikube bool, events event.Recorder) reconcile.Reconciler {
func newReconciler(mgr manager.Manager, local, minikube bool, events event.Recorder, clientSet kubernetes.Clientset, config rest.Config) reconcile.Reconciler {
return &ReconcileJenkins{
client: mgr.GetClient(),
scheme: mgr.GetScheme(),
local: local,
minikube: minikube,
events: events,
client: mgr.GetClient(),
scheme: mgr.GetScheme(),
local: local,
minikube: minikube,
events: events,
clientSet: clientSet,
config: config,
}
}
@ -103,6 +107,8 @@ type ReconcileJenkins struct {
scheme *runtime.Scheme
local, minikube bool
events event.Recorder
clientSet kubernetes.Clientset
config rest.Config
}
// Reconcile it's a main reconciliation loop which maintain desired state based on Jenkins.Spec
@ -181,7 +187,7 @@ func (r *ReconcileJenkins) reconcile(request reconcile.Request, logger logr.Logg
r.events.Emit(jenkins, event.TypeNormal, reasonBaseConfigurationSuccess, "Base configuration completed")
}
// Reconcile user configuration
userConfiguration := user.New(r.client, jenkinsClient, logger, jenkins)
userConfiguration := user.New(r.client, jenkinsClient, logger, jenkins, r.clientSet, r.config)
valid, err = userConfiguration.Validate(jenkins)
if err != nil {
@ -334,13 +340,25 @@ func (r *ReconcileJenkins) setDefaults(jenkins *v1alpha2.Jenkins, logger logr.Lo
}
if len(jenkins.Spec.Master.Containers) > 1 {
for i, container := range jenkins.Spec.Master.Containers[1:] {
if setDefaultsForContainer(jenkins, i, logger.WithValues("container", container.Name)) {
if setDefaultsForContainer(jenkins, i+1, logger.WithValues("container", container.Name)) {
changed = true
}
}
}
if len(jenkins.Spec.Backup.ContainerName) > 0 && jenkins.Spec.Backup.Interval == 0 {
logger.Info("Setting default backup interval")
changed = true
jenkins.Spec.Backup.Interval = 30
}
jenkins.Spec.Master.Containers = []v1alpha2.Container{jenkinsContainer}
if len(jenkins.Spec.Master.Containers) == 0 || len(jenkins.Spec.Master.Containers) == 1 {
jenkins.Spec.Master.Containers = []v1alpha2.Container{jenkinsContainer}
} else {
noJenkinsContainers := jenkins.Spec.Master.Containers[1:]
containers := []v1alpha2.Container{jenkinsContainer}
containers = append(containers, noJenkinsContainers...)
jenkins.Spec.Master.Containers = containers
}
if changed {
return errors.WithStack(r.client.Update(context.TODO(), jenkins))