From b78bff897eca86b6b06fe35c43ebdc4a4e49f8b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20S=C4=99k?= Date: Sat, 19 Jan 2019 13:13:50 +0100 Subject: [PATCH] Add AWS S3 backup provider --- pkg/controller/jenkins/backup/aws/s3.go | 240 ++++++++++++++++++ pkg/controller/jenkins/backup/aws/s3_test.go | 165 ++++++++++++ pkg/controller/jenkins/backup/backup.go | 7 +- pkg/controller/jenkins/constants/constants.go | 2 + 4 files changed, 412 insertions(+), 2 deletions(-) create mode 100644 pkg/controller/jenkins/backup/aws/s3.go create mode 100644 pkg/controller/jenkins/backup/aws/s3_test.go diff --git a/pkg/controller/jenkins/backup/aws/s3.go b/pkg/controller/jenkins/backup/aws/s3.go new file mode 100644 index 00000000..9c55469f --- /dev/null +++ b/pkg/controller/jenkins/backup/aws/s3.go @@ -0,0 +1,240 @@ +package aws + +import ( + "context" + "fmt" + + virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/base/resources" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/constants" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/plugins" + "github.com/VirtusLab/jenkins-operator/pkg/log" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + k8s "sigs.k8s.io/controller-runtime/pkg/client" +) + +// AmazonS3Backup is a backup strategy where backup is stored in AWS S3 bucket +// credentials required to make calls to AWS API are provided by user in backup credentials Kubernetes secret +type AmazonS3Backup struct{} + +// GetRestoreJobXML returns Jenkins restore backup job config XML +func (b *AmazonS3Backup) GetRestoreJobXML(jenkins virtuslabv1alpha1.Jenkins) (string, error) { + return ` + + + + false + + + + + + + false + + + false +`, nil +} + +// GetBackupJobXML returns Jenkins backup job config XML +func (b *AmazonS3Backup) GetBackupJobXML(jenkins virtuslabv1alpha1.Jenkins) (string, error) { + return ` + + + + false + + + + + + + H/60 * * * * + + + + + + + false + + + false +`, nil +} + +// IsConfigurationValidForBasePhase validates if user provided valid configuration of backup for base phase +func (b *AmazonS3Backup) IsConfigurationValidForBasePhase(jenkins virtuslabv1alpha1.Jenkins, logger logr.Logger) bool { + if len(jenkins.Spec.BackupAmazonS3.BucketName) == 0 { + logger.V(log.VWarn).Info("Bucket name not set in 'spec.backupAmazonS3.bucketName'") + return false + } + + if len(jenkins.Spec.BackupAmazonS3.BucketPath) == 0 { + logger.V(log.VWarn).Info("Bucket path not set in 'spec.backupAmazonS3.bucketPath'") + return false + } + + if len(jenkins.Spec.BackupAmazonS3.Region) == 0 { + logger.V(log.VWarn).Info("Region not set in 'spec.backupAmazonS3.region'") + return false + } + + return true +} + +// IsConfigurationValidForUserPhase validates if user provided valid configuration of backup for user phase +func (b *AmazonS3Backup) IsConfigurationValidForUserPhase(k8sClient k8s.Client, jenkins virtuslabv1alpha1.Jenkins, logger logr.Logger) (bool, error) { + backupSecretName := resources.GetBackupCredentialsSecretName(&jenkins) + backupSecret := &corev1.Secret{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: jenkins.Namespace, Name: backupSecretName}, backupSecret) + if err != nil { + return false, err + } + + if len(backupSecret.Data[constants.BackupAmazonS3SecretSecretKey]) == 0 { + logger.V(log.VWarn).Info(fmt.Sprintf("Secret '%s' doesn't contains key: %s", backupSecretName, constants.BackupAmazonS3SecretSecretKey)) + return false, nil + } + + if len(backupSecret.Data[constants.BackupAmazonS3SecretAccessKey]) == 0 { + logger.V(log.VWarn).Info(fmt.Sprintf("Secret '%s' doesn't contains key: %s", backupSecretName, constants.BackupAmazonS3SecretAccessKey)) + return false, nil + } + + return true, nil +} + +// GetRequiredPlugins returns all required Jenkins plugins by this backup strategy +func (b *AmazonS3Backup) GetRequiredPlugins() map[string][]plugins.Plugin { + return map[string][]plugins.Plugin{ + "aws-java-sdk:1.11.457": { + plugins.Must(plugins.New(plugins.ApacheComponentsClientPlugin)), + plugins.Must(plugins.New(plugins.Jackson2ADIPlugin)), + }, + } +} diff --git a/pkg/controller/jenkins/backup/aws/s3_test.go b/pkg/controller/jenkins/backup/aws/s3_test.go new file mode 100644 index 00000000..96d35dab --- /dev/null +++ b/pkg/controller/jenkins/backup/aws/s3_test.go @@ -0,0 +1,165 @@ +package aws + +import ( + "context" + "testing" + + virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/constants" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" +) + +func TestAmazonS3Backup_IsConfigurationValidForBasePhase(t *testing.T) { + tests := []struct { + name string + jenkins virtuslabv1alpha1.Jenkins + want bool + }{ + { + name: "happy", + jenkins: virtuslabv1alpha1.Jenkins{ + Spec: virtuslabv1alpha1.JenkinsSpec{ + BackupAmazonS3: virtuslabv1alpha1.JenkinsBackupAmazonS3{ + BucketName: "some-value", + BucketPath: "some-value", + Region: "some-value", + }, + }, + }, + want: true, + }, + { + name: "fail, no bucket name", + jenkins: virtuslabv1alpha1.Jenkins{ + Spec: virtuslabv1alpha1.JenkinsSpec{ + BackupAmazonS3: virtuslabv1alpha1.JenkinsBackupAmazonS3{ + BucketName: "", + BucketPath: "some-value", + Region: "some-value", + }, + }, + }, + want: false, + }, + { + name: "fail, no bucket path", + jenkins: virtuslabv1alpha1.Jenkins{ + Spec: virtuslabv1alpha1.JenkinsSpec{ + BackupAmazonS3: virtuslabv1alpha1.JenkinsBackupAmazonS3{ + BucketName: "some-value", + BucketPath: "", + Region: "some-value", + }, + }, + }, + want: false, + }, + { + name: "fail, no region", + jenkins: virtuslabv1alpha1.Jenkins{ + Spec: virtuslabv1alpha1.JenkinsSpec{ + BackupAmazonS3: virtuslabv1alpha1.JenkinsBackupAmazonS3{ + BucketName: "some-value", + BucketPath: "some-value", + Region: "", + }, + }, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &AmazonS3Backup{} + got := r.IsConfigurationValidForBasePhase(tt.jenkins, logf.ZapLogger(false)) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestAmazonS3Backup_IsConfigurationValidForUserPhase(t *testing.T) { + tests := []struct { + name string + jenkins *virtuslabv1alpha1.Jenkins + secret *corev1.Secret + want bool + wantErr bool + }{ + { + name: "happy", + jenkins: &virtuslabv1alpha1.Jenkins{ + ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-cr-name"}, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-operator-backup-credentials-jenkins-cr-name"}, + Data: map[string][]byte{ + constants.BackupAmazonS3SecretSecretKey: []byte("some-value"), + constants.BackupAmazonS3SecretAccessKey: []byte("some-value"), + }, + }, + want: true, + wantErr: false, + }, + { + name: "fail, no secret", + jenkins: &virtuslabv1alpha1.Jenkins{ + ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-cr-name"}, + }, + want: false, + wantErr: true, + }, + { + name: "fail, no secret key in secret", + jenkins: &virtuslabv1alpha1.Jenkins{ + ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-cr-name"}, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-operator-backup-credentials-jenkins-cr-name"}, + Data: map[string][]byte{ + constants.BackupAmazonS3SecretSecretKey: []byte(""), + constants.BackupAmazonS3SecretAccessKey: []byte("some-value"), + }, + }, + want: false, + wantErr: false, + }, + { + name: "fail, no access key in secret", + jenkins: &virtuslabv1alpha1.Jenkins{ + ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-cr-name"}, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-operator-backup-credentials-jenkins-cr-name"}, + Data: map[string][]byte{ + constants.BackupAmazonS3SecretSecretKey: []byte("some-value"), + constants.BackupAmazonS3SecretAccessKey: []byte(""), + }, + }, + want: false, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8sClient := fake.NewFakeClient() + logger := logf.ZapLogger(false) + b := &AmazonS3Backup{} + if tt.secret != nil { + e := k8sClient.Create(context.TODO(), tt.secret) + assert.NoError(t, e) + } + got, err := b.IsConfigurationValidForUserPhase(k8sClient, *tt.jenkins, logger) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/controller/jenkins/backup/backup.go b/pkg/controller/jenkins/backup/backup.go index 895f14d4..f4a62d3f 100644 --- a/pkg/controller/jenkins/backup/backup.go +++ b/pkg/controller/jenkins/backup/backup.go @@ -6,6 +6,7 @@ import ( "time" virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/backup/aws" "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/backup/nobackup" jenkinsclient "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/client" "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/base/resources" @@ -32,7 +33,7 @@ type Provider interface { GetRequiredPlugins() map[string][]plugins.Plugin } -// Backup defines backup manager which is responsible of backup jobs history +// Backup defines backup manager which is responsible of backup of jobs history type Backup struct { jenkins *virtuslabv1alpha1.Jenkins k8sClient k8s.Client @@ -130,6 +131,8 @@ func GetBackupProvider(backupType virtuslabv1alpha1.JenkinsBackup) (Provider, er switch backupType { case virtuslabv1alpha1.JenkinsBackupTypeNoBackup: return &nobackup.NoBackup{}, nil + case virtuslabv1alpha1.JenkinsBackupTypeAmazonS3: + return &aws.AmazonS3Backup{}, nil default: return nil, errors.Errorf("Invalid BackupManager type '%s'", backupType) } @@ -155,6 +158,6 @@ func GetPluginsRequiredByAllBackupProviders() map[string][]plugins.Plugin { func getAllProviders() []Provider { return []Provider{ - &nobackup.NoBackup{}, + &nobackup.NoBackup{}, &aws.AmazonS3Backup{}, } } diff --git a/pkg/controller/jenkins/constants/constants.go b/pkg/controller/jenkins/constants/constants.go index b42a6535..b968b842 100644 --- a/pkg/controller/jenkins/constants/constants.go +++ b/pkg/controller/jenkins/constants/constants.go @@ -15,4 +15,6 @@ const ( BackupAmazonS3SecretSecretKey = "secret-key" // BackupJobName is the Jenkins job name used to backup jobs history BackupJobName = OperatorName + "-backup" + // BackupLatestFileName is the latest backup file name + BackupLatestFileName = "build-history-latest.tar.gz" )