Remove backup mechanism

This commit is contained in:
Tomasz Sęk
2019-02-14 19:36:55 +01:00
parent ab87922bd1
commit dd04c0cf5b
21 changed files with 11 additions and 1370 deletions
@@ -7,7 +7,6 @@ import (
"time"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/backup"
jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
@@ -62,16 +61,7 @@ func (r *ReconcileJenkinsBaseConfiguration) Reconcile() (reconcile.Result, jenki
return reconcile.Result{}, nil, err
}
pluginsRequiredByAllBackupProviders := backup.GetPluginsRequiredByAllBackupProviders()
result, err := r.ensurePluginsRequiredByAllBackupProviders(pluginsRequiredByAllBackupProviders)
if err != nil {
return reconcile.Result{}, nil, err
}
if result.Requeue {
return result, nil, nil
}
result, err = r.ensureJenkinsMasterPod(metaObject)
result, err := r.ensureJenkinsMasterPod(metaObject)
if err != nil {
return reconcile.Result{}, nil, err
}
@@ -95,7 +85,7 @@ func (r *ReconcileJenkinsBaseConfiguration) Reconcile() (reconcile.Result, jenki
}
r.logger.V(log.VDebug).Info("Jenkins API client set")
ok, err := r.verifyPlugins(jenkinsClient, plugins.BasePluginsMap, pluginsRequiredByAllBackupProviders)
ok, err := r.verifyPlugins(jenkinsClient, plugins.BasePluginsMap)
if err != nil {
return reconcile.Result{}, nil, err
}
@@ -144,11 +134,6 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureResourcesRequiredForJenkinsPod
}
r.logger.V(log.VDebug).Info("Service is present")
if err := r.createBackupCredentialsSecret(metaObject); err != nil {
return err
}
r.logger.V(log.VDebug).Info("Backup credentials secret is present")
return nil
}
@@ -473,23 +458,6 @@ func (r *ReconcileJenkinsBaseConfiguration) ensureBaseConfiguration(jenkinsClien
return reconcile.Result{}, nil
}
func (r *ReconcileJenkinsBaseConfiguration) createBackupCredentialsSecret(meta metav1.ObjectMeta) error {
currentSecret := &corev1.Secret{}
err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: resources.GetBackupCredentialsSecretName(r.jenkins), Namespace: r.jenkins.Namespace}, currentSecret)
if err != nil && errors.IsNotFound(err) {
return r.k8sClient.Create(context.TODO(), resources.NewBackupCredentialsSecret(r.jenkins))
} else if err != nil {
return err
}
valid := r.verifyLabelsForWatchedResource(currentSecret)
if !valid {
currentSecret.ObjectMeta.Labels = resources.BuildLabelsForWatchedResources(r.jenkins)
return r.k8sClient.Update(context.TODO(), currentSecret)
}
return nil
}
func (r *ReconcileJenkinsBaseConfiguration) verifyLabelsForWatchedResource(object metav1.Object) bool {
requiredLabels := resources.BuildLabelsForWatchedResources(r.jenkins)
for key, value := range requiredLabels {
@@ -500,28 +468,3 @@ func (r *ReconcileJenkinsBaseConfiguration) verifyLabelsForWatchedResource(objec
return true
}
func (r *ReconcileJenkinsBaseConfiguration) ensurePluginsRequiredByAllBackupProviders(requiredPlugins map[string][]plugins.Plugin) (reconcile.Result, error) {
copiedPlugins := map[string][]string{}
for key, value := range r.jenkins.Spec.Master.Plugins {
copiedPlugins[key] = value
}
for key, value := range requiredPlugins {
copiedPlugins[key] = func() []string {
var pluginsWithVersion []string
for _, plugin := range value {
pluginsWithVersion = append(pluginsWithVersion, plugin.String())
}
return pluginsWithVersion
}()
}
if !reflect.DeepEqual(r.jenkins.Spec.Master.Plugins, copiedPlugins) {
r.logger.Info("Adding plugins required by backup providers to '.spec.master.plugins'")
r.jenkins.Spec.Master.Plugins = copiedPlugins
err := r.k8sClient.Update(context.TODO(), r.jenkins)
return reconcile.Result{Requeue: true}, err
}
return reconcile.Result{}, nil
}
@@ -1,116 +0,0 @@
package base
import (
"context"
"testing"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/plugins"
"github.com/stretchr/testify/assert"
"k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
)
func TestReconcileJenkinsBaseConfiguration_ensurePluginsRequiredByAllBackupProviders(t *testing.T) {
tests := []struct {
name string
jenkins *v1alpha1.Jenkins
requiredPlugins map[string][]plugins.Plugin
want reconcile.Result
wantErr bool
}{
{
name: "happy, no required plugins",
jenkins: &v1alpha1.Jenkins{
Spec: v1alpha1.JenkinsSpec{
Master: v1alpha1.JenkinsMaster{
Plugins: map[string][]string{
"first-plugin:0.0.1": {"second-plugin:0.0.1"},
},
},
},
},
want: reconcile.Result{Requeue: false},
wantErr: false,
},
{
name: "happy, required plugins are set",
jenkins: &v1alpha1.Jenkins{
Spec: v1alpha1.JenkinsSpec{
Master: v1alpha1.JenkinsMaster{
Plugins: map[string][]string{
"first-plugin:0.0.1": {"second-plugin:0.0.1"},
},
},
},
},
requiredPlugins: map[string][]plugins.Plugin{
"first-plugin:0.0.1": {plugins.Must(plugins.New("second-plugin:0.0.1"))},
},
want: reconcile.Result{Requeue: false},
wantErr: false,
},
{
name: "happy, jenkins CR must be updated",
jenkins: &v1alpha1.Jenkins{
Spec: v1alpha1.JenkinsSpec{
Master: v1alpha1.JenkinsMaster{
Plugins: map[string][]string{
"first-plugin:0.0.1": {"second-plugin:0.0.1"},
},
},
},
},
requiredPlugins: map[string][]plugins.Plugin{
"first-plugin:0.0.1": {plugins.Must(plugins.New("second-plugin:0.0.1"))},
"third-plugin:0.0.1": {},
},
want: reconcile.Result{Requeue: true},
wantErr: false,
},
{
name: "happy, jenkins CR must be updated",
jenkins: &v1alpha1.Jenkins{
Spec: v1alpha1.JenkinsSpec{
Master: v1alpha1.JenkinsMaster{
Plugins: map[string][]string{
"first-plugin:0.0.1": {"second-plugin:0.0.1"},
},
},
},
},
requiredPlugins: map[string][]plugins.Plugin{
"first-plugin:0.0.1": {plugins.Must(plugins.New("second-plugin:0.0.1"))},
"third-plugin:0.0.1": {plugins.Must(plugins.New("fourth-plugin:0.0.1"))},
},
want: reconcile.Result{Requeue: true},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := v1alpha1.SchemeBuilder.AddToScheme(scheme.Scheme)
assert.NoError(t, err)
r := &ReconcileJenkinsBaseConfiguration{
k8sClient: fake.NewFakeClient(),
scheme: nil,
logger: logf.ZapLogger(false),
jenkins: tt.jenkins,
local: false,
minikube: false,
}
err = r.k8sClient.Create(context.TODO(), tt.jenkins)
assert.NoError(t, err)
got, err := r.ensurePluginsRequiredByAllBackupProviders(tt.requiredPlugins)
if tt.wantErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
assert.Equal(t, tt.want, got)
})
}
}
@@ -1,30 +0,0 @@
package resources
import (
"fmt"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/constants"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetBackupCredentialsSecretName returns name of Kubernetes secret used to store backup credentials
func GetBackupCredentialsSecretName(jenkins *v1alpha1.Jenkins) string {
return fmt.Sprintf("%s-backup-credentials-%s", constants.OperatorName, jenkins.Name)
}
// NewBackupCredentialsSecret builds the Kubernetes secret used to store backup credentials
func NewBackupCredentialsSecret(jenkins *v1alpha1.Jenkins) *corev1.Secret {
meta := metav1.ObjectMeta{
Name: GetBackupCredentialsSecretName(jenkins),
Namespace: jenkins.ObjectMeta.Namespace,
Labels: BuildLabelsForWatchedResources(jenkins),
}
return &corev1.Secret{
TypeMeta: buildSecretTypeMeta(),
ObjectMeta: meta,
}
}
@@ -16,7 +16,7 @@ import jenkins.model.JenkinsLocationConfiguration
import hudson.model.Node.Mode
def jenkins = Jenkins.instance
//Number of jobs that run simultaneously on master, currently only backup and SeedJob.
//Number of jobs that run simultaneously on master, currently only SeedJob.
jenkins.setNumExecutors(%d)
//Jobs must specify that they want to run on master
jenkins.setMode(Mode.EXCLUSIVE)
@@ -16,7 +16,6 @@ const (
jenkinsScriptsVolumeName = "scripts"
jenkinsScriptsVolumePath = "/var/jenkins/scripts"
initScriptName = "init.sh"
backupScriptName = "backup.sh"
jenkinsOperatorCredentialsVolumeName = "operator-credentials"
jenkinsOperatorCredentialsVolumePath = "/var/jenkins/operator-credentials"
@@ -34,11 +33,6 @@ const (
// this scripts are provided by user
JenkinsUserConfigurationVolumePath = "/var/jenkins/user-configuration"
jenkinsBackupCredentialsVolumeName = "backup-credentials"
// JenkinsBackupCredentialsVolumePath is a path where are credentials used for backup/restore
// credentials are provided by user
JenkinsBackupCredentialsVolumePath = "/var/jenkins/backup-credentials"
httpPortName = "http"
slavePortName = "slavelistener"
// HTTPPortInt defines Jenkins master HTTP port
@@ -84,16 +78,6 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
"bash",
fmt.Sprintf("%s/%s", jenkinsScriptsVolumePath, initScriptName),
},
Lifecycle: &corev1.Lifecycle{
PreStop: &corev1.Handler{
Exec: &corev1.ExecAction{
Command: []string{
"bash",
fmt.Sprintf("%s/%s", jenkinsScriptsVolumePath, backupScriptName),
},
},
},
},
LivenessProbe: &corev1.Probe{
Handler: corev1.Handler{
HTTPGet: &corev1.HTTPGetAction{
@@ -168,11 +152,6 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
MountPath: jenkinsOperatorCredentialsVolumePath,
ReadOnly: true,
},
{
Name: jenkinsBackupCredentialsVolumeName,
MountPath: JenkinsBackupCredentialsVolumePath,
ReadOnly: true,
},
},
},
},
@@ -231,14 +210,6 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
},
},
},
{
Name: jenkinsBackupCredentialsVolumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: GetBackupCredentialsSecretName(jenkins),
},
},
},
},
},
}
@@ -264,70 +264,6 @@ echo "Installing plugins - end"
/sbin/tini -s -- /usr/local/bin/jenkins.sh
`))
const backupBashFmt = `#!/usr/bin/env bash
# don't add set -e
JENKINS_SERVER="http://%s:$(cat %s/%s)@localhost:%d"
JOB="%s"
JOB_QUERY=/job/${JOB}
echo 'Starting the build'
curl -f -v -X POST "${JENKINS_SERVER}${JOB_QUERY}/build?delay=0sec" || exit -1
sleep 3 # give some time for Jenkins to update builds numbers
BUILD_STATUS_QUERY=/lastBuild/api/json
CURRENT_BUILD_NUMBER_QUERY=/lastBuild/buildNumber
CURRENT_BUILD_JSON=$(curl -s -f "${JENKINS_SERVER}${JOB_QUERY}${CURRENT_BUILD_NUMBER_QUERY}")
LAST_STABLE_BUILD_NUMBER_QUERY=/lastStableBuild/buildNumber
check_build()
{
GOOD_BUILD="Last build successful. "
BAD_BUILD="Last build failed. "
JOB_STATUS_JSON=$(curl -s -f "${JENKINS_SERVER}${JOB_QUERY}${BUILD_STATUS_QUERY}")
RESULT=$(echo "${JOB_STATUS_JSON}" | sed -n 's/.*"result":\([\"A-Za-z]*\),.*/\1/p')
CURRENT_BUILD_NUMBER=${CURRENT_BUILD_JSON}
LAST_STABLE_BUILD_JSON=$(curl --silent "${JENKINS_SERVER}${JOB_QUERY}${LAST_STABLE_BUILD_NUMBER_QUERY}")
LAST_STABLE_BUILD_NUMBER=${LAST_STABLE_BUILD_JSON}
LAST_BUILD_STATUS=${GOOD_BUILD}
echo "${LAST_STABLE_BUILD_NUMBER}" | grep "is not available" > /dev/null
GREP_RETURN_CODE=$?
if [[ ${GREP_RETURN_CODE} -ne 0 ]]
then
if [[ $(expr ${CURRENT_BUILD_NUMBER} - 1) -gt ${LAST_STABLE_BUILD_NUMBER} ]]
then
LAST_BUILD_STATUS=${BAD_BUILD}
fi
fi
if [[ "${RESULT}" = "null" ]]
then
echo "${LAST_BUILD_STATUS}Building ${JOB} ${CURRENT_BUILD_NUMBER}... last stable was ${LAST_STABLE_BUILD_NUMBER}"
elif [[ "${RESULT}" = "\"SUCCESS\"" ]]
then
echo "${LAST_BUILD_STATUS}${JOB} ${CURRENT_BUILD_NUMBER} completed successfully."
exit 0
elif [[ "${RESULT}" = "\"FAILURE\"" ]]
then
LAST_BUILD_STATUS=${BAD_BUILD}
echo "${LAST_BUILD_STATUS}${JOB} ${CURRENT_BUILD_NUMBER} failed"
exit -1
else
LAST_BUILD_STATUS=${BAD_BUILD}
echo "${LAST_BUILD_STATUS}${JOB} ${CURRENT_BUILD_NUMBER} status unknown - '${RESULT}'"
exit -1
fi
}
while [[ true ]]
do
check_build
sleep 1
done
`
func buildConfigMapTypeMeta() metav1.TypeMeta {
return metav1.TypeMeta{
Kind: "ConfigMap",
@@ -377,8 +313,6 @@ func NewScriptsConfigMap(meta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins) (*co
Data: map[string]string{
initScriptName: *initBashScript,
installPluginsCommand: fmt.Sprintf(installPluginsBashFmt, jenkinsHomePath),
backupScriptName: fmt.Sprintf(backupBashFmt,
OperatorUserName, jenkinsOperatorCredentialsVolumePath, OperatorCredentialsSecretTokenKey, HTTPPortInt, constants.BackupJobName),
},
}, nil
}
@@ -1,20 +1,14 @@
package base
import (
"context"
"fmt"
"regexp"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/backup"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/plugins"
"github.com/jenkinsci/kubernetes-operator/pkg/log"
docker "github.com/docker/distribution/reference"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
)
var (
@@ -38,20 +32,6 @@ func (r *ReconcileJenkinsBaseConfiguration) Validate(jenkins *v1alpha1.Jenkins)
return false, nil
}
valid, err := r.verifyBackup()
if !valid || err != nil {
return valid, err
}
backupProvider, err := backup.GetBackupProvider(r.jenkins.Spec.Backup)
if err != nil {
return false, err
}
if !backupProvider.IsConfigurationValidForBasePhase(*r.jenkins, r.logger) {
return false, nil
}
return true, nil
}
@@ -84,39 +64,3 @@ func (r *ReconcileJenkinsBaseConfiguration) validatePlugins(pluginsWithVersions
return valid
}
func (r *ReconcileJenkinsBaseConfiguration) verifyBackup() (bool, error) {
if r.jenkins.Spec.Backup == "" {
r.logger.V(log.VWarn).Info("Backup strategy not set in 'spec.backup'")
return false, nil
}
valid := false
for _, backupType := range v1alpha1.AllowedJenkinsBackups {
if r.jenkins.Spec.Backup == backupType {
valid = true
}
}
if !valid {
r.logger.V(log.VWarn).Info(fmt.Sprintf("Invalid backup strategy '%s'", r.jenkins.Spec.Backup))
r.logger.V(log.VWarn).Info(fmt.Sprintf("Allowed backups '%+v'", v1alpha1.AllowedJenkinsBackups))
return false, nil
}
if r.jenkins.Spec.Backup == v1alpha1.JenkinsBackupTypeNoBackup {
return true, nil
}
backupSecretName := resources.GetBackupCredentialsSecretName(r.jenkins)
backupSecret := &corev1.Secret{}
err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: r.jenkins.Namespace, Name: backupSecretName}, backupSecret)
if err != nil && errors.IsNotFound(err) {
r.logger.V(log.VWarn).Info(fmt.Sprintf("Please create secret '%s' in namespace '%s'", backupSecretName, r.jenkins.Namespace))
return false, nil
} else if err != nil && !errors.IsNotFound(err) {
return false, err
}
return true, nil
}
@@ -1,16 +1,10 @@
package base
import (
"context"
"fmt"
"testing"
"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
)
@@ -62,87 +56,3 @@ func TestValidatePlugins(t *testing.T) {
})
}
}
func TestReconcileJenkinsBaseConfiguration_verifyBackup(t *testing.T) {
tests := []struct {
name string
jenkins *v1alpha1.Jenkins
secret *corev1.Secret
want bool
wantErr bool
}{
{
name: "happy, no backup",
jenkins: &v1alpha1.Jenkins{
ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-cr-name"},
Spec: v1alpha1.JenkinsSpec{
Backup: v1alpha1.JenkinsBackupTypeNoBackup,
},
},
want: true,
wantErr: false,
},
{
name: "happy",
jenkins: &v1alpha1.Jenkins{
ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-cr-name"},
Spec: v1alpha1.JenkinsSpec{
Backup: v1alpha1.JenkinsBackupTypeAmazonS3,
},
},
secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-operator-backup-credentials-jenkins-cr-name"},
},
want: true,
wantErr: false,
},
{
name: "fail, no secret",
jenkins: &v1alpha1.Jenkins{
ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-cr-name"},
Spec: v1alpha1.JenkinsSpec{
Backup: v1alpha1.JenkinsBackupTypeAmazonS3,
},
},
want: false,
wantErr: false,
},
{
name: "fail, empty backup type",
jenkins: &v1alpha1.Jenkins{
ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-cr-name"},
Spec: v1alpha1.JenkinsSpec{
Backup: "",
},
},
secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: "namespace-name", Name: "jenkins-operator-backup-credentials-jenkins-cr-name"},
},
want: false,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := &ReconcileJenkinsBaseConfiguration{
k8sClient: fake.NewFakeClient(),
scheme: nil,
logger: logf.ZapLogger(false),
jenkins: tt.jenkins,
local: false,
minikube: false,
}
if tt.secret != nil {
e := r.k8sClient.Create(context.TODO(), tt.secret)
assert.NoError(t, e)
}
got, err := r.verifyBackup()
if tt.wantErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
assert.Equal(t, tt.want, got)
})
}
}