#3 Fix custom auth override on Jenkins restart inside pod

docs/security.md

@@ -6,7 +6,10 @@ By default **jenkins-operator** performs an initial security hardening of Jenkin
 
 Currently **jenkins-operator** generates a username and random password and stores them in a Kubernetes Secret.
 However any other authorization mechanisms are possible and can be done via groovy scripts or configuration as code plugin.
-For more information take a look at [getting-started#jenkins-customization](getting-started.md#jenkins-customisation). 
+For more information take a look at [getting-started#jenkins-customization](getting-started.md#jenkins-customisation).
+
+Any change to Security Realm or Authorization requires that user called `jenkins-operator` must have admin rights 
+because **jenkins-operator** calls Jenkins API.
 
 ## Jenkins Hardening
 

internal/errors/format.go

@@ -0,0 +1,19 @@
+package errors
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/pkg/errors"
+)
+
+// Format helps to implement fmt.Formatter used by Sprint(f) or Fprint(f) etc.
+func Format(err error, s fmt.State, verb rune) {
+	formatter, ok := errors.WithStack(err).(fmt.Formatter)
+	if !ok {
+		// should never occur if the error was wrapped properly
+		panic(errors.New("this was unexpected, merged error is not fmt.Formatter"))
+	}
+	_, _ = io.WriteString(s, err.Error())
+	formatter.Format(s, verb)
+}

internal/time/time.go

@@ -0,0 +1,13 @@
+package time
+
+import "time"
+
+// Every will send the time with a period specified by the duration argument.
+// It id equivalent to time.NewTicker(d).C
+// It adjusts the intervals or drops ticks to make up for slow receivers.
+// The duration d must be greater than zero; if not, NewTicker will panic.
+// If efficiency is a concern, use NewTicker and call Ticker.Stop
+// if the ticker is no longer needed.
+func Every(d time.Duration) <-chan time.Time {
+	return time.NewTicker(d).C
+}

internal/try/until.go

@@ -0,0 +1,53 @@
+package try
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/jenkinsci/kubernetes-operator/internal/errors"
+	time2 "github.com/jenkinsci/kubernetes-operator/internal/time"
+)
+
+// ErrTimeout is used when the set timeout has been reached
+type ErrTimeout struct {
+	text  string
+	cause error
+}
+
+func (e *ErrTimeout) Error() string {
+	return fmt.Sprintf("%s: %s", e.text, e.cause.Error())
+}
+
+// Cause returns the error that caused ErrTimeout
+func (e *ErrTimeout) Cause() error {
+	return e.cause
+}
+
+// Format implements fmt.Formatter used by Sprint(f) or Fprint(f) etc.
+func (e *ErrTimeout) Format(s fmt.State, verb rune) {
+	errors.Format(e.cause, s, verb)
+}
+
+// Until keeps trying until timeout or there is a result or an error
+func Until(something func() (end bool, err error), tick, timeout time.Duration) error {
+	counter := 0
+	tickChan := time2.Every(tick)
+	timeoutChan := time.After(timeout)
+	var lastErr error
+	for {
+		select {
+		case <-tickChan:
+			end, err := something()
+			lastErr = err
+			if end {
+				return err
+			}
+			counter = counter + 1
+		case <-timeoutChan:
+			return &ErrTimeout{
+				text:  fmt.Sprintf("timed out after: %s, tries: %d", timeout, counter),
+				cause: lastErr,
+			}
+		}
+	}
+}

pkg/controller/jenkins/client/jenkins.go

@@ -51,7 +51,7 @@ type Jenkins interface {
 	GetAllViews() ([]*gojenkins.View, error)
 	CreateView(name string, viewType string) (*gojenkins.View, error)
 	Poll() (int, error)
-	ExecuteScript(groovyScript string) (output string, err error)
+	ExecuteScript(groovyScript string) (logs string, err error)
 }
 
 type jenkins struct {

pkg/controller/jenkins/configuration/base/reconcile.go

@@ -237,7 +237,7 @@ func (r *ReconcileJenkinsBaseConfiguration) createBaseConfigurationConfigMap(met
 
 func (r *ReconcileJenkinsBaseConfiguration) createUserConfigurationConfigMap(meta metav1.ObjectMeta) error {
 	currentConfigMap := &corev1.ConfigMap{}
-	err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: resources.GetUserConfigurationConfigMapName(r.jenkins), Namespace: r.jenkins.Namespace}, currentConfigMap)
+	err := r.k8sClient.Get(context.TODO(), types.NamespacedName{Name: resources.GetUserConfigurationConfigMapNameFromJenkins(r.jenkins), Namespace: r.jenkins.Namespace}, currentConfigMap)
 	if err != nil && errors.IsNotFound(err) {
 		return stackerr.WithStack(r.k8sClient.Create(context.TODO(), resources.NewUserConfigurationConfigMap(r.jenkins)))
 	} else if err != nil {

pkg/controller/jenkins/configuration/base/resources/init_configuration_configmap.go

@@ -17,28 +17,35 @@ var createOperatorUserGroovyFmtTemplate = template.Must(template.New(createOpera
 import hudson.security.*
 
 def jenkins = jenkins.model.Jenkins.getInstance()
+def operatorUserCreatedFile = new File('{{ .OperatorUserCreatedFilePath }}')
 
-def hudsonRealm = new HudsonPrivateSecurityRealm(false)
+if (!operatorUserCreatedFile.exists()) {
-hudsonRealm.createAccount(
+	def hudsonRealm = new HudsonPrivateSecurityRealm(false)
-	new File('{{ .OperatorCredentialsPath }}/{{ .OperatorUserNameFile }}').text,
+	hudsonRealm.createAccount(
-	new File('{{ .OperatorCredentialsPath }}/{{ .OperatorPasswordFile }}').text)
+		new File('{{ .OperatorCredentialsPath }}/{{ .OperatorUserNameFile }}').text,
-jenkins.setSecurityRealm(hudsonRealm)
+		new File('{{ .OperatorCredentialsPath }}/{{ .OperatorPasswordFile }}').text)
+	jenkins.setSecurityRealm(hudsonRealm)
 
-def strategy = new FullControlOnceLoggedInAuthorizationStrategy()
+	def strategy = new FullControlOnceLoggedInAuthorizationStrategy()
-strategy.setAllowAnonymousRead(false)
+	strategy.setAllowAnonymousRead(false)
-jenkins.setAuthorizationStrategy(strategy)
+	jenkins.setAuthorizationStrategy(strategy)
-jenkins.save()
+	jenkins.save()
+
+	operatorUserCreatedFile.createNewFile()
+}
 `))
 
 func buildCreateJenkinsOperatorUserGroovyScript() (*string, error) {
 	data := struct {
-		OperatorCredentialsPath string
+		OperatorCredentialsPath     string
-		OperatorUserNameFile    string
+		OperatorUserNameFile        string
-		OperatorPasswordFile    string
+		OperatorPasswordFile        string
+		OperatorUserCreatedFilePath string
 	}{
-		OperatorCredentialsPath: jenkinsOperatorCredentialsVolumePath,
+		OperatorCredentialsPath:     jenkinsOperatorCredentialsVolumePath,
-		OperatorUserNameFile:    OperatorCredentialsSecretUserNameKey,
+		OperatorUserNameFile:        OperatorCredentialsSecretUserNameKey,
-		OperatorPasswordFile:    OperatorCredentialsSecretPasswordKey,
+		OperatorPasswordFile:        OperatorCredentialsSecretPasswordKey,
+		OperatorUserCreatedFilePath: jenkinsHomePath + "/operatorUserCreated",
 	}
 
 	output, err := render(createOperatorUserGroovyFmtTemplate, data)

pkg/controller/jenkins/configuration/base/resources/pod.go

@@ -2,6 +2,7 @@ package resources
 
 import (
 	"fmt"
+
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
 
 	corev1 "k8s.io/api/core/v1"
@@ -11,27 +12,28 @@ import (
 
 const (
 	jenkinsHomeVolumeName = "home"
-	jenkinsHomePath       = "/var/jenkins/home"
+	jenkinsPath           = "/var/jenkins"
+	jenkinsHomePath       = jenkinsPath + "/home"
 
 	jenkinsScriptsVolumeName = "scripts"
-	jenkinsScriptsVolumePath = "/var/jenkins/scripts"
+	jenkinsScriptsVolumePath = jenkinsPath + "/scripts"
 	initScriptName           = "init.sh"
 
 	jenkinsOperatorCredentialsVolumeName = "operator-credentials"
-	jenkinsOperatorCredentialsVolumePath = "/var/jenkins/operator-credentials"
+	jenkinsOperatorCredentialsVolumePath = jenkinsPath + "/operator-credentials"
 
 	jenkinsInitConfigurationVolumeName = "init-configuration"
-	jenkinsInitConfigurationVolumePath = "/var/jenkins/init-configuration"
+	jenkinsInitConfigurationVolumePath = jenkinsPath + "/init-configuration"
 
 	jenkinsBaseConfigurationVolumeName = "base-configuration"
 	// JenkinsBaseConfigurationVolumePath is a path where are groovy scripts used to configure Jenkins
 	// this scripts are provided by jenkins-operator
-	JenkinsBaseConfigurationVolumePath = "/var/jenkins/base-configuration"
+	JenkinsBaseConfigurationVolumePath = jenkinsPath + "/base-configuration"
 
 	jenkinsUserConfigurationVolumeName = "user-configuration"
 	// JenkinsUserConfigurationVolumePath is a path where are groovy scripts used to configure Jenkins
 	// this scripts are provided by user
-	JenkinsUserConfigurationVolumePath = "/var/jenkins/user-configuration"
+	JenkinsUserConfigurationVolumePath = jenkinsPath + "/user-configuration"
 
 	httpPortName  = "http"
 	slavePortName = "slavelistener"
@@ -197,7 +199,7 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *v1alpha1.Jenkins
 					VolumeSource: corev1.VolumeSource{
 						ConfigMap: &corev1.ConfigMapVolumeSource{
 							LocalObjectReference: corev1.LocalObjectReference{
-								Name: GetUserConfigurationConfigMapName(jenkins),
+								Name: GetUserConfigurationConfigMapNameFromJenkins(jenkins),
 							},
 						},
 					},

pkg/controller/jenkins/configuration/base/resources/user_configuration_configmap.go

@@ -32,15 +32,20 @@ decorator.save();
 jenkins.save()
 `
 
-// GetUserConfigurationConfigMapName returns name of Kubernetes config map used to user configuration
+// GetUserConfigurationConfigMapNameFromJenkins returns name of Kubernetes config map used to user configuration
-func GetUserConfigurationConfigMapName(jenkins *v1alpha1.Jenkins) string {
+func GetUserConfigurationConfigMapNameFromJenkins(jenkins *v1alpha1.Jenkins) string {
 	return fmt.Sprintf("%s-user-configuration-%s", constants.OperatorName, jenkins.ObjectMeta.Name)
 }
 
+// GetUserConfigurationConfigMapName returns name of Kubernetes config map used to user configuration
+func GetUserConfigurationConfigMapName(jenkinsCRName string) string {
+	return fmt.Sprintf("%s-user-configuration-%s", constants.OperatorName, jenkinsCRName)
+}
+
 // NewUserConfigurationConfigMap builds Kubernetes config map used to user configuration
 func NewUserConfigurationConfigMap(jenkins *v1alpha1.Jenkins) *corev1.ConfigMap {
 	meta := metav1.ObjectMeta{
-		Name:      GetUserConfigurationConfigMapName(jenkins),
+		Name:      GetUserConfigurationConfigMapNameFromJenkins(jenkins),
 		Namespace: jenkins.ObjectMeta.Namespace,
 		Labels:    BuildLabelsForWatchedResources(jenkins),
 	}

pkg/controller/jenkins/configuration/user/reconcile.go

@@ -92,7 +92,7 @@ func (r *ReconcileUserConfiguration) ensureUserConfiguration(jenkinsClient jenki
 	}
 
 	configuration := &corev1.ConfigMap{}
-	namespaceName := types.NamespacedName{Namespace: r.jenkins.Namespace, Name: resources.GetUserConfigurationConfigMapName(r.jenkins)}
+	namespaceName := types.NamespacedName{Namespace: r.jenkins.Namespace, Name: resources.GetUserConfigurationConfigMapNameFromJenkins(r.jenkins)}
 	err = r.k8sClient.Get(context.TODO(), namespaceName, configuration)
 	if err != nil {
 		return reconcile.Result{}, errors.WithStack(err)

test/e2e/configuration_test.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
+	jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/user/seedjobs"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/plugins"
 
@@ -27,7 +28,7 @@ func TestConfiguration(t *testing.T) {
 	defer ctx.Cleanup()
 
 	// base
-	jenkins := createJenkinsCR(t, namespace)
+	jenkins := createJenkinsCR(t, "e2e", namespace)
 	createDefaultLimitsForContainersInNamespace(t, namespace)
 	waitForJenkinsBaseConfigurationToComplete(t, jenkins)
 
@@ -90,7 +91,7 @@ func verifyJenkinsMasterPodAttributes(t *testing.T, jenkins *v1alpha1.Jenkins) {
 	t.Log("Jenkins pod attributes are valid")
 }
 
-func verifyPlugins(t *testing.T, jenkinsClient *gojenkins.Jenkins, jenkins *v1alpha1.Jenkins) {
+func verifyPlugins(t *testing.T, jenkinsClient jenkinsclient.Jenkins, jenkins *v1alpha1.Jenkins) {
 	installedPlugins, err := jenkinsClient.GetPlugins(1)
 	if err != nil {
 		t.Fatal(err)
@@ -134,7 +135,7 @@ func isPluginValid(plugins *gojenkins.Plugins, requiredPlugin plugins.Plugin) (*
 	return p, requiredPlugin.Version == p.Version
 }
 
-func verifyJenkinsSeedJobs(t *testing.T, client *gojenkins.Jenkins, jenkins *v1alpha1.Jenkins) {
+func verifyJenkinsSeedJobs(t *testing.T, client jenkinsclient.Jenkins, jenkins *v1alpha1.Jenkins) {
 	t.Logf("Attempting to get configure seed job status '%v'", seedjobs.ConfigureSeedJobsName)
 
 	configureSeedJobs, err := client.GetJob(seedjobs.ConfigureSeedJobsName)

test/e2e/jenkins.go

@@ -2,15 +2,12 @@ package e2e
 
 import (
 	"context"
-	"fmt"
-	"net/http"
 	"testing"
 
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
 	jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
 
-	"github.com/bndr/gojenkins"
 	framework "github.com/operator-framework/operator-sdk/pkg/test"
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -42,7 +39,7 @@ func getJenkinsMasterPod(t *testing.T, jenkins *v1alpha1.Jenkins) *v1.Pod {
 	return &podList.Items[0]
 }
 
-func createJenkinsAPIClient(jenkins *v1alpha1.Jenkins) (*gojenkins.Jenkins, error) {
+func createJenkinsAPIClient(jenkins *v1alpha1.Jenkins) (jenkinsclient.Jenkins, error) {
 	adminSecret := &v1.Secret{}
 	namespaceName := types.NamespacedName{Namespace: jenkins.Namespace, Name: resources.GetOperatorCredentialsSecretName(jenkins)}
 	if err := framework.Global.Client.Get(context.TODO(), namespaceName, adminSecret); err != nil {
@@ -54,31 +51,17 @@ func createJenkinsAPIClient(jenkins *v1alpha1.Jenkins) (*gojenkins.Jenkins, erro
 		return nil, err
 	}
 
-	jenkinsClient := gojenkins.CreateJenkins(
+	return jenkinsclient.New(
-		nil,
 		jenkinsAPIURL,
 		string(adminSecret.Data[resources.OperatorCredentialsSecretUserNameKey]),
 		string(adminSecret.Data[resources.OperatorCredentialsSecretTokenKey]),
 	)
-	if _, err := jenkinsClient.Init(); err != nil {
-		return nil, err
-	}
-
-	status, err := jenkinsClient.Poll()
-	if err != nil {
-		return nil, err
-	}
-	if status != http.StatusOK {
-		return nil, fmt.Errorf("invalid status code returned: %d", status)
-	}
-
-	return jenkinsClient, nil
 }
 
-func createJenkinsCR(t *testing.T, namespace string) *v1alpha1.Jenkins {
+func createJenkinsCR(t *testing.T, name, namespace string) *v1alpha1.Jenkins {
 	jenkins := &v1alpha1.Jenkins{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "e2e",
+			Name:      name,
 			Namespace: namespace,
 		},
 		Spec: v1alpha1.JenkinsSpec{
@@ -111,7 +94,7 @@ func createJenkinsCR(t *testing.T, namespace string) *v1alpha1.Jenkins {
 	return jenkins
 }
 
-func verifyJenkinsAPIConnection(t *testing.T, jenkins *v1alpha1.Jenkins) *gojenkins.Jenkins {
+func verifyJenkinsAPIConnection(t *testing.T, jenkins *v1alpha1.Jenkins) jenkinsclient.Jenkins {
 	client, err := createJenkinsAPIClient(jenkins)
 	if err != nil {
 		t.Fatal(err)

test/e2e/restart_pod_test.go

@@ -1,37 +0,0 @@
-package e2e
-
-import (
-	"context"
-	"testing"
-
-	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
-
-	framework "github.com/operator-framework/operator-sdk/pkg/test"
-	"k8s.io/apimachinery/pkg/types"
-)
-
-func TestJenkinsMasterPodRestart(t *testing.T) {
-	t.Parallel()
-	namespace, ctx := setupTest(t)
-	// Deletes test namespace
-	defer ctx.Cleanup()
-
-	jenkins := createJenkinsCR(t, namespace)
-	waitForJenkinsBaseConfigurationToComplete(t, jenkins)
-	restartJenkinsMasterPod(t, jenkins)
-	waitForRecreateJenkinsMasterPod(t, jenkins)
-	checkBaseConfigurationCompleteTimeIsNotSet(t, jenkins)
-	waitForJenkinsBaseConfigurationToComplete(t, jenkins)
-}
-
-func checkBaseConfigurationCompleteTimeIsNotSet(t *testing.T, jenkins *v1alpha1.Jenkins) {
-	jenkinsStatus := &v1alpha1.Jenkins{}
-	namespaceName := types.NamespacedName{Namespace: jenkins.Namespace, Name: jenkins.Name}
-	err := framework.Global.Client.Get(context.TODO(), namespaceName, jenkinsStatus)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if jenkinsStatus.Status.BaseConfigurationCompletedTime != nil {
-		t.Fatalf("Status.BaseConfigurationCompletedTime is set after pod restart, status %+v", jenkinsStatus.Status)
-	}
-}

test/e2e/restart_test.go

@@ -0,0 +1,99 @@
+package e2e
+
+import (
+	"context"
+	"testing"
+
+	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
+	jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
+	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
+
+	framework "github.com/operator-framework/operator-sdk/pkg/test"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func TestJenkinsMasterPodRestart(t *testing.T) {
+	t.Parallel()
+	namespace, ctx := setupTest(t)
+	// Deletes test namespace
+	defer ctx.Cleanup()
+
+	jenkins := createJenkinsCR(t, "e2e", namespace)
+	waitForJenkinsBaseConfigurationToComplete(t, jenkins)
+	restartJenkinsMasterPod(t, jenkins)
+	waitForRecreateJenkinsMasterPod(t, jenkins)
+	checkBaseConfigurationCompleteTimeIsNotSet(t, jenkins)
+	waitForJenkinsBaseConfigurationToComplete(t, jenkins)
+}
+
+func TestSafeRestart(t *testing.T) {
+	t.Parallel()
+	namespace, ctx := setupTest(t)
+	// Deletes test namespace
+	defer ctx.Cleanup()
+
+	jenkinsCRName := "e2e"
+	configureAuthorizationToUnSecure(t, jenkinsCRName, namespace)
+	jenkins := createJenkinsCR(t, jenkinsCRName, namespace)
+	waitForJenkinsBaseConfigurationToComplete(t, jenkins)
+	waitForJenkinsUserConfigurationToComplete(t, jenkins)
+	jenkinsClient := verifyJenkinsAPIConnection(t, jenkins)
+	checkIfAuthorizationStrategyUnsecuredIsSet(t, jenkinsClient)
+
+	err := jenkinsClient.SafeRestart()
+	require.NoError(t, err)
+	waitForJenkinsSafeRestart(t, jenkinsClient)
+
+	checkIfAuthorizationStrategyUnsecuredIsSet(t, jenkinsClient)
+}
+
+func configureAuthorizationToUnSecure(t *testing.T, jenkinsCRName, namespace string) {
+	limitRange := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      resources.GetUserConfigurationConfigMapName(jenkinsCRName),
+			Namespace: namespace,
+		},
+		Data: map[string]string{
+			"set-unsecured-authorization.groovy": `
+import hudson.security.*
+
+def jenkins = jenkins.model.Jenkins.getInstance()
+
+def strategy = new AuthorizationStrategy.Unsecured()
+jenkins.setAuthorizationStrategy(strategy)
+jenkins.save()
+`,
+		},
+	}
+
+	err := framework.Global.Client.Create(context.TODO(), limitRange, nil)
+	require.NoError(t, err)
+}
+
+func checkIfAuthorizationStrategyUnsecuredIsSet(t *testing.T, jenkinsClient jenkinsclient.Jenkins) {
+	logs, err := jenkinsClient.ExecuteScript(`
+	import hudson.security.*
+	  
+	def jenkins = jenkins.model.Jenkins.getInstance()
+	
+	if (!(jenkins.getAuthorizationStrategy() instanceof AuthorizationStrategy.Unsecured)) {
+	  throw new Exception('AuthorizationStrategy.Unsecured is not set')
+	}
+	`)
+	require.NoError(t, err, logs)
+}
+
+func checkBaseConfigurationCompleteTimeIsNotSet(t *testing.T, jenkins *v1alpha1.Jenkins) {
+	jenkinsStatus := &v1alpha1.Jenkins{}
+	namespaceName := types.NamespacedName{Namespace: jenkins.Namespace, Name: jenkins.Name}
+	err := framework.Global.Client.Get(context.TODO(), namespaceName, jenkinsStatus)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if jenkinsStatus.Status.BaseConfigurationCompletedTime != nil {
+		t.Fatalf("Status.BaseConfigurationCompletedTime is set after pod restart, status %+v", jenkinsStatus.Status)
+	}
+}

test/e2e/wait.go

@@ -3,13 +3,18 @@ package e2e
 import (
 	goctx "context"
 	"fmt"
+	"net/http"
 	"testing"
 	"time"
 
+	"github.com/jenkinsci/kubernetes-operator/internal/try"
 	"github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkinsio/v1alpha1"
+	jenkinsclient "github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/client"
 	"github.com/jenkinsci/kubernetes-operator/pkg/controller/jenkins/configuration/base/resources"
 
 	framework "github.com/operator-framework/operator-sdk/pkg/test"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
@@ -69,6 +74,20 @@ func waitForJenkinsUserConfigurationToComplete(t *testing.T, jenkins *v1alpha1.J
 	t.Log("Jenkins pod is running")
 }
 
+func waitForJenkinsSafeRestart(t *testing.T, jenkinsClient jenkinsclient.Jenkins) {
+	err := try.Until(func() (end bool, err error) {
+		status, err := jenkinsClient.Poll()
+		if err != nil {
+			return false, err
+		}
+		if status != http.StatusOK {
+			return false, errors.Wrap(err, "couldn't poll data from Jenkins API")
+		}
+		return true, nil
+	}, time.Second, time.Second*70)
+	require.NoError(t, err)
+}
+
 // WaitUntilJenkinsConditionTrue retries until the specified condition check becomes true for the jenkins CR
 func WaitUntilJenkinsConditionTrue(retryInterval time.Duration, retries int, jenkins *v1alpha1.Jenkins, checkCondition checkConditionFunc) (*v1alpha1.Jenkins, error) {
 	jenkinsStatus := &v1alpha1.Jenkins{}