Trigger backup before Jenkins master pod deletion

README.md

@@ -55,7 +55,6 @@ Base configuration:
 
 User configuration:
 * backup and restore for Jenkins jobs running as standalone job (AWS, GCP, Azure)
-* trigger backup job before pod deletion using preStop k8s hooks
 * verify Jenkins configuration events
 
 [installation]:docs/installation.md

pkg/controller/jenkins/configuration/base/resources/pod.go

@@ -16,6 +16,7 @@ const (
 	jenkinsScriptsVolumeName = "scripts"
 	jenkinsScriptsVolumePath = "/var/jenkins/scripts"
 	initScriptName           = "init.sh"
+	backupScriptName         = "backup.sh"
 
 	jenkinsOperatorCredentialsVolumeName = "operator-credentials"
 	jenkinsOperatorCredentialsVolumePath = "/var/jenkins/operator-credentials"
@@ -34,7 +35,9 @@ const (
 	JenkinsUserConfigurationVolumePath = "/var/jenkins/user-configuration"
 
 	jenkinsBackupCredentialsVolumeName = "backup-credentials"
-	jenkinsBackupCredentialsVolumePath = "/var/jenkins/backup-credentials"
+	// JenkinsBackupCredentialsVolumePath is a path where are credentials used for backup/restore
+	// credentials are provided by user
+	JenkinsBackupCredentialsVolumePath = "/var/jenkins/backup-credentials"
 
 	httpPortName  = "http"
 	slavePortName = "slavelistener"
@@ -81,6 +84,16 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *virtuslabv1alpha
 						"bash",
 						fmt.Sprintf("%s/%s", jenkinsScriptsVolumePath, initScriptName),
 					},
+					Lifecycle: &corev1.Lifecycle{
+						PreStop: &corev1.Handler{
+							Exec: &corev1.ExecAction{
+								Command: []string{
+									"bash",
+									fmt.Sprintf("%s/%s", jenkinsScriptsVolumePath, backupScriptName),
+								},
+							},
+						},
+					},
 					LivenessProbe: &corev1.Probe{
 						Handler: corev1.Handler{
 							HTTPGet: &corev1.HTTPGetAction{
@@ -157,7 +170,7 @@ func NewJenkinsMasterPod(objectMeta metav1.ObjectMeta, jenkins *virtuslabv1alpha
 						},
 						{
 							Name:      jenkinsBackupCredentialsVolumeName,
-							MountPath: jenkinsBackupCredentialsVolumePath,
+							MountPath: JenkinsBackupCredentialsVolumePath,
 							ReadOnly:  true,
 						},
 					},

pkg/controller/jenkins/configuration/base/resources/scripts_configmap.go

@@ -264,6 +264,70 @@ echo "Installing plugins - end"
 /sbin/tini -s -- /usr/local/bin/jenkins.sh
 `))
 
+const backupBashFmt = `#!/usr/bin/env bash
+
+# don't add set -e
+
+JENKINS_SERVER="http://%s:$(cat %s/%s)@localhost:%d"
+JOB="%s"
+JOB_QUERY=/job/${JOB}
+
+echo 'Starting the build'
+curl -f -v -X POST "${JENKINS_SERVER}${JOB_QUERY}/build?delay=0sec" || exit -1
+sleep 3 # give some time for Jenkins to update builds numbers
+
+BUILD_STATUS_QUERY=/lastBuild/api/json
+
+CURRENT_BUILD_NUMBER_QUERY=/lastBuild/buildNumber
+CURRENT_BUILD_JSON=$(curl -s -f "${JENKINS_SERVER}${JOB_QUERY}${CURRENT_BUILD_NUMBER_QUERY}")
+LAST_STABLE_BUILD_NUMBER_QUERY=/lastStableBuild/buildNumber
+
+check_build()
+{
+    GOOD_BUILD="Last build successful. "
+    BAD_BUILD="Last build failed. "
+    JOB_STATUS_JSON=$(curl -s -f "${JENKINS_SERVER}${JOB_QUERY}${BUILD_STATUS_QUERY}")
+    RESULT=$(echo "${JOB_STATUS_JSON}" | sed -n 's/.*"result":\([\"A-Za-z]*\),.*/\1/p')
+    CURRENT_BUILD_NUMBER=${CURRENT_BUILD_JSON}
+    LAST_STABLE_BUILD_JSON=$(curl --silent "${JENKINS_SERVER}${JOB_QUERY}${LAST_STABLE_BUILD_NUMBER_QUERY}")
+    LAST_STABLE_BUILD_NUMBER=${LAST_STABLE_BUILD_JSON}
+    LAST_BUILD_STATUS=${GOOD_BUILD}
+    echo "${LAST_STABLE_BUILD_NUMBER}" | grep "is not available" > /dev/null
+    GREP_RETURN_CODE=$?
+    if [[ ${GREP_RETURN_CODE} -ne 0 ]]
+    then
+        if [[ $(expr ${CURRENT_BUILD_NUMBER} - 1) -gt ${LAST_STABLE_BUILD_NUMBER} ]]
+        then
+            LAST_BUILD_STATUS=${BAD_BUILD}
+        fi
+    fi
+
+    if [[ "${RESULT}" = "null" ]]
+    then
+        echo "${LAST_BUILD_STATUS}Building ${JOB} ${CURRENT_BUILD_NUMBER}... last stable was ${LAST_STABLE_BUILD_NUMBER}"
+    elif [[ "${RESULT}" = "\"SUCCESS\"" ]]
+    then
+        echo "${LAST_BUILD_STATUS}${JOB} ${CURRENT_BUILD_NUMBER} completed successfully."
+        exit 0
+    elif [[ "${RESULT}" = "\"FAILURE\"" ]]
+    then
+        LAST_BUILD_STATUS=${BAD_BUILD}
+        echo "${LAST_BUILD_STATUS}${JOB} ${CURRENT_BUILD_NUMBER} failed"
+        exit -1
+    else
+        LAST_BUILD_STATUS=${BAD_BUILD}
+        echo "${LAST_BUILD_STATUS}${JOB} ${CURRENT_BUILD_NUMBER} status unknown - '${RESULT}'"
+        exit -1
+    fi
+}
+
+while [[ true ]]
+do
+    check_build
+    sleep 1
+done
+`
+
 func buildConfigMapTypeMeta() metav1.TypeMeta {
 	return metav1.TypeMeta{
 		Kind:       "ConfigMap",
@@ -313,6 +377,8 @@ func NewScriptsConfigMap(meta metav1.ObjectMeta, jenkins *virtuslabv1alpha1.Jenk
 		Data: map[string]string{
 			initScriptName:        *initBashScript,
 			installPluginsCommand: fmt.Sprintf(installPluginsBashFmt, jenkinsHomePath),
+			backupScriptName: fmt.Sprintf(backupBashFmt,
+				OperatorUserName, jenkinsOperatorCredentialsVolumePath, OperatorCredentialsSecretTokenKey, HTTPPortInt, constants.BackupJobName),
 		},
 	}, nil
 }