fix(reconcile): ensure backup reconciliation succeeds during bats tests

* now stdout and stderr from Exec calls are properly returned so output of backup jobs can be checked
* now runs backup reconciliation before seed jobs reconciliation
* backups would never be performed during bats tests because of errors and/or requeuing requests from ensureSeedJobs

backup/pvc/bin/get-latest.sh

@@ -24,6 +24,7 @@ is_backup_not_exist() {
 
 # Check if we have any backup
 if is_backup_not_exist "${BACKUP_DIR}"; then
+  _log "No backups exist in ${BACKUP_DIR}"
   echo "-1"
   exit 0
 fi
@@ -32,6 +33,7 @@ fi
 latest=$(find "${BACKUP_DIR}"/*.tar.* -maxdepth 0 -exec basename {} \; | sort -g | tail -n 1)
 
 if [[ "${latest}" == "" ]]; then
+  _log "Could not get the latest backup."
   echo "-1"
 else
   echo "${latest%%.*}"

pkg/configuration/backuprestore/backuprestore.go

@@ -149,7 +149,7 @@ func (bar *BackupAndRestore) Restore(jenkinsClient jenkinsclient.Jenkins) error
 
 		backupNumberString := strings.TrimSuffix(backupNumberRaw.String(), "\n")
 		if backupNumberString == noBackup {
-			bar.logger.V(log.VDebug).Info("Skipping restore backup, get latest action returned -1")
+			bar.logger.V(log.VDebug).Info("Skipping restore backup, get latest action returned -1 (no backups found)")
 			jenkins.Status.LastBackup = 0
 			jenkins.Status.PendingBackup = 1
 			return bar.Client.Status().Update(context.TODO(), jenkins)
@@ -209,7 +209,7 @@ func (bar *BackupAndRestore) Restore(jenkinsClient jenkinsclient.Jenkins) error
 func (bar *BackupAndRestore) Backup(setBackupDoneBeforePodDeletion bool) error {
 	jenkins := bar.Configuration.Jenkins
 	if len(jenkins.Spec.Backup.ContainerName) == 0 || jenkins.Spec.Backup.Action.Exec == nil {
-		bar.logger.V(log.VDebug).Info("Skipping restore backup, backup restore not configured")
+		bar.logger.V(log.VDebug).Info("Skipping backup, backup creation not configured")
 		return nil
 	}
 	if jenkins.Status.PendingBackup == jenkins.Status.LastBackup {

pkg/configuration/configuration.go

@@ -165,7 +165,7 @@ func (c *Configuration) Exec(podName, containerName string, command []string) (s
 		return stdout, stderr, stackerr.Wrapf(err, "pod exec error operation on stream: stdout '%s' stderr '%s'", stdout.String(), stderr.String())
 	}
 
-	return bytes.Buffer{}, bytes.Buffer{}, nil
+	return stdout, stderr, nil
 }
 
 // GetJenkinsMasterContainer returns the Jenkins master container from the CR.

pkg/configuration/user/reconcile.go

@@ -60,6 +60,14 @@ func (r *reconcileUserConfiguration) ReconcileOthers() (reconcile.Result, error)
 		return reconcile.Result{}, err
 	}
 
+	if err := backupAndRestore.Backup(false); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	if err := backupAndRestore.EnsureBackupTrigger(); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	result, err := r.ensureSeedJobs()
 	if err != nil {
 		return reconcile.Result{}, err
@@ -68,13 +76,6 @@ func (r *reconcileUserConfiguration) ReconcileOthers() (reconcile.Result, error)
 		return result, nil
 	}
 
-	if err := backupAndRestore.Backup(false); err != nil {
-		return reconcile.Result{}, err
-	}
-	if err := backupAndRestore.EnsureBackupTrigger(); err != nil {
-		return reconcile.Result{}, err
-	}
-
 	return reconcile.Result{}, nil
 }
 

test/bats/2-deploy-with-more-options.bats

@@ -89,7 +89,11 @@ setup() {
 @test "2.8  Helm: check backup" {
   [[ ! -f "chart/jenkins-operator/deploy.tmp" ]] && skip "Jenkins helm chart have not been deployed correctly"
   sleep 120
-  run ${KUBECTL} logs -l app.kubernetes.io/name=jenkins-operator --tail 10000
+
+  # use --tail -1 to get all logs and reduce flakiness
+  # using -l to select a label usually sets --tail 10
+  run ${KUBECTL} logs -l app.kubernetes.io/name=jenkins-operator --tail -1
+
   assert_success
   assert_output --partial "Performing backup '1'"
   assert_output --partial "Backup completed '1', updating status"

test/bats/test_helper.bash

@@ -3,7 +3,7 @@ _common_setup() {
     bats_load_library bats-support
     bats_load_library bats-assert
     bats_load_library bats-file
-    bats_load_library bats-detik/detik.bash
+    bats_load_library bats-detik/lib/detik.bash
 
     CONTEXT="kind-jenkins"
     export DETIK_CLIENT_NAME="kubectl"