diff --git a/backup/pvc/bin/backup.sh b/backup/pvc/bin/backup.sh index 40c9f8dd..7e947a1e 100755 --- a/backup/pvc/bin/backup.sh +++ b/backup/pvc/bin/backup.sh @@ -22,8 +22,17 @@ done _log "INFO" "[backup] running backup ${BACKUP_NUMBER}" touch "${TRAP_FILE}" + +# since at this point we are sure that no other backup process is running, we can clean up stale tmp dirs +if [[ ! -z "${BACKUP_TMP_PREFIX}" ]]; then + find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -name "${BACKUP_TMP_PREFIX}.*" -type d -exec rm -rf {} + +else + _log "WARNING" "[backup] BACKUP_TMP_PREFIX is empty, refusing to run 'rm -rf'" +fi + + # create temp dir on the same filesystem with a BACKUP_DIR to be able use atomic mv enstead of copy -BACKUP_TMP_DIR=$(mktemp -d --tmpdir="${BACKUP_DIR}") +BACKUP_TMP_DIR=$(mktemp -d --tmpdir="${BACKUP_DIR}" "${BACKUP_TMP_PATTERN}") _clean(){ test -d "${BACKUP_TMP_DIR}" && rm -fr "${BACKUP_TMP_DIR}" diff --git a/backup/pvc/bin/utils.sh b/backup/pvc/bin/utils.sh index 718dfcf1..830af14d 100755 --- a/backup/pvc/bin/utils.sh +++ b/backup/pvc/bin/utils.sh @@ -1,6 +1,11 @@ #!/usr/bin/env bash # Common utils +# Explicitly specify default tmp prefix used by mktemp for compatibility +# see https://www.gnu.org/software/autogen/mktemp.html +BACKUP_TMP_PREFIX="tmp" +BACKUP_TMP_PATTERN="$BACKUP_TMP_PREFIX.XXXXXXXXXX" + _log() { local level="$1" local message="$2" diff --git a/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/orgjob/config.xml b/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/orgjob/config.xml new file mode 100644 index 00000000..e69de29b diff --git a/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/orgjob/jobs/branchjob/branches/master/build b/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/orgjob/jobs/branchjob/branches/master/build new file mode 100644 index 00000000..e69de29b diff --git a/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/orgjob/jobs/branchjob/branches/master/config.xml b/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/orgjob/jobs/branchjob/branches/master/config.xml new file mode 100644 index 00000000..e69de29b diff --git a/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/orgjob/jobs/branchjob/config.xml b/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/orgjob/jobs/branchjob/config.xml new file mode 100644 index 00000000..e69de29b diff --git a/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/some-job/builds/1/build.xml b/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/some-job/builds/1/build.xml new file mode 100644 index 00000000..5d0bb326 --- /dev/null +++ b/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/some-job/builds/1/build.xml @@ -0,0 +1 @@ +build.xml \ No newline at end of file diff --git a/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/some-job/builds/1/log b/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/some-job/builds/1/log new file mode 100644 index 00000000..7f236fe6 --- /dev/null +++ b/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/some-job/builds/1/log @@ -0,0 +1 @@ +build logs \ No newline at end of file diff --git a/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/some-job/config.xml b/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/some-job/config.xml new file mode 100644 index 00000000..e69de29b diff --git a/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/some-job/nextBuildNumber b/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/some-job/nextBuildNumber new file mode 100644 index 00000000..d8263ee9 --- /dev/null +++ b/backup/pvc/e2e/stale_tmp_dir_cleanup/jenkins_home/jobs/some-job/nextBuildNumber @@ -0,0 +1 @@ +2 \ No newline at end of file diff --git a/backup/pvc/e2e/stale_tmp_dir_cleanup/test.sh b/backup/pvc/e2e/stale_tmp_dir_cleanup/test.sh new file mode 100755 index 00000000..1705e301 --- /dev/null +++ b/backup/pvc/e2e/stale_tmp_dir_cleanup/test.sh @@ -0,0 +1,81 @@ +#!/bin/bash +set -eo pipefail + +echo "Running stale_tmp_dir_cleanup e2e test..." + +[[ "${DEBUG}" ]] && set -x + +# set current working directory to the directory of the script +cd "$(dirname "$0")" + +docker_image=$1 + +if ! docker inspect ${docker_image} &> /dev/null; then + echo "Image '${docker_image}' does not exists" + false +fi + +JENKINS_HOME="$(pwd)/jenkins_home" +BACKUP_DIR="$(pwd)/backup" +mkdir -p ${BACKUP_DIR} + +# Create stale directories that should be cleaned up +mkdir -p ${BACKUP_DIR}/tmp.stale1 +mkdir -p ${BACKUP_DIR}/tmp.stale2 +touch ${BACKUP_DIR}/tmp.stale1/somefile + +# Create directories that should NOT be cleaned up +mkdir -p ${BACKUP_DIR}/other_dir +mkdir -p ${BACKUP_DIR}/tmp_but_no_dot +mkdir -p ${BACKUP_DIR}/tmp +touch ${BACKUP_DIR}/other_dir/keepme + +# Create an instance of the container under testing +cid="$(docker run -e BACKUP_CLEANUP_INTERVAL=1 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})" +echo "Docker container ID '${cid}'" + +# Remove test directory and container afterwards +trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR}" EXIT + +backup_number=1 +docker exec ${cid} /home/user/bin/backup.sh ${backup_number} + +# Check cleanup results +if [ -d "${BACKUP_DIR}/tmp.stale1" ]; then + echo "FAIL: Stale directory tmp.stale1 was not removed" + exit 1 +fi + +if [ -d "${BACKUP_DIR}/tmp.stale2" ]; then + echo "FAIL: Stale directory tmp.stale2 was not removed" + exit 1 +fi + +if [ ! -d "${BACKUP_DIR}/other_dir" ]; then + echo "FAIL: Directory other_dir was incorrectly removed" + exit 1 +fi + +if [ ! -d "${BACKUP_DIR}/tmp_but_no_dot" ]; then + echo "FAIL: Directory tmp_but_no_dot was incorrectly removed" + exit 1 +fi + +if [ ! -d "${BACKUP_DIR}/tmp" ]; then + echo "FAIL: Directory tmp was incorrectly removed" + exit 1 +fi + +# Verify backup success +backup_file="${BACKUP_DIR}/${backup_number}.tar.zstd" +[[ ! -f ${backup_file} ]] && echo "Backup file ${backup_file} not found" && exit 1; + +# Verify no unexpected tmp directories remain +remaining_tmp=$(find "${BACKUP_DIR}" -maxdepth 1 -name "tmp.*") +if [ ! -z "$remaining_tmp" ]; then + echo "FAIL: Unexpected tmp directories remaining: $remaining_tmp" + exit 1 +fi + +echo "Stale temporary directories cleaned up successfully" +echo PASS