diff --git a/backup/pvc/Dockerfile b/backup/pvc/Dockerfile index 834c8c92..3b283e16 100644 --- a/backup/pvc/Dockerfile +++ b/backup/pvc/Dockerfile @@ -13,7 +13,10 @@ ARG GID ENV USER=user -RUN addgroup --gid "$GID" "$USER" && \ +RUN apt update \ + && apt install -y procps zstd \ + && rm -rf /var/lib/apt/lists/* \ + && addgroup --gid "$GID" "$USER" && \ adduser \ --disabled-password \ --gecos "" \ diff --git a/backup/pvc/bin/backup.sh b/backup/pvc/bin/backup.sh index 74f32f88..fad869da 100644 --- a/backup/pvc/bin/backup.sh +++ b/backup/pvc/bin/backup.sh @@ -6,7 +6,7 @@ set -eo pipefail [[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1; [[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1; BACKUP_TMP_DIR=$(mktemp -d) -trap "test -d "${BACKUP_TMP_DIR}" && rm -fr "${BACKUP_TMP_DIR}"" EXIT ERR SIGINT SIGTERM +trap "test -d "${BACKUP_TMP_DIR}" && rm -fr "${BACKUP_TMP_DIR}"" EXIT SIGINT SIGTERM backup_number=$1 echo "Running backup" @@ -15,12 +15,22 @@ echo "Running backup" # config.xml in child directories is state that should. For example- # branches/myorg/branches/myrepo/branches/master/config.xml should be retained while # branches/myorg/config.xml should not -tar -C "${JENKINS_HOME}" -czf "${BACKUP_TMP_DIR}/${backup_number}.tar.gz" --exclude jobs/*/workspace* --no-wildcards-match-slash --anchored --exclude jobs/*/config.xml -c jobs && \ -mv "${BACKUP_TMP_DIR}/${backup_number}.tar.gz" "${BACKUP_DIR}/${backup_number}.tar.gz" +tar --zstd -C "${JENKINS_HOME}" -cf "${BACKUP_TMP_DIR}/${backup_number}.tar.zstd" \ + --exclude jobs/*/workspace* \ + --no-wildcards-match-slash --anchored \ + --ignore-failed-read \ + --exclude jobs/*/config.xml -c jobs || ret=$? + +if [[ "$ret" -eq 0 ]]; then + echo "Backup was completed without warnings" + mv "${BACKUP_TMP_DIR}/${backup_number}.tar.zstd" "${BACKUP_DIR}/${backup_number}.tar.zstd" +elif [[ "$ret" -eq 1 ]]; then + echo "Backup was completed with some warnings" + mv "${BACKUP_TMP_DIR}/${backup_number}.tar.zstd" "${BACKUP_DIR}/${backup_number}.tar.zstd" +fi rm -rf "${BACKUP_TMP_DIR}" - -[[ ! -s ${BACKUP_DIR}/${backup_number}.tar.gz ]] && echo "backup file '${BACKUP_DIR}/${backup_number}.tar.gz' is empty" && exit 1; +[[ ! -s ${BACKUP_DIR}/${backup_number}.tar.zstd ]] && echo "backup file '${BACKUP_DIR}/${backup_number}.tar.zstd' is empty" && exit 1; echo Done exit 0 diff --git a/backup/pvc/bin/get-latest.sh b/backup/pvc/bin/get-latest.sh index 2f418a5f..5deba063 100644 --- a/backup/pvc/bin/get-latest.sh +++ b/backup/pvc/bin/get-latest.sh @@ -4,7 +4,7 @@ set -eo pipefail [[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1 -latest=$(find ${BACKUP_DIR} -name '*.tar.gz' -exec basename {} \; | sort -g | tail -n 1) +latest=$(find ${BACKUP_DIR} -name '*.tar.zstd' -exec basename {} \; | sort -g | tail -n 1) if [[ "${latest}" == "" ]]; then echo "-1" diff --git a/backup/pvc/bin/restore.sh b/backup/pvc/bin/restore.sh index 5b6f504b..994208d3 100644 --- a/backup/pvc/bin/restore.sh +++ b/backup/pvc/bin/restore.sh @@ -7,9 +7,23 @@ set -eo pipefail [[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1; backup_number=$1 +backup_file="${BACKUP_DIR}/${backup_number}" echo "Running restore backup with backup number #${backup_number}" -tar -C ${JENKINS_HOME} -zxf "${BACKUP_DIR}/${backup_number}.tar.gz" +if [[ -f "$backup_file.tar.gz" ]]; then + echo "Old format tar.gz found, restoring it" + OPTS="" + EXT="tar.gz" +elif [[ -f "$backup_file.tar.zstd" ]]; then + echo "Backup file found, proceeding" + OPTS="--zstd" + EXT="tar.zstd" +else + echo "ERR: Backup file not found: $backup_file" + exit 1 +fi + +tar $OPTS -C "${JENKINS_HOME}" -xf "${BACKUP_DIR}/${backup_number}.${EXT}" echo Done exit 0 diff --git a/backup/pvc/bin/run.sh b/backup/pvc/bin/run.sh index d764c696..56c2d687 100644 --- a/backup/pvc/bin/run.sh +++ b/backup/pvc/bin/run.sh @@ -11,6 +11,6 @@ do if [[ ! -z "${BACKUP_COUNT}" ]]; then echo "Trimming to only ${BACKUP_COUNT} recent backups in preparation for new backup" #TODO: add the list of exceeding backup before delete - find ${BACKUP_DIR} -maxdepth 1 -name '*.tar.gz' -exec basename {} \; | sort -gr | tail -n +$((BACKUP_COUNT +1)) | xargs -I '{}' rm ${BACKUP_DIR}/'{}' + find ${BACKUP_DIR} -maxdepth 1 -name '*.tar.*' -exec basename {} \; | sort -gr | tail -n +$((BACKUP_COUNT +1)) | xargs -I '{}' rm ${BACKUP_DIR}/'{}' fi done diff --git a/backup/pvc/e2e/backup_and_restore/test.sh b/backup/pvc/e2e/backup_and_restore/test.sh index df5d1519..e69901c5 100755 --- a/backup/pvc/e2e/backup_and_restore/test.sh +++ b/backup/pvc/e2e/backup_and_restore/test.sh @@ -30,7 +30,7 @@ trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${RESTORE_FOLDE backup_number=1 docker exec ${cid} /home/user/bin/backup.sh ${backup_number} -backup_file="${BACKUP_DIR}/${backup_number}.tar.gz" +backup_file="${BACKUP_DIR}/${backup_number}.tar.zstd" [[ ! -f ${backup_file} ]] && echo "Backup file ${backup_file} not found" && exit 1; docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/restore.sh ${backup_number}" @@ -38,4 +38,4 @@ docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/r echo "Compare directories" diff --brief --recursive "${RESTORE_FOLDER}" "${JENKINS_HOME_AFTER_RESTORE}" echo "Directories are the same" -echo PASS \ No newline at end of file +echo PASS diff --git a/backup/pvc/e2e/get-latest/test.sh b/backup/pvc/e2e/get-latest/test.sh index b9b228d4..a0aabe9a 100755 --- a/backup/pvc/e2e/get-latest/test.sh +++ b/backup/pvc/e2e/get-latest/test.sh @@ -19,17 +19,17 @@ mkdir -p ${BACKUP_DIR} mkdir -p ${JENKINS_HOME} mkdir -p ${BACKUP_DIR}/lost+found -touch ${BACKUP_DIR}/1.tar.gz -touch ${BACKUP_DIR}/2.tar.gz -touch ${BACKUP_DIR}/3.tar.gz -touch ${BACKUP_DIR}/4.tar.gz -touch ${BACKUP_DIR}/5.tar.gz -touch ${BACKUP_DIR}/6.tar.gz -touch ${BACKUP_DIR}/7.tar.gz -touch ${BACKUP_DIR}/8.tar.gz -touch ${BACKUP_DIR}/9.tar.gz -touch ${BACKUP_DIR}/10.tar.gz -touch ${BACKUP_DIR}/11.tar.gz +touch ${BACKUP_DIR}/1.tar.zstd +touch ${BACKUP_DIR}/2.tar.zstd +touch ${BACKUP_DIR}/3.tar.zstd +touch ${BACKUP_DIR}/4.tar.zstd +touch ${BACKUP_DIR}/5.tar.zstd +touch ${BACKUP_DIR}/6.tar.zstd +touch ${BACKUP_DIR}/7.tar.zstd +touch ${BACKUP_DIR}/8.tar.zstd +touch ${BACKUP_DIR}/9.tar.zstd +touch ${BACKUP_DIR}/10.tar.zstd +touch ${BACKUP_DIR}/11.tar.zstd # Create an instance of the container under testing cid="$(docker run -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})" @@ -39,7 +39,7 @@ echo "Docker container ID '${cid}'" trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh") -rm ${BACKUP_DIR}/*.tar.gz +rm ${BACKUP_DIR}/*.tar.zstd empty_latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh") if [[ "${DEBUG}" ]]; then diff --git a/backup/pvc/e2e/limit_backup_count/test.sh b/backup/pvc/e2e/limit_backup_count/test.sh index 7c4c66ee..469b3e44 100755 --- a/backup/pvc/e2e/limit_backup_count/test.sh +++ b/backup/pvc/e2e/limit_backup_count/test.sh @@ -19,17 +19,17 @@ mkdir -p ${BACKUP_DIR} mkdir -p ${JENKINS_HOME} mkdir -p ${BACKUP_DIR}/lost+found -touch ${BACKUP_DIR}/1.tar.gz -touch ${BACKUP_DIR}/2.tar.gz -touch ${BACKUP_DIR}/3.tar.gz -touch ${BACKUP_DIR}/4.tar.gz -touch ${BACKUP_DIR}/5.tar.gz -touch ${BACKUP_DIR}/6.tar.gz -touch ${BACKUP_DIR}/7.tar.gz -touch ${BACKUP_DIR}/8.tar.gz -touch ${BACKUP_DIR}/9.tar.gz -touch ${BACKUP_DIR}/10.tar.gz -touch ${BACKUP_DIR}/11.tar.gz +touch ${BACKUP_DIR}/1.tar.zstd +touch ${BACKUP_DIR}/2.tar.zstd +touch ${BACKUP_DIR}/3.tar.zstd +touch ${BACKUP_DIR}/4.tar.zstd +touch ${BACKUP_DIR}/5.tar.zstd +touch ${BACKUP_DIR}/6.tar.zstd +touch ${BACKUP_DIR}/7.tar.zstd +touch ${BACKUP_DIR}/8.tar.zstd +touch ${BACKUP_DIR}/9.tar.zstd +touch ${BACKUP_DIR}/10.tar.zstd +touch ${BACKUP_DIR}/11.tar.zstd # Create an instance of the container under testing cid="$(docker run -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})" @@ -39,7 +39,7 @@ echo "Docker container ID '${cid}'" trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT sleep 11 -touch ${BACKUP_DIR}/12.tar.gz +touch ${BACKUP_DIR}/12.tar.zstd sleep 11 if [[ "${DEBUG}" ]]; then @@ -48,7 +48,7 @@ if [[ "${DEBUG}" ]]; then fi # only two latest backup should exists -[[ $(ls -1 ${BACKUP_DIR} | grep 'tar.gz' | wc -l) -eq 2 ]] || exit 1 -[[ -f ${BACKUP_DIR}/11.tar.gz ]] || exit 2 -[[ -f ${BACKUP_DIR}/12.tar.gz ]] || exit 3 +[[ $(ls -1 ${BACKUP_DIR} | grep 'tar.zstd' | wc -l) -eq 2 ]] || exit 1 +[[ -f ${BACKUP_DIR}/11.tar.zstd ]] || exit 2 +[[ -f ${BACKUP_DIR}/12.tar.zstd ]] || exit 3 echo PASS diff --git a/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh b/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh index 7b963357..662809c8 100755 --- a/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh +++ b/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh @@ -29,8 +29,8 @@ docker exec ${cid} /home/user/bin/backup.sh ${backup_number} [ "$(docker exec ${cid} ls /tmp | grep 'tmp')" ] && echo "tmp directory not empty" && exit 1; -backup_file="${BACKUP_DIR}/${backup_number}.tar.gz" +backup_file="${BACKUP_DIR}/${backup_number}.tar.zstd" [[ ! -f ${backup_file} ]] && echo "Backup file ${backup_file} not found" && exit 1; echo "tmp directory empty, backup in backup directory present" -echo PASS \ No newline at end of file +echo PASS