public_git
/
kubernetes-operator
mirror of https://github.com/jenkinsci/kubernetes-operator.git
1
0
Fork 0
Code Issues Releases Wiki Activity

feat: use zstd instead of gz by default, fix the backup script (#841)

This commit is contained in:
Luigi Operoso 2023-05-26 16:10:24 +02:00 committed by GitHub
parent 44a7d2460a
commit 3275be357a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 67 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backup/pvc/Dockerfile
View File

@ -13,7 +13,10 @@ ARG GID
ENV USER=user ENV USER=user
RUN addgroup --gid "$GID" "$USER" && \ RUN apt update \
&& apt install -y procps zstd \
&& rm -rf /var/lib/apt/lists/* \
&& addgroup --gid "$GID" "$USER" && \
adduser \ adduser \
--disabled-password \ --disabled-password \
--gecos "" \ --gecos "" \

20
backup/pvc/bin/backup.sh
View File

@ -6,7 +6,7 @@ set -eo pipefail
[[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1; [[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1;
[[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1; [[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1;
BACKUP_TMP_DIR=$(mktemp -d) BACKUP_TMP_DIR=$(mktemp -d)
trap "test -d "${BACKUP_TMP_DIR}" && rm -fr "${BACKUP_TMP_DIR}"" EXIT ERR SIGINT SIGTERM trap "test -d "${BACKUP_TMP_DIR}" && rm -fr "${BACKUP_TMP_DIR}"" EXIT SIGINT SIGTERM
backup_number=$1 backup_number=$1
echo "Running backup" echo "Running backup"
@ -15,12 +15,22 @@ echo "Running backup"
# config.xml in child directories is state that should. For example- # config.xml in child directories is state that should. For example-
# branches/myorg/branches/myrepo/branches/master/config.xml should be retained while # branches/myorg/branches/myrepo/branches/master/config.xml should be retained while
# branches/myorg/config.xml should not # branches/myorg/config.xml should not
tar -C "${JENKINS_HOME}" -czf "${BACKUP_TMP_DIR}/${backup_number}.tar.gz" --exclude jobs/*/workspace* --no-wildcards-match-slash --anchored --exclude jobs/*/config.xml -c jobs && \ tar --zstd -C "${JENKINS_HOME}" -cf "${BACKUP_TMP_DIR}/${backup_number}.tar.zstd" \
mv "${BACKUP_TMP_DIR}/${backup_number}.tar.gz" "${BACKUP_DIR}/${backup_number}.tar.gz" --exclude jobs/*/workspace* \
--no-wildcards-match-slash --anchored \
--ignore-failed-read \
--exclude jobs/*/config.xml -c jobs || ret=$?
if [[ "$ret" -eq 0 ]]; then
echo "Backup was completed without warnings"
mv "${BACKUP_TMP_DIR}/${backup_number}.tar.zstd" "${BACKUP_DIR}/${backup_number}.tar.zstd"
elif [[ "$ret" -eq 1 ]]; then
echo "Backup was completed with some warnings"
mv "${BACKUP_TMP_DIR}/${backup_number}.tar.zstd" "${BACKUP_DIR}/${backup_number}.tar.zstd"
fi
rm -rf "${BACKUP_TMP_DIR}" rm -rf "${BACKUP_TMP_DIR}"
[[ ! -s ${BACKUP_DIR}/${backup_number}.tar.zstd ]] && echo "backup file '${BACKUP_DIR}/${backup_number}.tar.zstd' is empty" && exit 1;
[[ ! -s ${BACKUP_DIR}/${backup_number}.tar.gz ]] && echo "backup file '${BACKUP_DIR}/${backup_number}.tar.gz' is empty" && exit 1;
echo Done echo Done
exit 0 exit 0

2
backup/pvc/bin/get-latest.sh
View File

@ -4,7 +4,7 @@ set -eo pipefail
[[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1 [[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1
latest=$(find ${BACKUP_DIR} -name '*.tar.gz' -exec basename {} \; | sort -g | tail -n 1) latest=$(find ${BACKUP_DIR} -name '*.tar.zstd' -exec basename {} \; | sort -g | tail -n 1)
if [[ "${latest}" == "" ]]; then if [[ "${latest}" == "" ]]; then
echo "-1" echo "-1"

16
backup/pvc/bin/restore.sh
View File

@ -7,9 +7,23 @@ set -eo pipefail
[[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1; [[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1;
backup_number=$1 backup_number=$1
backup_file="${BACKUP_DIR}/${backup_number}"
echo "Running restore backup with backup number #${backup_number}" echo "Running restore backup with backup number #${backup_number}"
tar -C ${JENKINS_HOME} -zxf "${BACKUP_DIR}/${backup_number}.tar.gz" if [[ -f "$backup_file.tar.gz" ]]; then
echo "Old format tar.gz found, restoring it"
OPTS=""
EXT="tar.gz"
elif [[ -f "$backup_file.tar.zstd" ]]; then
echo "Backup file found, proceeding"
OPTS="--zstd"
EXT="tar.zstd"
else
echo "ERR: Backup file not found: $backup_file"
exit 1
fi
tar $OPTS -C "${JENKINS_HOME}" -xf "${BACKUP_DIR}/${backup_number}.${EXT}"
echo Done echo Done
exit 0 exit 0

2
backup/pvc/bin/run.sh
View File

@ -11,6 +11,6 @@ do
if [[ ! -z "${BACKUP_COUNT}" ]]; then if [[ ! -z "${BACKUP_COUNT}" ]]; then
echo "Trimming to only ${BACKUP_COUNT} recent backups in preparation for new backup" echo "Trimming to only ${BACKUP_COUNT} recent backups in preparation for new backup"
#TODO: add the list of exceeding backup before delete #TODO: add the list of exceeding backup before delete
find ${BACKUP_DIR} -maxdepth 1 -name '*.tar.gz' -exec basename {} \; | sort -gr | tail -n +$((BACKUP_COUNT +1)) | xargs -I '{}' rm ${BACKUP_DIR}/'{}' find ${BACKUP_DIR} -maxdepth 1 -name '*.tar.*' -exec basename {} \; | sort -gr | tail -n +$((BACKUP_COUNT +1)) | xargs -I '{}' rm ${BACKUP_DIR}/'{}'
fi fi
done done

2
backup/pvc/e2e/backup_and_restore/test.sh
View File

@ -30,7 +30,7 @@ trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${RESTORE_FOLDE
backup_number=1 backup_number=1
docker exec ${cid} /home/user/bin/backup.sh ${backup_number} docker exec ${cid} /home/user/bin/backup.sh ${backup_number}
backup_file="${BACKUP_DIR}/${backup_number}.tar.gz" backup_file="${BACKUP_DIR}/${backup_number}.tar.zstd"
[[ ! -f ${backup_file} ]] && echo "Backup file ${backup_file} not found" && exit 1; [[ ! -f ${backup_file} ]] && echo "Backup file ${backup_file} not found" && exit 1;
docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/restore.sh ${backup_number}" docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/restore.sh ${backup_number}"

24
backup/pvc/e2e/get-latest/test.sh
View File

@ -19,17 +19,17 @@ mkdir -p ${BACKUP_DIR}
mkdir -p ${JENKINS_HOME} mkdir -p ${JENKINS_HOME}
mkdir -p ${BACKUP_DIR}/lost+found mkdir -p ${BACKUP_DIR}/lost+found
touch ${BACKUP_DIR}/1.tar.gz touch ${BACKUP_DIR}/1.tar.zstd
touch ${BACKUP_DIR}/2.tar.gz touch ${BACKUP_DIR}/2.tar.zstd
touch ${BACKUP_DIR}/3.tar.gz touch ${BACKUP_DIR}/3.tar.zstd
touch ${BACKUP_DIR}/4.tar.gz touch ${BACKUP_DIR}/4.tar.zstd
touch ${BACKUP_DIR}/5.tar.gz touch ${BACKUP_DIR}/5.tar.zstd
touch ${BACKUP_DIR}/6.tar.gz touch ${BACKUP_DIR}/6.tar.zstd
touch ${BACKUP_DIR}/7.tar.gz touch ${BACKUP_DIR}/7.tar.zstd
touch ${BACKUP_DIR}/8.tar.gz touch ${BACKUP_DIR}/8.tar.zstd
touch ${BACKUP_DIR}/9.tar.gz touch ${BACKUP_DIR}/9.tar.zstd
touch ${BACKUP_DIR}/10.tar.gz touch ${BACKUP_DIR}/10.tar.zstd
touch ${BACKUP_DIR}/11.tar.gz touch ${BACKUP_DIR}/11.tar.zstd
# Create an instance of the container under testing # Create an instance of the container under testing
cid="$(docker run -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})" cid="$(docker run -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
@ -39,7 +39,7 @@ echo "Docker container ID '${cid}'"
trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT
latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh") latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
rm ${BACKUP_DIR}/*.tar.gz rm ${BACKUP_DIR}/*.tar.zstd
empty_latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh") empty_latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
if [[ "${DEBUG}" ]]; then if [[ "${DEBUG}" ]]; then

30
backup/pvc/e2e/limit_backup_count/test.sh
View File

@ -19,17 +19,17 @@ mkdir -p ${BACKUP_DIR}
mkdir -p ${JENKINS_HOME} mkdir -p ${JENKINS_HOME}
mkdir -p ${BACKUP_DIR}/lost+found mkdir -p ${BACKUP_DIR}/lost+found
touch ${BACKUP_DIR}/1.tar.gz touch ${BACKUP_DIR}/1.tar.zstd
touch ${BACKUP_DIR}/2.tar.gz touch ${BACKUP_DIR}/2.tar.zstd
touch ${BACKUP_DIR}/3.tar.gz touch ${BACKUP_DIR}/3.tar.zstd
touch ${BACKUP_DIR}/4.tar.gz touch ${BACKUP_DIR}/4.tar.zstd
touch ${BACKUP_DIR}/5.tar.gz touch ${BACKUP_DIR}/5.tar.zstd
touch ${BACKUP_DIR}/6.tar.gz touch ${BACKUP_DIR}/6.tar.zstd
touch ${BACKUP_DIR}/7.tar.gz touch ${BACKUP_DIR}/7.tar.zstd
touch ${BACKUP_DIR}/8.tar.gz touch ${BACKUP_DIR}/8.tar.zstd
touch ${BACKUP_DIR}/9.tar.gz touch ${BACKUP_DIR}/9.tar.zstd
touch ${BACKUP_DIR}/10.tar.gz touch ${BACKUP_DIR}/10.tar.zstd
touch ${BACKUP_DIR}/11.tar.gz touch ${BACKUP_DIR}/11.tar.zstd
# Create an instance of the container under testing # Create an instance of the container under testing
cid="$(docker run -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})" cid="$(docker run -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
@ -39,7 +39,7 @@ echo "Docker container ID '${cid}'"
trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT
sleep 11 sleep 11
touch ${BACKUP_DIR}/12.tar.gz touch ${BACKUP_DIR}/12.tar.zstd
sleep 11 sleep 11
if [[ "${DEBUG}" ]]; then if [[ "${DEBUG}" ]]; then
@ -48,7 +48,7 @@ if [[ "${DEBUG}" ]]; then
fi fi
# only two latest backup should exists # only two latest backup should exists
[[ $(ls -1 ${BACKUP_DIR} | grep 'tar.gz' | wc -l) -eq 2 ]] || exit 1 [[ $(ls -1 ${BACKUP_DIR} | grep 'tar.zstd' | wc -l) -eq 2 ]] || exit 1
[[ -f ${BACKUP_DIR}/11.tar.gz ]] || exit 2 [[ -f ${BACKUP_DIR}/11.tar.zstd ]] || exit 2
[[ -f ${BACKUP_DIR}/12.tar.gz ]] || exit 3 [[ -f ${BACKUP_DIR}/12.tar.zstd ]] || exit 3
echo PASS echo PASS

2
backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh
View File

@ -29,7 +29,7 @@ docker exec ${cid} /home/user/bin/backup.sh ${backup_number}
[ "$(docker exec ${cid} ls /tmp | grep 'tmp')" ] && echo "tmp directory not empty" && exit 1; [ "$(docker exec ${cid} ls /tmp | grep 'tmp')" ] && echo "tmp directory not empty" && exit 1;
backup_file="${BACKUP_DIR}/${backup_number}.tar.gz" backup_file="${BACKUP_DIR}/${backup_number}.tar.zstd"
[[ ! -f ${backup_file} ]] && echo "Backup file ${backup_file} not found" && exit 1; [[ ! -f ${backup_file} ]] && echo "Backup file ${backup_file} not found" && exit 1;
echo "tmp directory empty, backup in backup directory present" echo "tmp directory empty, backup in backup directory present"